Store all BLOB properties inline.

BLOB properties had a couple of intended purposes:
* Allow large payloads to be streamed directly to disk, and then be
handled by reference.
* Allow zero-copy handling.
* Keep the database values compact so we can avoid traversing large
BLOBS.

However, they came at the cost of code-complexity, and we lost all the
benefits of our storage layer, such as transactions.

Measurements showed, that for email (the intended primary usecase),
the overhead is hardly measurable, with most parts performing
better, or at least not worse. We additionally also gain file-system
independence, which may help on other platforms.

The biggest drawback is probably that large payloads need to be written
to disk twice, because of the synchronizer queue (once for the queue,
once for the actual data).

9 files changed, 86 insertions, 116 deletions

diff --git a/common/definitions.cpp b/common/definitions.cpp
index 1f4c0cf..b22137a 100644
--- a/common/definitions.cpp
+++ b/common/definitions.cpp
@@ -90,5 +90,5 @@ QString Sink::resourceStorageLocation(const QByteArray &resourceInstanceIdentifi
 
 qint64 Sink::latestDatabaseVersion()
 {
-    return 0;
+    return 1;
 }
diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h
index 77754c1..8a0daef 100644
--- a/common/domain/applicationdomaintype.h
+++ b/common/domain/applicationdomaintype.h
@@ -439,7 +439,8 @@ struct SINK_EXPORT Mail : public Entity {
     SINK_PROPERTY(bool, Unread, unread);
     SINK_PROPERTY(bool, Important, important);
     SINK_REFERENCE_PROPERTY(Folder, Folder, folder);
-    SINK_BLOB_PROPERTY(MimeMessage, mimeMessage);
+    // SINK_BLOB_PROPERTY(MimeMessage, mimeMessage);
+    SINK_PROPERTY(QByteArray, MimeMessage, mimeMessage);
     SINK_EXTRACTED_PROPERTY(bool, FullPayloadAvailable, fullPayloadAvailable);
     SINK_PROPERTY(bool, Draft, draft);
     SINK_PROPERTY(bool, Trash, trash);
diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp
index b1cb1d5..8f5a77d 100644
--- a/common/mailpreprocessor.cpp
+++ b/common/mailpreprocessor.cpp
@@ -29,65 +29,6 @@
 
 using namespace Sink;
 
-QString MailPropertyExtractor::getFilePathFromMimeMessagePath(const QString &s) const
-{
-    return s;
-}
-
-struct MimeMessageReader {
-    MimeMessageReader(const QString &mimeMessagePath)
-        : f(mimeMessagePath),
-        mapped(0)
-    {
-        if (mimeMessagePath.isNull()) {
-            SinkTrace() << "No mime message";
-            return;
-        }
-        SinkTrace() << "Updating indexed properties " << mimeMessagePath;
-        if (!f.open(QIODevice::ReadOnly)) {
-            SinkWarning() << "Failed to open the file: " << mimeMessagePath;
-            return;
-        }
-        if (!f.size()) {
-            SinkWarning() << "The file is empty.";
-            return;
-        }
-        mapped = f.map(0, f.size());
-        if (!mapped) {
-            SinkWarning() << "Failed to map the file: " << f.errorString();
-            return;
-        }
-    }
-
-    KMime::Message::Ptr mimeMessage()
-    {
-        if (!mapped) {
-            return {};
-        }
-        QByteArray result;
-        //Seek for end of headers
-        const auto content = QByteArray::fromRawData(reinterpret_cast<const char*>(mapped), f.size());
-        int pos = content.indexOf("\r\n\r\n", 0);
-        int offset = 2;
-        if (pos < 0) {
-            pos = content.indexOf("\n\n", 0);
-            offset = 1;
-        }
-        if (pos > -1) {
-            const auto header = content.left(pos + offset);    //header *must* end with "\n" !!
-            auto msg = KMime::Message::Ptr(new KMime::Message);
-            msg->setHead(KMime::CRLFtoLF(header));
-            msg->parse();
-            return msg;
-        }
-        SinkWarning() << "Failed to find end of headers" << content;
-        return {};
-    }
-
-    QFile f;
-    uchar *mapped;
-};
-
 static Sink::ApplicationDomain::Mail::Contact getContact(const KMime::Headers::Generics::MailboxList *header)
 {
     const auto name = header->displayNames().isEmpty() ? QString() : header->displayNames().first();
@@ -104,8 +45,18 @@ static QList<Sink::ApplicationDomain::Mail::Contact> getContactList(const KMime:
     return list;
 }
 
-static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, KMime::Message::Ptr msg)
+void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data)
 {
+    if (data.isEmpty()) {
+        return;
+    }
+    auto msg = KMime::Message::Ptr(new KMime::Message);
+    msg->setHead(KMime::CRLFtoLF(data));
+    msg->parse();
+    if (!msg) {
+        return;
+    }
+
     mail.setExtractedSubject(msg->subject(true)->asUnicodeString());
     mail.setExtractedSender(getContact(msg->from(true)));
     mail.setExtractedTo(getContactList(msg->to(true)));
@@ -156,19 +107,11 @@ static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, KMime:
 
 void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail)
 {
-    MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(mail.getMimeMessagePath()));
-    auto msg = mimeMessageReader.mimeMessage();
-    if (msg) {
-        updatedIndexedProperties(mail, msg);
-    }
+    updatedIndexedProperties(mail, mail.getMimeMessage());
 }
 
 void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail)
 {
-    MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(newMail.getMimeMessagePath()));
-    auto msg = mimeMessageReader.mimeMessage();
-    if (msg) {
-        updatedIndexedProperties(newMail, msg);
-    }
+    updatedIndexedProperties(newMail, newMail.getMimeMessage());
 }
 
diff --git a/common/mailpreprocessor.h b/common/mailpreprocessor.h
index a24a8d3..d2e79ca 100644
--- a/common/mailpreprocessor.h
+++ b/common/mailpreprocessor.h
@@ -27,6 +27,6 @@ public:
     virtual void newEntity(Sink::ApplicationDomain::Mail &mail) Q_DECL_OVERRIDE;
     virtual void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail) Q_DECL_OVERRIDE;
 protected:
-    virtual QString getFilePathFromMimeMessagePath(const QString &) const;
+    static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data);
 };
 
diff --git a/examples/maildirresource/facade.cpp b/examples/maildirresource/facade.cpp
index 5ea3d98..ea02968 100644
--- a/examples/maildirresource/facade.cpp
+++ b/examples/maildirresource/facade.cpp
@@ -31,7 +31,7 @@ MaildirResourceMailFacade::MaildirResourceMailFacade(const Sink::ResourceContext
         Sink::Log::Context ctx{"maildirfacade"};
         if (value.hasProperty(Sink::ApplicationDomain::Mail::MimeMessage::name)) {
             auto mail = Sink::ApplicationDomain::Mail{value};
-            const auto mimeMessage = mail.getMimeMessagePath();
+            const auto mimeMessage = mail.getMimeMessage();
             //Transform the mime message property into the actual path on disk.
             auto parts = mimeMessage.split('/');
             auto key = parts.takeLast();
@@ -45,7 +45,10 @@ MaildirResourceMailFacade::MaildirResourceMailFacade(const Sink::ResourceContext
                 SinkErrorCtx(ctx) << "Failed to find message. Directory: " << path << "Key: " << key << "Number of matching files: " << list.size();
                 mail.setProperty(Sink::ApplicationDomain::Mail::MimeMessage::name, QVariant());
             } else {
-                mail.setMimeMessagePath(list.at(0).filePath());
+                QFile file{list.at(0).filePath()};
+                if (file.open(QIODevice::ReadOnly)) {
+                    mail.setMimeMessage(file.readAll());
+                }
             }
         }
         value.setChangedProperties(QSet<QByteArray>());
diff --git a/examples/maildirresource/maildirresource.cpp b/examples/maildirresource/maildirresource.cpp
index 735665a..4c94d1d 100644
--- a/examples/maildirresource/maildirresource.cpp
+++ b/examples/maildirresource/maildirresource.cpp
@@ -54,8 +54,7 @@ static QString getFilePathFromMimeMessagePath(const QString &mimeMessagePath)
     QDir dir(path);
     const QFileInfoList list = dir.entryInfoList(QStringList() << (key+"*"), QDir::Files);
     if (list.size() != 1) {
-        SinkWarning() << "Failed to find message " << mimeMessagePath;
-        SinkWarning() << "Failed to find message " << path;
+        SinkWarning() << "Failed to find message. Property value:" << mimeMessagePath << "Assembled path: " << path;
         return QString();
     }
     return list.first().filePath();
@@ -63,10 +62,25 @@ static QString getFilePathFromMimeMessagePath(const QString &mimeMessagePath)
 
 class MaildirMailPropertyExtractor : public MailPropertyExtractor
 {
+    void update(Sink::ApplicationDomain::Mail &mail)
+    {
+        QFile file{::getFilePathFromMimeMessagePath(mail.getMimeMessage())};
+        if (file.open(QIODevice::ReadOnly)) {
+            updatedIndexedProperties(mail, file.readAll());
+        } else {
+            SinkWarning() << "Failed to open file message " << mail.getMimeMessage();
+        }
+    }
+
 protected:
-    virtual QString getFilePathFromMimeMessagePath(const QString &mimeMessagePath) const Q_DECL_OVERRIDE
+    void newEntity(Sink::ApplicationDomain::Mail &mail) Q_DECL_OVERRIDE
     {
-        return ::getFilePathFromMimeMessagePath(mimeMessagePath);
+        update(mail);
+    }
+
+    void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail) Q_DECL_OVERRIDE
+    {
+        update(newMail);
     }
 };
 
@@ -92,6 +106,18 @@ public:
         return folderPath;
     }
 
+    QString storeMessage(const QByteArray &data, const QByteArray &folder)
+    {
+        const auto path = getPath(folder);
+        KPIM::Maildir maildir(path, false);
+        if (!maildir.isValid(true)) {
+            SinkWarning() << "Maildir is not existing: " << path;
+        }
+        SinkWarning() << "Storing message: " << data;
+        auto identifier = maildir.addEntry(data);
+        return path + "/" + identifier;
+    }
+
     QString moveMessage(const QString &oldPath, const QByteArray &folder)
     {
         if (oldPath.startsWith(Sink::temporaryFileLocation())) {
@@ -125,15 +151,21 @@ public:
         }
     }
 
+    bool isPath(const QByteArray &data)
+    {
+        return data.startsWith('/');
+    }
+
     void newEntity(Sink::ApplicationDomain::ApplicationDomainType &newEntity) Q_DECL_OVERRIDE
     {
         auto mail = newEntity.cast<ApplicationDomain::Mail>();
-        const auto mimeMessage = mail.getMimeMessagePath();
+        const auto mimeMessage = mail.getMimeMessage();
         if (!mimeMessage.isNull()) {
-            const auto path = moveMessage(mimeMessage, mail.getFolder());
-            auto blob = ApplicationDomain::BLOB{path};
-            blob.isExternal = false;
-            mail.setProperty(ApplicationDomain::Mail::MimeMessage::name, QVariant::fromValue(blob));
+            if (isPath(mimeMessage)) {
+                mail.setMimeMessage(moveMessage(mimeMessage, mail.getFolder()).toUtf8());
+            } else {
+                mail.setMimeMessage(storeMessage(mimeMessage, mail.getFolder()).toUtf8());
+            }
         }
     }
 
@@ -141,28 +173,28 @@ public:
     {
         auto newMail = newEntity.cast<ApplicationDomain::Mail>();
         const ApplicationDomain::Mail oldMail{oldEntity};
-        const auto mimeMessage = newMail.getMimeMessagePath();
         const auto newFolder = newMail.getFolder();
-        const bool mimeMessageChanged = !mimeMessage.isNull() && mimeMessage != oldMail.getMimeMessagePath();
         const bool folderChanged = !newFolder.isNull() && newFolder != oldMail.getFolder();
-        if (mimeMessageChanged || folderChanged) {
-            SinkTrace() << "Moving mime message: " << mimeMessageChanged << folderChanged;
-            auto newPath = moveMessage(mimeMessage, newMail.getFolder());
-            if (newPath != oldMail.getMimeMessagePath()) {
-                const auto oldPath = getFilePathFromMimeMessagePath(oldMail.getMimeMessagePath());
-                auto blob = ApplicationDomain::BLOB{newPath};
-                blob.isExternal = false;
-                newMail.setProperty(ApplicationDomain::Mail::MimeMessage::name, QVariant::fromValue(blob));
+        if (!newMail.getMimeMessage().isNull() || folderChanged) {
+            const auto data = newMail.getMimeMessage();
+            if (isPath(data)) {
+                auto newPath = moveMessage(data, newMail.getFolder());
+                if (newPath != oldMail.getMimeMessage()) {
+                    newMail.setMimeMessage(newPath.toUtf8());
+                    //Remove the olde mime message if there is a new one
+                    QFile::remove(getFilePathFromMimeMessagePath(oldMail.getMimeMessage()));
+                }
+            } else {
+                newMail.setMimeMessage(storeMessage(data, newMail.getFolder()).toUtf8());
                 //Remove the olde mime message if there is a new one
-                QFile::remove(oldPath);
+                QFile::remove(getFilePathFromMimeMessagePath(oldMail.getMimeMessage()));
             }
         }
 
-        auto mimeMessagePath = newMail.getMimeMessagePath();
+        auto mimeMessagePath = newMail.getMimeMessage();
         const auto maildirPath = getPath(newMail.getFolder());
         KPIM::Maildir maildir(maildirPath, false);
-        const auto file = getFilePathFromMimeMessagePath(mimeMessagePath);
-        QString identifier = KPIM::Maildir::getKeyFromFile(file);
+        QString identifier = KPIM::Maildir::getKeyFromFile(getFilePathFromMimeMessagePath(mimeMessagePath));
 
         //get flags from
         KPIM::Maildir::Flags flags;
@@ -179,7 +211,7 @@ public:
     void deletedEntity(const Sink::ApplicationDomain::ApplicationDomainType &oldEntity) Q_DECL_OVERRIDE
     {
         const ApplicationDomain::Mail oldMail{oldEntity};
-        const auto filePath = getFilePathFromMimeMessagePath(oldMail.getMimeMessagePath());
+        const auto filePath = getFilePathFromMimeMessagePath(oldMail.getMimeMessage());
         QFile::remove(filePath);
     }
     QByteArray mResourceInstanceIdentifier;
@@ -327,9 +359,7 @@ public:
             mail.setFolder(folderLocalId);
             //We only store the directory path + key, so we facade can add the changing bits (flags)
             auto path = KPIM::Maildir::getDirectoryFromFile(filePath) + maildirKey;
-            auto blob = ApplicationDomain::BLOB{path};
-            blob.isExternal = false;
-            mail.setProperty(ApplicationDomain::Mail::MimeMessage::name, QVariant::fromValue(blob));
+            mail.setMimeMessage(path.toUtf8());
             mail.setUnread(!flags.testFlag(KPIM::Maildir::Seen));
             mail.setImportant(flags.testFlag(KPIM::Maildir::Flagged));
             mail.setExtractedFullPayloadAvailable(true);
@@ -396,7 +426,7 @@ public:
     KAsync::Job<QByteArray> replay(const ApplicationDomain::Mail &mail, Sink::Operation operation, const QByteArray &oldRemoteId, const QList<QByteArray> &changedProperties) Q_DECL_OVERRIDE
     {
         if (operation == Sink::Operation_Creation) {
-            const auto remoteId = getFilePathFromMimeMessagePath(mail.getMimeMessagePath());
+            const auto remoteId = getFilePathFromMimeMessagePath(mail.getMimeMessage());
             SinkTrace() << "Mail created: " << remoteId;
             return KAsync::value(remoteId.toUtf8());
         } else if (operation == Sink::Operation_Removal) {
@@ -404,7 +434,7 @@ public:
             return KAsync::null<QByteArray>();
         } else if (operation == Sink::Operation_Modification) {
             SinkTrace() << "Modifying a mail: " << oldRemoteId;
-            const auto remoteId = getFilePathFromMimeMessagePath(mail.getMimeMessagePath());
+            const auto remoteId = getFilePathFromMimeMessagePath(mail.getMimeMessage());
             return KAsync::value(remoteId.toUtf8());
         }
         return KAsync::null<QByteArray>();
@@ -460,7 +490,7 @@ protected:
 
         if (domainType == ENTITY_TYPE_MAIL) {
             auto mail = entityStore.readLatest<Sink::ApplicationDomain::Mail>(entityId);
-            const auto filePath = getFilePathFromMimeMessagePath(mail.getMimeMessagePath());
+            const auto filePath = getFilePathFromMimeMessagePath(mail.getMimeMessage());
 
             if (inspectionType == Sink::ResourceControl::Inspection::PropertyInspectionType) {
                 if (property == "unread") {
diff --git a/tests/interresourcemovetest.cpp b/tests/interresourcemovetest.cpp
index 174befc..37e040e 100644
--- a/tests/interresourcemovetest.cpp
+++ b/tests/interresourcemovetest.cpp
@@ -106,7 +106,6 @@ private slots:
             auto list = Sink::Store::read<Mail>(query.filter<Mail::MessageId>(testuid));
             QCOMPARE(list.size(), 1);
             const auto mail = list.first();
-            QVERIFY(!mail.getMimeMessagePath().isEmpty());
             QCOMPARE(mail.getSubject(), subject);
             QCOMPARE(mail.getMimeMessage(), mimeMessage);
         }
@@ -153,7 +152,6 @@ private slots:
             auto list = Sink::Store::read<Mail>(query.filter<Mail::MessageId>(testuid));
             QCOMPARE(list.size(), 1);
             const auto mail = list.first();
-            QVERIFY(!mail.getMimeMessagePath().isEmpty());
             QCOMPARE(mail.getSubject(), subject);
             QCOMPARE(mail.getMimeMessage(), mimeMessage);
         }
@@ -164,7 +162,6 @@ private slots:
             auto list = Sink::Store::read<Mail>(query.filter<Mail::MessageId>(testuid));
             QCOMPARE(list.size(), 1);
             const auto mail = list.first();
-            QVERIFY(!mail.getMimeMessagePath().isEmpty());
             QCOMPARE(mail.getSubject(), subject);
             QCOMPARE(mail.getMimeMessage(), mimeMessage);
         }
diff --git a/tests/mailsynctest.cpp b/tests/mailsynctest.cpp
index 811eace..764b006 100644
--- a/tests/mailsynctest.cpp
+++ b/tests/mailsynctest.cpp
@@ -318,7 +318,7 @@ void MailSyncTest::testResyncMails()
         ASYNCCOMPARE(mails.size(), 1);
         auto mail = mails.first();
         ASYNCVERIFY(!mail->getSubject().isEmpty());
-        ASYNCVERIFY(!mail->getMimeMessagePath().isEmpty());
+        ASYNCVERIFY(!mail->getMimeMessage().isEmpty());
         return KAsync::null();
     });
     VERIFYEXEC(job);
diff --git a/tests/mailtest.cpp b/tests/mailtest.cpp
index 1c24268..88cacd2 100644
--- a/tests/mailtest.cpp
+++ b/tests/mailtest.cpp
@@ -164,7 +164,6 @@ void MailTest::testCreateModifyDeleteMail()
                 auto mail = *mails.first();
                 QCOMPARE(mail.getSubject(), subject);
                 QCOMPARE(mail.getFolder(), folder.identifier());
-                QVERIFY(QFile(mail.getMimeMessagePath()).exists());
                 KMime::Message m;
                 m.setContent(KMime::CRLFtoLF(mail.getMimeMessage()));
                 m.parse();
@@ -193,7 +192,6 @@ void MailTest::testCreateModifyDeleteMail()
                 auto mail = *mails.first();
                 QCOMPARE(mail.getSubject(), subject2);
                 QCOMPARE(mail.getFolder(), folder.identifier());
-                QVERIFY(QFile(mail.getMimeMessagePath()).exists());
                 KMime::Message m;
                 m.setContent(KMime::CRLFtoLF(mail.getMimeMessage()));
                 m.parse();
@@ -251,8 +249,7 @@ void MailTest::testMoveMail()
                 auto mail = *mails.first();
                 modifiedMail = mail;
                 QCOMPARE(mail.getFolder(), folder.identifier());
-                SinkWarning() << "path: " << mail.getMimeMessagePath();
-                QVERIFY(QFile(mail.getMimeMessagePath()).exists());
+                QVERIFY(!mail.getMimeMessage().isEmpty());
             });
         VERIFYEXEC(job);
     }
@@ -269,8 +266,7 @@ void MailTest::testMoveMail()
                 QCOMPARE(mails.size(), 1);
                 auto mail = *mails.first();
                 QCOMPARE(mail.getFolder(), folder1.identifier());
-                QVERIFY(QFile(mail.getMimeMessagePath()).exists());
-                SinkTrace() << "Mime message path: " << mail.getMimeMessagePath();
+                QVERIFY(!mail.getMimeMessage().isEmpty());
             });
         VERIFYEXEC(job);
     }
@@ -324,7 +320,7 @@ void MailTest::testMarkMailAsRead()
             auto mail = mails.first();
             ASYNCVERIFY(!mail->getSubject().isEmpty());
             ASYNCCOMPARE(mail->getUnread(), false);
-            ASYNCVERIFY(QFileInfo(mail->getMimeMessagePath()).exists());
+            ASYNCVERIFY(!mail->getMimeMessage().isEmpty());
             return KAsync::null<void>();
         });
     VERIFYEXEC(job2);