Store all BLOB properties inline.

BLOB properties had a couple of intended purposes:
* Allow large payloads to be streamed directly to disk, and then be
handled by reference.
* Allow zero-copy handling.
* Keep the database values compact so we can avoid traversing large
BLOBS.

However, they came at the cost of code-complexity, and we lost all the
benefits of our storage layer, such as transactions.

Measurements showed, that for email (the intended primary usecase),
the overhead is hardly measurable, with most parts performing
better, or at least not worse. We additionally also gain file-system
independence, which may help on other platforms.

The biggest drawback is probably that large payloads need to be written
to disk twice, because of the synchronizer queue (once for the queue,
once for the actual data).

4 files changed, 17 insertions, 73 deletions

diff --git a/common/definitions.cpp b/common/definitions.cpp
index 1f4c0cf..b22137a 100644
--- a/common/definitions.cpp
+++ b/common/definitions.cpp
@@ -90,5 +90,5 @@ QString Sink::resourceStorageLocation(const QByteArray &resourceInstanceIdentifi
 
 qint64 Sink::latestDatabaseVersion()
 {
-    return 0;
+    return 1;
 }
diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h
index 77754c1..8a0daef 100644
--- a/common/domain/applicationdomaintype.h
+++ b/common/domain/applicationdomaintype.h
@@ -439,7 +439,8 @@ struct SINK_EXPORT Mail : public Entity {
     SINK_PROPERTY(bool, Unread, unread);
     SINK_PROPERTY(bool, Important, important);
     SINK_REFERENCE_PROPERTY(Folder, Folder, folder);
-    SINK_BLOB_PROPERTY(MimeMessage, mimeMessage);
+    // SINK_BLOB_PROPERTY(MimeMessage, mimeMessage);
+    SINK_PROPERTY(QByteArray, MimeMessage, mimeMessage);
     SINK_EXTRACTED_PROPERTY(bool, FullPayloadAvailable, fullPayloadAvailable);
     SINK_PROPERTY(bool, Draft, draft);
     SINK_PROPERTY(bool, Trash, trash);
diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp
index b1cb1d5..8f5a77d 100644
--- a/common/mailpreprocessor.cpp
+++ b/common/mailpreprocessor.cpp
@@ -29,65 +29,6 @@
 
 using namespace Sink;
 
-QString MailPropertyExtractor::getFilePathFromMimeMessagePath(const QString &s) const
-{
-    return s;
-}
-
-struct MimeMessageReader {
-    MimeMessageReader(const QString &mimeMessagePath)
-        : f(mimeMessagePath),
-        mapped(0)
-    {
-        if (mimeMessagePath.isNull()) {
-            SinkTrace() << "No mime message";
-            return;
-        }
-        SinkTrace() << "Updating indexed properties " << mimeMessagePath;
-        if (!f.open(QIODevice::ReadOnly)) {
-            SinkWarning() << "Failed to open the file: " << mimeMessagePath;
-            return;
-        }
-        if (!f.size()) {
-            SinkWarning() << "The file is empty.";
-            return;
-        }
-        mapped = f.map(0, f.size());
-        if (!mapped) {
-            SinkWarning() << "Failed to map the file: " << f.errorString();
-            return;
-        }
-    }
-
-    KMime::Message::Ptr mimeMessage()
-    {
-        if (!mapped) {
-            return {};
-        }
-        QByteArray result;
-        //Seek for end of headers
-        const auto content = QByteArray::fromRawData(reinterpret_cast<const char*>(mapped), f.size());
-        int pos = content.indexOf("\r\n\r\n", 0);
-        int offset = 2;
-        if (pos < 0) {
-            pos = content.indexOf("\n\n", 0);
-            offset = 1;
-        }
-        if (pos > -1) {
-            const auto header = content.left(pos + offset);    //header *must* end with "\n" !!
-            auto msg = KMime::Message::Ptr(new KMime::Message);
-            msg->setHead(KMime::CRLFtoLF(header));
-            msg->parse();
-            return msg;
-        }
-        SinkWarning() << "Failed to find end of headers" << content;
-        return {};
-    }
-
-    QFile f;
-    uchar *mapped;
-};
-
 static Sink::ApplicationDomain::Mail::Contact getContact(const KMime::Headers::Generics::MailboxList *header)
 {
     const auto name = header->displayNames().isEmpty() ? QString() : header->displayNames().first();
@@ -104,8 +45,18 @@ static QList<Sink::ApplicationDomain::Mail::Contact> getContactList(const KMime:
     return list;
 }
 
-static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, KMime::Message::Ptr msg)
+void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data)
 {
+    if (data.isEmpty()) {
+        return;
+    }
+    auto msg = KMime::Message::Ptr(new KMime::Message);
+    msg->setHead(KMime::CRLFtoLF(data));
+    msg->parse();
+    if (!msg) {
+        return;
+    }
+
     mail.setExtractedSubject(msg->subject(true)->asUnicodeString());
     mail.setExtractedSender(getContact(msg->from(true)));
     mail.setExtractedTo(getContactList(msg->to(true)));
@@ -156,19 +107,11 @@ static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, KMime:
 
 void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail)
 {
-    MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(mail.getMimeMessagePath()));
-    auto msg = mimeMessageReader.mimeMessage();
-    if (msg) {
-        updatedIndexedProperties(mail, msg);
-    }
+    updatedIndexedProperties(mail, mail.getMimeMessage());
 }
 
 void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail)
 {
-    MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(newMail.getMimeMessagePath()));
-    auto msg = mimeMessageReader.mimeMessage();
-    if (msg) {
-        updatedIndexedProperties(newMail, msg);
-    }
+    updatedIndexedProperties(newMail, newMail.getMimeMessage());
 }
 
diff --git a/common/mailpreprocessor.h b/common/mailpreprocessor.h
index a24a8d3..d2e79ca 100644
--- a/common/mailpreprocessor.h
+++ b/common/mailpreprocessor.h
@@ -27,6 +27,6 @@ public:
     virtual void newEntity(Sink::ApplicationDomain::Mail &mail) Q_DECL_OVERRIDE;
     virtual void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail) Q_DECL_OVERRIDE;
 protected:
-    virtual QString getFilePathFromMimeMessagePath(const QString &) const;
+    static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data);
 };