From 9b84aff4b68c3cef3328c85ac12418048b169cee Mon Sep 17 00:00:00 2001
From: Christian Mollekopf <chrigi_1@fastmail.fm>
Date: Thu, 25 Jan 2018 16:29:00 +0100
Subject: Store all BLOB properties inline.

BLOB properties had a couple of intended purposes:
* Allow large payloads to be streamed directly to disk, and then be
handled by reference.
* Allow zero-copy handling.
* Keep the database values compact so we can avoid traversing large
BLOBS.

However, they came at the cost of code-complexity, and we lost all the
benefits of our storage layer, such as transactions.

Measurements showed, that for email (the intended primary usecase),
the overhead is hardly measurable, with most parts performing
better, or at least not worse. We additionally also gain file-system
independence, which may help on other platforms.

The biggest drawback is probably that large payloads need to be written
to disk twice, because of the synchronizer queue (once for the queue,
once for the actual data).
---
 common/mailpreprocessor.cpp | 83 +++++++--------------------------------------
 1 file changed, 13 insertions(+), 70 deletions(-)

(limited to 'common/mailpreprocessor.cpp')

diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp
index b1cb1d5..8f5a77d 100644
--- a/common/mailpreprocessor.cpp
+++ b/common/mailpreprocessor.cpp
@@ -29,65 +29,6 @@
 
 using namespace Sink;
 
-QString MailPropertyExtractor::getFilePathFromMimeMessagePath(const QString &s) const
-{
-    return s;
-}
-
-struct MimeMessageReader {
-    MimeMessageReader(const QString &mimeMessagePath)
-        : f(mimeMessagePath),
-        mapped(0)
-    {
-        if (mimeMessagePath.isNull()) {
-            SinkTrace() << "No mime message";
-            return;
-        }
-        SinkTrace() << "Updating indexed properties " << mimeMessagePath;
-        if (!f.open(QIODevice::ReadOnly)) {
-            SinkWarning() << "Failed to open the file: " << mimeMessagePath;
-            return;
-        }
-        if (!f.size()) {
-            SinkWarning() << "The file is empty.";
-            return;
-        }
-        mapped = f.map(0, f.size());
-        if (!mapped) {
-            SinkWarning() << "Failed to map the file: " << f.errorString();
-            return;
-        }
-    }
-
-    KMime::Message::Ptr mimeMessage()
-    {
-        if (!mapped) {
-            return {};
-        }
-        QByteArray result;
-        //Seek for end of headers
-        const auto content = QByteArray::fromRawData(reinterpret_cast<const char*>(mapped), f.size());
-        int pos = content.indexOf("\r\n\r\n", 0);
-        int offset = 2;
-        if (pos < 0) {
-            pos = content.indexOf("\n\n", 0);
-            offset = 1;
-        }
-        if (pos > -1) {
-            const auto header = content.left(pos + offset);    //header *must* end with "\n" !!
-            auto msg = KMime::Message::Ptr(new KMime::Message);
-            msg->setHead(KMime::CRLFtoLF(header));
-            msg->parse();
-            return msg;
-        }
-        SinkWarning() << "Failed to find end of headers" << content;
-        return {};
-    }
-
-    QFile f;
-    uchar *mapped;
-};
-
 static Sink::ApplicationDomain::Mail::Contact getContact(const KMime::Headers::Generics::MailboxList *header)
 {
     const auto name = header->displayNames().isEmpty() ? QString() : header->displayNames().first();
@@ -104,8 +45,18 @@ static QList<Sink::ApplicationDomain::Mail::Contact> getContactList(const KMime:
     return list;
 }
 
-static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, KMime::Message::Ptr msg)
+void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data)
 {
+    if (data.isEmpty()) {
+        return;
+    }
+    auto msg = KMime::Message::Ptr(new KMime::Message);
+    msg->setHead(KMime::CRLFtoLF(data));
+    msg->parse();
+    if (!msg) {
+        return;
+    }
+
     mail.setExtractedSubject(msg->subject(true)->asUnicodeString());
     mail.setExtractedSender(getContact(msg->from(true)));
     mail.setExtractedTo(getContactList(msg->to(true)));
@@ -156,19 +107,11 @@ static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, KMime:
 
 void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail)
 {
-    MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(mail.getMimeMessagePath()));
-    auto msg = mimeMessageReader.mimeMessage();
-    if (msg) {
-        updatedIndexedProperties(mail, msg);
-    }
+    updatedIndexedProperties(mail, mail.getMimeMessage());
 }
 
 void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail)
 {
-    MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(newMail.getMimeMessagePath()));
-    auto msg = mimeMessageReader.mimeMessage();
-    if (msg) {
-        updatedIndexedProperties(newMail, msg);
-    }
+    updatedIndexedProperties(newMail, newMail.getMimeMessage());
 }
 
-- 
cgit v1.2.3