From 9b84aff4b68c3cef3328c85ac12418048b169cee Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 25 Jan 2018 16:29:00 +0100 Subject: Store all BLOB properties inline. BLOB properties had a couple of intended purposes: * Allow large payloads to be streamed directly to disk, and then be handled by reference. * Allow zero-copy handling. * Keep the database values compact so we can avoid traversing large BLOBS. However, they came at the cost of code-complexity, and we lost all the benefits of our storage layer, such as transactions. Measurements showed, that for email (the intended primary usecase), the overhead is hardly measurable, with most parts performing better, or at least not worse. We additionally also gain file-system independence, which may help on other platforms. The biggest drawback is probably that large payloads need to be written to disk twice, because of the synchronizer queue (once for the queue, once for the actual data). --- common/definitions.cpp | 2 +- common/domain/applicationdomaintype.h | 3 +- common/mailpreprocessor.cpp | 83 ++++++----------------------------- common/mailpreprocessor.h | 2 +- 4 files changed, 17 insertions(+), 73 deletions(-) (limited to 'common') diff --git a/common/definitions.cpp b/common/definitions.cpp index 1f4c0cf..b22137a 100644 --- a/common/definitions.cpp +++ b/common/definitions.cpp @@ -90,5 +90,5 @@ QString Sink::resourceStorageLocation(const QByteArray &resourceInstanceIdentifi qint64 Sink::latestDatabaseVersion() { - return 0; + return 1; } diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h index 77754c1..8a0daef 100644 --- a/common/domain/applicationdomaintype.h +++ b/common/domain/applicationdomaintype.h @@ -439,7 +439,8 @@ struct SINK_EXPORT Mail : public Entity { SINK_PROPERTY(bool, Unread, unread); SINK_PROPERTY(bool, Important, important); SINK_REFERENCE_PROPERTY(Folder, Folder, folder); - SINK_BLOB_PROPERTY(MimeMessage, mimeMessage); + // SINK_BLOB_PROPERTY(MimeMessage, mimeMessage); + SINK_PROPERTY(QByteArray, MimeMessage, mimeMessage); SINK_EXTRACTED_PROPERTY(bool, FullPayloadAvailable, fullPayloadAvailable); SINK_PROPERTY(bool, Draft, draft); SINK_PROPERTY(bool, Trash, trash); diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp index b1cb1d5..8f5a77d 100644 --- a/common/mailpreprocessor.cpp +++ b/common/mailpreprocessor.cpp @@ -29,65 +29,6 @@ using namespace Sink; -QString MailPropertyExtractor::getFilePathFromMimeMessagePath(const QString &s) const -{ - return s; -} - -struct MimeMessageReader { - MimeMessageReader(const QString &mimeMessagePath) - : f(mimeMessagePath), - mapped(0) - { - if (mimeMessagePath.isNull()) { - SinkTrace() << "No mime message"; - return; - } - SinkTrace() << "Updating indexed properties " << mimeMessagePath; - if (!f.open(QIODevice::ReadOnly)) { - SinkWarning() << "Failed to open the file: " << mimeMessagePath; - return; - } - if (!f.size()) { - SinkWarning() << "The file is empty."; - return; - } - mapped = f.map(0, f.size()); - if (!mapped) { - SinkWarning() << "Failed to map the file: " << f.errorString(); - return; - } - } - - KMime::Message::Ptr mimeMessage() - { - if (!mapped) { - return {}; - } - QByteArray result; - //Seek for end of headers - const auto content = QByteArray::fromRawData(reinterpret_cast(mapped), f.size()); - int pos = content.indexOf("\r\n\r\n", 0); - int offset = 2; - if (pos < 0) { - pos = content.indexOf("\n\n", 0); - offset = 1; - } - if (pos > -1) { - const auto header = content.left(pos + offset); //header *must* end with "\n" !! - auto msg = KMime::Message::Ptr(new KMime::Message); - msg->setHead(KMime::CRLFtoLF(header)); - msg->parse(); - return msg; - } - SinkWarning() << "Failed to find end of headers" << content; - return {}; - } - - QFile f; - uchar *mapped; -}; - static Sink::ApplicationDomain::Mail::Contact getContact(const KMime::Headers::Generics::MailboxList *header) { const auto name = header->displayNames().isEmpty() ? QString() : header->displayNames().first(); @@ -104,8 +45,18 @@ static QList getContactList(const KMime: return list; } -static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, KMime::Message::Ptr msg) +void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data) { + if (data.isEmpty()) { + return; + } + auto msg = KMime::Message::Ptr(new KMime::Message); + msg->setHead(KMime::CRLFtoLF(data)); + msg->parse(); + if (!msg) { + return; + } + mail.setExtractedSubject(msg->subject(true)->asUnicodeString()); mail.setExtractedSender(getContact(msg->from(true))); mail.setExtractedTo(getContactList(msg->to(true))); @@ -156,19 +107,11 @@ static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, KMime: void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail) { - MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(mail.getMimeMessagePath())); - auto msg = mimeMessageReader.mimeMessage(); - if (msg) { - updatedIndexedProperties(mail, msg); - } + updatedIndexedProperties(mail, mail.getMimeMessage()); } void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail) { - MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(newMail.getMimeMessagePath())); - auto msg = mimeMessageReader.mimeMessage(); - if (msg) { - updatedIndexedProperties(newMail, msg); - } + updatedIndexedProperties(newMail, newMail.getMimeMessage()); } diff --git a/common/mailpreprocessor.h b/common/mailpreprocessor.h index a24a8d3..d2e79ca 100644 --- a/common/mailpreprocessor.h +++ b/common/mailpreprocessor.h @@ -27,6 +27,6 @@ public: virtual void newEntity(Sink::ApplicationDomain::Mail &mail) Q_DECL_OVERRIDE; virtual void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail) Q_DECL_OVERRIDE; protected: - virtual QString getFilePathFromMimeMessagePath(const QString &) const; + static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data); }; -- cgit v1.2.3