From 9b84aff4b68c3cef3328c85ac12418048b169cee Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 25 Jan 2018 16:29:00 +0100 Subject: Store all BLOB properties inline. BLOB properties had a couple of intended purposes: * Allow large payloads to be streamed directly to disk, and then be handled by reference. * Allow zero-copy handling. * Keep the database values compact so we can avoid traversing large BLOBS. However, they came at the cost of code-complexity, and we lost all the benefits of our storage layer, such as transactions. Measurements showed, that for email (the intended primary usecase), the overhead is hardly measurable, with most parts performing better, or at least not worse. We additionally also gain file-system independence, which may help on other platforms. The biggest drawback is probably that large payloads need to be written to disk twice, because of the synchronizer queue (once for the queue, once for the actual data). --- examples/maildirresource/facade.cpp | 7 ++- examples/maildirresource/maildirresource.cpp | 90 ++++++++++++++++++---------- 2 files changed, 65 insertions(+), 32 deletions(-) (limited to 'examples') diff --git a/examples/maildirresource/facade.cpp b/examples/maildirresource/facade.cpp index 5ea3d98..ea02968 100644 --- a/examples/maildirresource/facade.cpp +++ b/examples/maildirresource/facade.cpp @@ -31,7 +31,7 @@ MaildirResourceMailFacade::MaildirResourceMailFacade(const Sink::ResourceContext Sink::Log::Context ctx{"maildirfacade"}; if (value.hasProperty(Sink::ApplicationDomain::Mail::MimeMessage::name)) { auto mail = Sink::ApplicationDomain::Mail{value}; - const auto mimeMessage = mail.getMimeMessagePath(); + const auto mimeMessage = mail.getMimeMessage(); //Transform the mime message property into the actual path on disk. auto parts = mimeMessage.split('/'); auto key = parts.takeLast(); @@ -45,7 +45,10 @@ MaildirResourceMailFacade::MaildirResourceMailFacade(const Sink::ResourceContext SinkErrorCtx(ctx) << "Failed to find message. Directory: " << path << "Key: " << key << "Number of matching files: " << list.size(); mail.setProperty(Sink::ApplicationDomain::Mail::MimeMessage::name, QVariant()); } else { - mail.setMimeMessagePath(list.at(0).filePath()); + QFile file{list.at(0).filePath()}; + if (file.open(QIODevice::ReadOnly)) { + mail.setMimeMessage(file.readAll()); + } } } value.setChangedProperties(QSet()); diff --git a/examples/maildirresource/maildirresource.cpp b/examples/maildirresource/maildirresource.cpp index 735665a..4c94d1d 100644 --- a/examples/maildirresource/maildirresource.cpp +++ b/examples/maildirresource/maildirresource.cpp @@ -54,8 +54,7 @@ static QString getFilePathFromMimeMessagePath(const QString &mimeMessagePath) QDir dir(path); const QFileInfoList list = dir.entryInfoList(QStringList() << (key+"*"), QDir::Files); if (list.size() != 1) { - SinkWarning() << "Failed to find message " << mimeMessagePath; - SinkWarning() << "Failed to find message " << path; + SinkWarning() << "Failed to find message. Property value:" << mimeMessagePath << "Assembled path: " << path; return QString(); } return list.first().filePath(); @@ -63,10 +62,25 @@ static QString getFilePathFromMimeMessagePath(const QString &mimeMessagePath) class MaildirMailPropertyExtractor : public MailPropertyExtractor { + void update(Sink::ApplicationDomain::Mail &mail) + { + QFile file{::getFilePathFromMimeMessagePath(mail.getMimeMessage())}; + if (file.open(QIODevice::ReadOnly)) { + updatedIndexedProperties(mail, file.readAll()); + } else { + SinkWarning() << "Failed to open file message " << mail.getMimeMessage(); + } + } + protected: - virtual QString getFilePathFromMimeMessagePath(const QString &mimeMessagePath) const Q_DECL_OVERRIDE + void newEntity(Sink::ApplicationDomain::Mail &mail) Q_DECL_OVERRIDE { - return ::getFilePathFromMimeMessagePath(mimeMessagePath); + update(mail); + } + + void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail) Q_DECL_OVERRIDE + { + update(newMail); } }; @@ -92,6 +106,18 @@ public: return folderPath; } + QString storeMessage(const QByteArray &data, const QByteArray &folder) + { + const auto path = getPath(folder); + KPIM::Maildir maildir(path, false); + if (!maildir.isValid(true)) { + SinkWarning() << "Maildir is not existing: " << path; + } + SinkWarning() << "Storing message: " << data; + auto identifier = maildir.addEntry(data); + return path + "/" + identifier; + } + QString moveMessage(const QString &oldPath, const QByteArray &folder) { if (oldPath.startsWith(Sink::temporaryFileLocation())) { @@ -125,15 +151,21 @@ public: } } + bool isPath(const QByteArray &data) + { + return data.startsWith('/'); + } + void newEntity(Sink::ApplicationDomain::ApplicationDomainType &newEntity) Q_DECL_OVERRIDE { auto mail = newEntity.cast(); - const auto mimeMessage = mail.getMimeMessagePath(); + const auto mimeMessage = mail.getMimeMessage(); if (!mimeMessage.isNull()) { - const auto path = moveMessage(mimeMessage, mail.getFolder()); - auto blob = ApplicationDomain::BLOB{path}; - blob.isExternal = false; - mail.setProperty(ApplicationDomain::Mail::MimeMessage::name, QVariant::fromValue(blob)); + if (isPath(mimeMessage)) { + mail.setMimeMessage(moveMessage(mimeMessage, mail.getFolder()).toUtf8()); + } else { + mail.setMimeMessage(storeMessage(mimeMessage, mail.getFolder()).toUtf8()); + } } } @@ -141,28 +173,28 @@ public: { auto newMail = newEntity.cast(); const ApplicationDomain::Mail oldMail{oldEntity}; - const auto mimeMessage = newMail.getMimeMessagePath(); const auto newFolder = newMail.getFolder(); - const bool mimeMessageChanged = !mimeMessage.isNull() && mimeMessage != oldMail.getMimeMessagePath(); const bool folderChanged = !newFolder.isNull() && newFolder != oldMail.getFolder(); - if (mimeMessageChanged || folderChanged) { - SinkTrace() << "Moving mime message: " << mimeMessageChanged << folderChanged; - auto newPath = moveMessage(mimeMessage, newMail.getFolder()); - if (newPath != oldMail.getMimeMessagePath()) { - const auto oldPath = getFilePathFromMimeMessagePath(oldMail.getMimeMessagePath()); - auto blob = ApplicationDomain::BLOB{newPath}; - blob.isExternal = false; - newMail.setProperty(ApplicationDomain::Mail::MimeMessage::name, QVariant::fromValue(blob)); + if (!newMail.getMimeMessage().isNull() || folderChanged) { + const auto data = newMail.getMimeMessage(); + if (isPath(data)) { + auto newPath = moveMessage(data, newMail.getFolder()); + if (newPath != oldMail.getMimeMessage()) { + newMail.setMimeMessage(newPath.toUtf8()); + //Remove the olde mime message if there is a new one + QFile::remove(getFilePathFromMimeMessagePath(oldMail.getMimeMessage())); + } + } else { + newMail.setMimeMessage(storeMessage(data, newMail.getFolder()).toUtf8()); //Remove the olde mime message if there is a new one - QFile::remove(oldPath); + QFile::remove(getFilePathFromMimeMessagePath(oldMail.getMimeMessage())); } } - auto mimeMessagePath = newMail.getMimeMessagePath(); + auto mimeMessagePath = newMail.getMimeMessage(); const auto maildirPath = getPath(newMail.getFolder()); KPIM::Maildir maildir(maildirPath, false); - const auto file = getFilePathFromMimeMessagePath(mimeMessagePath); - QString identifier = KPIM::Maildir::getKeyFromFile(file); + QString identifier = KPIM::Maildir::getKeyFromFile(getFilePathFromMimeMessagePath(mimeMessagePath)); //get flags from KPIM::Maildir::Flags flags; @@ -179,7 +211,7 @@ public: void deletedEntity(const Sink::ApplicationDomain::ApplicationDomainType &oldEntity) Q_DECL_OVERRIDE { const ApplicationDomain::Mail oldMail{oldEntity}; - const auto filePath = getFilePathFromMimeMessagePath(oldMail.getMimeMessagePath()); + const auto filePath = getFilePathFromMimeMessagePath(oldMail.getMimeMessage()); QFile::remove(filePath); } QByteArray mResourceInstanceIdentifier; @@ -327,9 +359,7 @@ public: mail.setFolder(folderLocalId); //We only store the directory path + key, so we facade can add the changing bits (flags) auto path = KPIM::Maildir::getDirectoryFromFile(filePath) + maildirKey; - auto blob = ApplicationDomain::BLOB{path}; - blob.isExternal = false; - mail.setProperty(ApplicationDomain::Mail::MimeMessage::name, QVariant::fromValue(blob)); + mail.setMimeMessage(path.toUtf8()); mail.setUnread(!flags.testFlag(KPIM::Maildir::Seen)); mail.setImportant(flags.testFlag(KPIM::Maildir::Flagged)); mail.setExtractedFullPayloadAvailable(true); @@ -396,7 +426,7 @@ public: KAsync::Job replay(const ApplicationDomain::Mail &mail, Sink::Operation operation, const QByteArray &oldRemoteId, const QList &changedProperties) Q_DECL_OVERRIDE { if (operation == Sink::Operation_Creation) { - const auto remoteId = getFilePathFromMimeMessagePath(mail.getMimeMessagePath()); + const auto remoteId = getFilePathFromMimeMessagePath(mail.getMimeMessage()); SinkTrace() << "Mail created: " << remoteId; return KAsync::value(remoteId.toUtf8()); } else if (operation == Sink::Operation_Removal) { @@ -404,7 +434,7 @@ public: return KAsync::null(); } else if (operation == Sink::Operation_Modification) { SinkTrace() << "Modifying a mail: " << oldRemoteId; - const auto remoteId = getFilePathFromMimeMessagePath(mail.getMimeMessagePath()); + const auto remoteId = getFilePathFromMimeMessagePath(mail.getMimeMessage()); return KAsync::value(remoteId.toUtf8()); } return KAsync::null(); @@ -460,7 +490,7 @@ protected: if (domainType == ENTITY_TYPE_MAIL) { auto mail = entityStore.readLatest(entityId); - const auto filePath = getFilePathFromMimeMessagePath(mail.getMimeMessagePath()); + const auto filePath = getFilePathFromMimeMessagePath(mail.getMimeMessage()); if (inspectionType == Sink::ResourceControl::Inspection::PropertyInspectionType) { if (property == "unread") { -- cgit v1.2.3