From b289dd9bf618a2512cbf15b2a6fc4ae77e4792c8 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sun, 19 Jun 2016 12:27:38 +0200 Subject: Moved mailpreprocessors to a shared location --- common/CMakeLists.txt | 3 +- common/mailpreprocessor.cpp | 117 ++++++++++++++++++++++++++++++++++++++++++++ common/mailpreprocessor.h | 45 +++++++++++++++++ common/pipeline.cpp | 36 +++++++++++--- common/pipeline.h | 6 ++- 5 files changed, 196 insertions(+), 11 deletions(-) create mode 100644 common/mailpreprocessor.cpp create mode 100644 common/mailpreprocessor.h (limited to 'common') diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 752e4e4..3083a2e 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -73,6 +73,7 @@ set(command_SRCS remoteidmap.cpp sourcewriteback.cpp entityreader.cpp + mailpreprocessor.cpp ${storage_SRCS}) add_library(${PROJECT_NAME} SHARED ${command_SRCS}) @@ -108,7 +109,7 @@ SET_TARGET_PROPERTIES(${PROJECT_NAME} EXPORT_NAME ${PROJECT_NAME} ) qt5_use_modules(${PROJECT_NAME} Network) -target_link_libraries(${PROJECT_NAME} ${storage_LIBS} KF5::Async) +target_link_libraries(${PROJECT_NAME} ${storage_LIBS} KF5::Async KF5::Mime) install(TARGETS ${PROJECT_NAME} EXPORT SinkTargets ${KDE_INSTALL_TARGETS_DEFAULT_ARGS} ${LIBRARY_NAMELINK} ) diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp new file mode 100644 index 0000000..64cb3d9 --- /dev/null +++ b/common/mailpreprocessor.cpp @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "mailpreprocessor.h" + +#include +#include +#include + +#include "pipeline.h" +#include "definitions.h" +#include "applicationdomaintype.h" + +using namespace Sink; + +void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail) +{ + const auto mimeMessagePath = mail.getMimeMessagePath(); + Trace() << "Updating indexed properties " << mimeMessagePath; + QFile f(mimeMessagePath); + if (!f.open(QIODevice::ReadOnly)) { + Warning() << "Failed to open the file: " << mimeMessagePath; + return; + } + if (!f.size()) { + Warning() << "The file is empty."; + return; + } + const auto mappedSize = qMin((qint64)8000, f.size()); + auto mapped = f.map(0, mappedSize); + if (!mapped) { + Warning() << "Failed to map the file: " << f.errorString(); + return; + } + + KMime::Message *msg = new KMime::Message; + msg->setHead(KMime::CRLFtoLF(QByteArray::fromRawData(reinterpret_cast(mapped), mappedSize))); + msg->parse(); + + mail.setExtractedSubject(msg->subject(true)->asUnicodeString()); + mail.setExtractedSender(msg->from(true)->asUnicodeString()); + mail.setExtractedSenderName(msg->from(true)->asUnicodeString()); + mail.setExtractedDate(msg->date(true)->dateTime()); +} + +void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) +{ + updatedIndexedProperties(mail); +} + +void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail,Sink::Storage::Transaction &transaction) +{ + updatedIndexedProperties(newMail); +} + + +MimeMessageMover::MimeMessageMover() : Sink::EntityPreprocessor() +{ +} + +QString MimeMessageMover::moveMessage(const QString &oldPath, const Sink::ApplicationDomain::Mail &mail) +{ + const auto directory = Sink::resourceStorageLocation(resourceInstanceIdentifier()); + const auto filePath = directory + "/" + mail.identifier(); + if (oldPath != filePath) { + if (!QDir().mkpath(directory)) { + Warning() << "Failed to create the directory: " << directory; + } + QFile::remove(filePath); + QFile origFile(oldPath); + if (!origFile.open(QIODevice::ReadWrite)) { + Warning() << "Failed to open the original file with write rights: " << origFile.errorString(); + } + if (!origFile.rename(filePath)) { + Warning() << "Failed to move the file from: " << oldPath << " to " << filePath << ". " << origFile.errorString(); + } + origFile.close(); + return filePath; + } + return oldPath; +} + +void MimeMessageMover::newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) +{ + if (!mail.getMimeMessagePath().isEmpty()) { + mail.setMimeMessagePath(moveMessage(mail.getMimeMessagePath(), mail)); + } +} + +void MimeMessageMover::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail, Sink::Storage::Transaction &transaction) +{ + if (!newMail.getMimeMessagePath().isEmpty()) { + newMail.setMimeMessagePath(moveMessage(newMail.getMimeMessagePath(), newMail)); + } +} + +void MimeMessageMover::deletedEntity(const Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) +{ + QFile::remove(mail.getMimeMessagePath()); +} + diff --git a/common/mailpreprocessor.h b/common/mailpreprocessor.h new file mode 100644 index 0000000..715e336 --- /dev/null +++ b/common/mailpreprocessor.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "sink_export.h" + +#include "pipeline.h" + +class SINK_EXPORT MailPropertyExtractor : public Sink::EntityPreprocessor +{ +public: + virtual ~MailPropertyExtractor(){} + virtual void newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE; + virtual void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail,Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE; +private: + void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail); +}; + +class SINK_EXPORT MimeMessageMover : public Sink::EntityPreprocessor +{ +public: + MimeMessageMover(); + virtual ~MimeMessageMover(){} + + void newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE; + void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE; + void deletedEntity(const Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE; + +private: + QString moveMessage(const QString &oldPath, const Sink::ApplicationDomain::Mail &mail); +}; diff --git a/common/pipeline.cpp b/common/pipeline.cpp index 5d8a34c..63a60ce 100644 --- a/common/pipeline.cpp +++ b/common/pipeline.cpp @@ -46,7 +46,7 @@ namespace Sink { class Pipeline::Private { public: - Private(const QString &resourceName) : storage(Sink::storageLocation(), resourceName, Storage::ReadWrite), revisionChanged(false) + Private(const QString &resourceName) : storage(Sink::storageLocation(), resourceName, Storage::ReadWrite), revisionChanged(false), resourceInstanceIdentifier(resourceName.toUtf8()) { } @@ -58,6 +58,7 @@ public: QTime transactionTime; int transactionItemCount; QByteArray resourceType; + QByteArray resourceInstanceIdentifier; }; void Pipeline::Private::storeNewRevision(qint64 newRevision, const flatbuffers::FlatBufferBuilder &fbb, const QByteArray &bufferType, const QByteArray &uid) @@ -82,6 +83,9 @@ Pipeline::~Pipeline() void Pipeline::setPreprocessors(const QString &entityType, const QVector &processors) { + for (auto p : processors) { + p->setup(d->resourceType, d->resourceInstanceIdentifier, this); + } d->processors[entityType] = processors; } @@ -191,8 +195,6 @@ KAsync::Job Pipeline::newEntity(void const *command, size_t size) auto adaptor = adaptorFactory->createAdaptor(*entity); auto memoryAdaptor = QSharedPointer::create(*(adaptor), adaptor->availableProperties()); for (auto processor : d->processors[bufferType]) { - processor->resourceType = d->resourceType; - processor->pipeline = this; processor->newEntity(key, Storage::maxRevision(d->transaction) + 1, *memoryAdaptor, d->transaction); } //The maxRevision may have changed meanwhile if the entity created sub-entities @@ -303,8 +305,6 @@ KAsync::Job Pipeline::modifiedEntity(void const *command, size_t size) } for (auto processor : d->processors[bufferType]) { - processor->resourceType = d->resourceType; - processor->pipeline = this; processor->modifiedEntity(key, Storage::maxRevision(d->transaction) + 1, *current, *newAdaptor, d->transaction); } //The maxRevision may have changed meanwhile if the entity created sub-entities @@ -452,12 +452,27 @@ qint64 Pipeline::cleanedUpRevision() return Storage::cleanedUpRevision(d->transaction); } -Preprocessor::Preprocessor() : d(0) +class Preprocessor::Private { +public: + QByteArray resourceType; + QByteArray resourceInstanceIdentifier; + Pipeline *pipeline; +}; + +Preprocessor::Preprocessor() : d(new Preprocessor::Private) { } Preprocessor::~Preprocessor() { + delete d; +} + +void Preprocessor::setup(const QByteArray &resourceType, const QByteArray &resourceInstanceIdentifier, Pipeline *pipeline) +{ + d->resourceType = resourceType; + d->resourceInstanceIdentifier = resourceInstanceIdentifier; + d->pipeline = pipeline; } void Preprocessor::startBatch() @@ -468,10 +483,15 @@ void Preprocessor::finalize() { } +QByteArray Preprocessor::resourceInstanceIdentifier() const +{ + return d->resourceInstanceIdentifier; +} + void Preprocessor::createEntity(const Sink::ApplicationDomain::ApplicationDomainType &entity, const QByteArray &typeName) { flatbuffers::FlatBufferBuilder entityFbb; - auto adaptorFactory = Sink::AdaptorFactoryRegistry::instance().getFactory(resourceType, typeName); + auto adaptorFactory = Sink::AdaptorFactoryRegistry::instance().getFactory(d->resourceType, typeName); adaptorFactory->createBuffer(entity, entityFbb); const auto entityBuffer = BufferUtils::extractBuffer(entityFbb); @@ -484,7 +504,7 @@ void Preprocessor::createEntity(const Sink::ApplicationDomain::ApplicationDomain Sink::Commands::FinishCreateEntityBuffer(fbb, location); const auto data = BufferUtils::extractBuffer(fbb); - pipeline->newEntity(data, data.size()).exec(); + d->pipeline->newEntity(data, data.size()).exec(); } } // namespace Sink diff --git a/common/pipeline.h b/common/pipeline.h index 1afda21..d04d795 100644 --- a/common/pipeline.h +++ b/common/pipeline.h @@ -88,6 +88,8 @@ public: virtual void deletedEntity(const QByteArray &uid, qint64 revision, const Sink::ApplicationDomain::BufferAdaptor &oldEntity, Sink::Storage::Transaction &transaction) {}; virtual void finalize(); + void setup(const QByteArray &resourceType, const QByteArray &resourceInstanceIdentifier, Pipeline *); + protected: template void createEntity(const DomainType &entity) @@ -96,12 +98,12 @@ protected: } void createEntity(const Sink::ApplicationDomain::ApplicationDomainType &entity, const QByteArray &type); + QByteArray resourceInstanceIdentifier() const; + private: friend class Pipeline; class Private; Private *const d; - Pipeline *pipeline; - QByteArray resourceType; }; template -- cgit v1.2.3