7 files changed, 199 insertions, 105 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c53075d..1eb16a1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,7 +20,7 @@ include(ECMSetupVersion)
 include(KDEInstallDirs)
 
 find_package(Qt5 COMPONENTS REQUIRED Core Widgets Network)
-find_package(KF5 COMPONENTS REQUIRED Async)
+find_package(KF5 COMPONENTS REQUIRED Async Mime)
 find_package(FlatBuffers REQUIRED)
 
 #Clang-format support
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
index 752e4e4..3083a2e 100644
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -73,6 +73,7 @@ set(command_SRCS
     remoteidmap.cpp
     sourcewriteback.cpp
     entityreader.cpp
+    mailpreprocessor.cpp
     ${storage_SRCS})
 
 add_library(${PROJECT_NAME} SHARED ${command_SRCS})
@@ -108,7 +109,7 @@ SET_TARGET_PROPERTIES(${PROJECT_NAME}
     EXPORT_NAME ${PROJECT_NAME}
 )
 qt5_use_modules(${PROJECT_NAME} Network)
-target_link_libraries(${PROJECT_NAME} ${storage_LIBS} KF5::Async)
+target_link_libraries(${PROJECT_NAME} ${storage_LIBS} KF5::Async KF5::Mime)
 install(TARGETS ${PROJECT_NAME}
     EXPORT SinkTargets ${KDE_INSTALL_TARGETS_DEFAULT_ARGS} ${LIBRARY_NAMELINK} )
 
diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp
new file mode 100644
index 0000000..64cb3d9
--- /dev/null
+++ b/common/mailpreprocessor.cpp
@@ -0,0 +1,117 @@
+/*
+ *   Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+
+#include "mailpreprocessor.h"
+
+#include <QFile>
+#include <QDir>
+#include <KMime/KMime/KMimeMessage>
+
+#include "pipeline.h"
+#include "definitions.h"
+#include "applicationdomaintype.h"
+
+using namespace Sink;
+
+void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail)
+{
+    const auto mimeMessagePath = mail.getMimeMessagePath();
+    Trace() << "Updating indexed properties " << mimeMessagePath;
+    QFile f(mimeMessagePath);
+    if (!f.open(QIODevice::ReadOnly)) {
+        Warning() << "Failed to open the file: " << mimeMessagePath;
+        return;
+    }
+    if (!f.size()) {
+        Warning() << "The file is empty.";
+        return;
+    }
+    const auto mappedSize = qMin((qint64)8000, f.size());
+    auto mapped = f.map(0, mappedSize);
+    if (!mapped) {
+        Warning() << "Failed to map the file: " << f.errorString();
+        return;
+    }
+
+    KMime::Message *msg = new KMime::Message;
+    msg->setHead(KMime::CRLFtoLF(QByteArray::fromRawData(reinterpret_cast<const char*>(mapped), mappedSize)));
+    msg->parse();
+
+    mail.setExtractedSubject(msg->subject(true)->asUnicodeString());
+    mail.setExtractedSender(msg->from(true)->asUnicodeString());
+    mail.setExtractedSenderName(msg->from(true)->asUnicodeString());
+    mail.setExtractedDate(msg->date(true)->dateTime());
+}
+
+void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction)
+{
+    updatedIndexedProperties(mail);
+}
+
+void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail,Sink::Storage::Transaction &transaction)
+{
+    updatedIndexedProperties(newMail);
+}
+
+
+MimeMessageMover::MimeMessageMover() : Sink::EntityPreprocessor<ApplicationDomain::Mail>()
+{
+}
+
+QString MimeMessageMover::moveMessage(const QString &oldPath, const Sink::ApplicationDomain::Mail &mail)
+{
+    const auto directory = Sink::resourceStorageLocation(resourceInstanceIdentifier());
+    const auto filePath = directory + "/" + mail.identifier();
+    if (oldPath != filePath) {
+        if (!QDir().mkpath(directory)) {
+            Warning() << "Failed to create the directory: " << directory;
+        }
+        QFile::remove(filePath);
+        QFile origFile(oldPath);
+        if (!origFile.open(QIODevice::ReadWrite)) {
+            Warning() << "Failed to open the original file with write rights: " << origFile.errorString();
+        }
+        if (!origFile.rename(filePath)) {
+            Warning() << "Failed to move the file from: " << oldPath << " to " << filePath << ". " << origFile.errorString();
+        }
+        origFile.close();
+        return filePath;
+    }
+    return oldPath;
+}
+
+void MimeMessageMover::newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction)
+{
+    if (!mail.getMimeMessagePath().isEmpty()) {
+        mail.setMimeMessagePath(moveMessage(mail.getMimeMessagePath(), mail));
+    }
+}
+
+void MimeMessageMover::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail, Sink::Storage::Transaction &transaction)
+{
+    if (!newMail.getMimeMessagePath().isEmpty()) {
+        newMail.setMimeMessagePath(moveMessage(newMail.getMimeMessagePath(), newMail));
+    }
+}
+
+void MimeMessageMover::deletedEntity(const Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction)
+{
+    QFile::remove(mail.getMimeMessagePath());
+}
+
diff --git a/common/mailpreprocessor.h b/common/mailpreprocessor.h
new file mode 100644
index 0000000..715e336
--- /dev/null
+++ b/common/mailpreprocessor.h
@@ -0,0 +1,45 @@
+/*
+ *   Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+#include "sink_export.h"
+
+#include "pipeline.h"
+
+class SINK_EXPORT MailPropertyExtractor : public Sink::EntityPreprocessor<Sink::ApplicationDomain::Mail>
+{
+public:
+    virtual ~MailPropertyExtractor(){}
+    virtual void newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE;
+    virtual void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail,Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE;
+private:
+    void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail);
+};
+
+class SINK_EXPORT MimeMessageMover : public Sink::EntityPreprocessor<Sink::ApplicationDomain::Mail>
+{
+public:
+    MimeMessageMover();
+    virtual ~MimeMessageMover(){}
+
+    void newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE;
+    void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE;
+    void deletedEntity(const Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE;
+
+private:
+    QString moveMessage(const QString &oldPath, const Sink::ApplicationDomain::Mail &mail);
+};
diff --git a/common/pipeline.cpp b/common/pipeline.cpp
index 5d8a34c..63a60ce 100644
--- a/common/pipeline.cpp
+++ b/common/pipeline.cpp
@@ -46,7 +46,7 @@ namespace Sink {
 class Pipeline::Private
 {
 public:
-    Private(const QString &resourceName) : storage(Sink::storageLocation(), resourceName, Storage::ReadWrite), revisionChanged(false)
+    Private(const QString &resourceName) : storage(Sink::storageLocation(), resourceName, Storage::ReadWrite), revisionChanged(false), resourceInstanceIdentifier(resourceName.toUtf8())
     {
     }
 
@@ -58,6 +58,7 @@ public:
     QTime transactionTime;
     int transactionItemCount;
     QByteArray resourceType;
+    QByteArray resourceInstanceIdentifier;
 };
 
 void Pipeline::Private::storeNewRevision(qint64 newRevision, const flatbuffers::FlatBufferBuilder &fbb, const QByteArray &bufferType, const QByteArray &uid)
@@ -82,6 +83,9 @@ Pipeline::~Pipeline()
 
 void Pipeline::setPreprocessors(const QString &entityType, const QVector<Preprocessor *> &processors)
 {
+    for (auto p : processors) {
+        p->setup(d->resourceType, d->resourceInstanceIdentifier, this);
+    }
     d->processors[entityType] = processors;
 }
 
@@ -191,8 +195,6 @@ KAsync::Job<qint64> Pipeline::newEntity(void const *command, size_t size)
     auto adaptor = adaptorFactory->createAdaptor(*entity);
     auto memoryAdaptor = QSharedPointer<Sink::ApplicationDomain::MemoryBufferAdaptor>::create(*(adaptor), adaptor->availableProperties());
     for (auto processor : d->processors[bufferType]) {
-        processor->resourceType = d->resourceType;
-        processor->pipeline = this;
         processor->newEntity(key, Storage::maxRevision(d->transaction) + 1, *memoryAdaptor, d->transaction);
     }
     //The maxRevision may have changed meanwhile if the entity created sub-entities
@@ -303,8 +305,6 @@ KAsync::Job<qint64> Pipeline::modifiedEntity(void const *command, size_t size)
     }
 
     for (auto processor : d->processors[bufferType]) {
-        processor->resourceType = d->resourceType;
-        processor->pipeline = this;
         processor->modifiedEntity(key, Storage::maxRevision(d->transaction) + 1, *current, *newAdaptor, d->transaction);
     }
     //The maxRevision may have changed meanwhile if the entity created sub-entities
@@ -452,12 +452,27 @@ qint64 Pipeline::cleanedUpRevision()
     return Storage::cleanedUpRevision(d->transaction);
 }
 
-Preprocessor::Preprocessor() : d(0)
+class Preprocessor::Private {
+public:
+    QByteArray resourceType;
+    QByteArray resourceInstanceIdentifier;
+    Pipeline *pipeline;
+};
+
+Preprocessor::Preprocessor() : d(new Preprocessor::Private)
 {
 }
 
 Preprocessor::~Preprocessor()
 {
+    delete d;
+}
+
+void Preprocessor::setup(const QByteArray &resourceType, const QByteArray &resourceInstanceIdentifier, Pipeline *pipeline)
+{
+    d->resourceType = resourceType;
+    d->resourceInstanceIdentifier = resourceInstanceIdentifier;
+    d->pipeline = pipeline;
 }
 
 void Preprocessor::startBatch()
@@ -468,10 +483,15 @@ void Preprocessor::finalize()
 {
 }
 
+QByteArray Preprocessor::resourceInstanceIdentifier() const
+{
+    return d->resourceInstanceIdentifier;
+}
+
 void Preprocessor::createEntity(const Sink::ApplicationDomain::ApplicationDomainType &entity, const QByteArray &typeName)
 {
     flatbuffers::FlatBufferBuilder entityFbb;
-    auto adaptorFactory = Sink::AdaptorFactoryRegistry::instance().getFactory(resourceType, typeName);
+    auto adaptorFactory = Sink::AdaptorFactoryRegistry::instance().getFactory(d->resourceType, typeName);
     adaptorFactory->createBuffer(entity, entityFbb);
     const auto entityBuffer = BufferUtils::extractBuffer(entityFbb);
 
@@ -484,7 +504,7 @@ void Preprocessor::createEntity(const Sink::ApplicationDomain::ApplicationDomain
     Sink::Commands::FinishCreateEntityBuffer(fbb, location);
 
     const auto data = BufferUtils::extractBuffer(fbb);
-    pipeline->newEntity(data, data.size()).exec();
+    d->pipeline->newEntity(data, data.size()).exec();
 }
 
 } // namespace Sink
diff --git a/common/pipeline.h b/common/pipeline.h
index 1afda21..d04d795 100644
--- a/common/pipeline.h
+++ b/common/pipeline.h
@@ -88,6 +88,8 @@ public:
     virtual void deletedEntity(const QByteArray &uid, qint64 revision, const Sink::ApplicationDomain::BufferAdaptor &oldEntity, Sink::Storage::Transaction &transaction) {};
     virtual void finalize();
 
+    void setup(const QByteArray &resourceType, const QByteArray &resourceInstanceIdentifier, Pipeline *);
+
 protected:
     template <typename DomainType>
     void createEntity(const DomainType &entity)
@@ -96,12 +98,12 @@ protected:
     }
     void createEntity(const Sink::ApplicationDomain::ApplicationDomainType &entity, const QByteArray &type);
 
+    QByteArray resourceInstanceIdentifier() const;
+
 private:
     friend class Pipeline;
     class Private;
     Private *const d;
-    Pipeline *pipeline;
-    QByteArray resourceType;
 };
 
 template<typename DomainType>
diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp
index 7226fd5..f9168e4 100644
--- a/examples/imapresource/imapresource.cpp
+++ b/examples/imapresource/imapresource.cpp
@@ -47,6 +47,7 @@
 
 #include "imapserverproxy.h"
 #include "entityreader.h"
+#include "mailpreprocessor.h"
 
 //This is the resources entity type, and not the domain type
 #define ENTITY_TYPE_MAIL "mail"
@@ -136,98 +137,6 @@ public:
     QByteArray mResourceInstanceIdentifier;
 };
 
-class MailPropertyExtractor : public Sink::EntityPreprocessor<ApplicationDomain::Mail>
-{
-public:
-
-    void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail)
-    {
-        const auto mimeMessagePath = mail.getMimeMessagePath();
-        Trace() << "Updating indexed properties " << mimeMessagePath;
-        QFile f(mimeMessagePath);
-        if (!f.open(QIODevice::ReadOnly)) {
-            Warning() << "Failed to open the file: " << mimeMessagePath;
-            return;
-        }
-        if (!f.size()) {
-            Warning() << "The file is empty.";
-            return;
-        }
-        const auto mappedSize = qMin((qint64)8000, f.size());
-        auto mapped = f.map(0, mappedSize);
-        if (!mapped) {
-            Warning() << "Failed to map the file: " << f.errorString();
-            return;
-        }
-
-        KMime::Message *msg = new KMime::Message;
-        msg->setHead(KMime::CRLFtoLF(QByteArray::fromRawData(reinterpret_cast<const char*>(mapped), mappedSize)));
-        msg->parse();
-
-        mail.setExtractedSubject(msg->subject(true)->asUnicodeString());
-        mail.setExtractedSender(msg->from(true)->asUnicodeString());
-        mail.setExtractedSenderName(msg->from(true)->asUnicodeString());
-        mail.setExtractedDate(msg->date(true)->dateTime());
-    }
-
-    void newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE
-    {
-        updatedIndexedProperties(mail);
-    }
-
-    void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail,Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE
-    {
-        updatedIndexedProperties(newMail);
-    }
-};
-
-class MimeMessageMover : public Sink::EntityPreprocessor<ApplicationDomain::Mail>
-{
-public:
-    MimeMessageMover(const QByteArray &resourceInstanceIdentifier) : Sink::EntityPreprocessor<ApplicationDomain::Mail>(), mResourceInstanceIdentifier(resourceInstanceIdentifier) {}
-
-    QString moveMessage(const QString &oldPath, const Sink::ApplicationDomain::Mail &mail)
-    {
-        const auto directory = Sink::resourceStorageLocation(mResourceInstanceIdentifier);
-        const auto filePath = directory + "/" + mail.identifier();
-        if (oldPath != filePath) {
-            if (!QDir().mkpath(directory)) {
-                Warning() << "Failed to create the directory: " << directory;
-            }
-            QFile::remove(filePath);
-            QFile origFile(oldPath);
-            if (!origFile.open(QIODevice::ReadWrite)) {
-                Warning() << "Failed to open the original file with write rights: " << origFile.errorString();
-            }
-            if (!origFile.rename(filePath)) {
-                Warning() << "Failed to move the file from: " << oldPath << " to " << filePath << ". " << origFile.errorString();
-            }
-            origFile.close();
-            return filePath;
-        }
-        return oldPath;
-    }
-
-    void newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE
-    {
-        if (!mail.getMimeMessagePath().isEmpty()) {
-            mail.setMimeMessagePath(moveMessage(mail.getMimeMessagePath(), mail));
-        }
-    }
-
-    void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE
-    {
-        if (!newMail.getMimeMessagePath().isEmpty()) {
-            newMail.setMimeMessagePath(moveMessage(newMail.getMimeMessagePath(), newMail));
-        }
-    }
-
-    void deletedEntity(const Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE
-    {
-        QFile::remove(mail.getMimeMessagePath());
-    }
-    QByteArray mResourceInstanceIdentifier;
-};
 
 static qint64 uidFromMailRid(const QByteArray &remoteId)
 {
@@ -651,7 +560,7 @@ ImapResource::ImapResource(const QByteArray &instanceIdentifier, const QSharedPo
     changereplay->mPassword = mPassword;
     setupChangereplay(changereplay);
 
-    setupPreprocessors(ENTITY_TYPE_MAIL, QVector<Sink::Preprocessor*>() << new SpecialPurposeProcessor(mResourceType, mResourceInstanceIdentifier) << new MimeMessageMover(mResourceInstanceIdentifier) << new MailPropertyExtractor << new DefaultIndexUpdater<Sink::ApplicationDomain::Mail>);
+    setupPreprocessors(ENTITY_TYPE_MAIL, QVector<Sink::Preprocessor*>() << new SpecialPurposeProcessor(mResourceType, mResourceInstanceIdentifier) << new MimeMessageMover << new MailPropertyExtractor << new DefaultIndexUpdater<Sink::ApplicationDomain::Mail>);
     setupPreprocessors(ENTITY_TYPE_FOLDER, QVector<Sink::Preprocessor*>() << new DefaultIndexUpdater<Sink::ApplicationDomain::Folder>);
 }