ImapResource prototype

15 files changed, 998 insertions, 0 deletions

diff --git a/common/domain/applicationdomaintype.cpp b/common/domain/applicationdomaintype.cpp
index 166d3e6..7be0295 100644
--- a/common/domain/applicationdomaintype.cpp
+++ b/common/domain/applicationdomaintype.cpp
@@ -261,6 +261,17 @@ namespace MailtransportResource {
     }
 }
 
+namespace ImapResource {
+    SinkResource create(const QByteArray &account)
+    {
+        auto &&resource = ApplicationDomainType::createEntity<SinkResource>();
+        resource.setProperty("type", "org.kde.imap");
+        resource.setProperty("account", account);
+        resource.setProperty("capabilities", QVariant::fromValue(QByteArrayList() << "storage" << "drafts"));
+        return resource;
+    }
+}
+
 template<>
 QByteArray getTypeName<Event>()
 {
diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h
index 4eec4a3..1b0ae9d 100644
--- a/common/domain/applicationdomaintype.h
+++ b/common/domain/applicationdomaintype.h
@@ -245,6 +245,9 @@ namespace MaildirResource {
 namespace MailtransportResource {
     SinkResource SINK_EXPORT create(const QByteArray &account);
 };
+namespace ImapResource {
+    SinkResource SINK_EXPORT create(const QByteArray &account);
+};
 
 /**
  * All types need to be registered here an MUST return a different name.
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index d5fcacf..59753bb 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -7,5 +7,6 @@ add_subdirectory(dummyresource)
 if (BUILD_MAILDIR)
     # a maildir resource implementation
     add_subdirectory(maildirresource)
+    add_subdirectory(imapresource)
 endif()
 add_subdirectory(mailtransportresource)
diff --git a/examples/imapresource/CMakeLists.txt b/examples/imapresource/CMakeLists.txt
new file mode 100644
index 0000000..d5320a6
--- /dev/null
+++ b/examples/imapresource/CMakeLists.txt
@@ -0,0 +1,18 @@
+project(sink_resource_imap)
+
+add_definitions(-DQT_PLUGIN)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+
+find_package(KF5 COMPONENTS REQUIRED Mime IMAP KIO)
+find_package(KF5CoreAddons REQUIRED)
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+
+add_library(${PROJECT_NAME} SHARED facade.cpp imapresource.cpp domainadaptor.cpp imapserverproxy.cpp)
+qt5_use_modules(${PROJECT_NAME} Core Network)
+#We need CoreAddons for KJob and KIOCore for KTcpSocket. Both used in KIMAP
+target_link_libraries(${PROJECT_NAME} sink KF5::Mime KF5::IMAP KF5::CoreAddons KF5::KIOCore)
+
+install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${SINK_RESOURCE_PLUGINS_PATH})
+
+add_subdirectory(tests)
diff --git a/examples/imapresource/domainadaptor.cpp b/examples/imapresource/domainadaptor.cpp
new file mode 100644
index 0000000..4e74ad2
--- /dev/null
+++ b/examples/imapresource/domainadaptor.cpp
@@ -0,0 +1,35 @@
+/*
+ *   Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+
+#include "domainadaptor.h"
+
+using namespace flatbuffers;
+
+ImapMailAdaptorFactory::ImapMailAdaptorFactory()
+    : DomainTypeAdaptorFactory()
+{
+
+}
+
+ImapFolderAdaptorFactory::ImapFolderAdaptorFactory()
+    : DomainTypeAdaptorFactory()
+{
+
+}
+
diff --git a/examples/imapresource/domainadaptor.h b/examples/imapresource/domainadaptor.h
new file mode 100644
index 0000000..06a513c
--- /dev/null
+++ b/examples/imapresource/domainadaptor.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+#pragma once
+
+#include <common/domainadaptor.h>
+#include "mail_generated.h"
+#include "folder_generated.h"
+#include "dummy_generated.h"
+
+class ImapMailAdaptorFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Mail, Sink::ApplicationDomain::Buffer::Dummy, Sink::ApplicationDomain::Buffer::DummyBuilder>
+{
+public:
+    ImapMailAdaptorFactory();
+    virtual ~ImapMailAdaptorFactory() {};
+};
+
+class ImapFolderAdaptorFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Folder, Sink::ApplicationDomain::Buffer::Dummy, Sink::ApplicationDomain::Buffer::DummyBuilder>
+{
+public:
+    ImapFolderAdaptorFactory();
+    virtual ~ImapFolderAdaptorFactory() {};
+};
diff --git a/examples/imapresource/facade.cpp b/examples/imapresource/facade.cpp
new file mode 100644
index 0000000..d338b01
--- /dev/null
+++ b/examples/imapresource/facade.cpp
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2014 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+
+#include "facade.h"
+
+#include <QDir>
+#include <QFileInfo>
+
+#include "domainadaptor.h"
+#include "queryrunner.h"
+
+ImapResourceMailFacade::ImapResourceMailFacade(const QByteArray &instanceIdentifier)
+    : Sink::GenericFacade<Sink::ApplicationDomain::Mail>(instanceIdentifier, QSharedPointer<ImapMailAdaptorFactory>::create())
+{
+}
+
+ImapResourceMailFacade::~ImapResourceMailFacade()
+{
+}
+
+ImapResourceFolderFacade::ImapResourceFolderFacade(const QByteArray &instanceIdentifier)
+    : Sink::GenericFacade<Sink::ApplicationDomain::Folder>(instanceIdentifier, QSharedPointer<ImapFolderAdaptorFactory>::create())
+{
+}
+
+ImapResourceFolderFacade::~ImapResourceFolderFacade()
+{
+}
diff --git a/examples/imapresource/facade.h b/examples/imapresource/facade.h
new file mode 100644
index 0000000..479ad96
--- /dev/null
+++ b/examples/imapresource/facade.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2014 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+
+#pragma once
+
+#include "common/facade.h"
+
+class ImapResourceMailFacade : public Sink::GenericFacade<Sink::ApplicationDomain::Mail>
+{
+public:
+    ImapResourceMailFacade(const QByteArray &instanceIdentifier);
+    virtual ~ImapResourceMailFacade();
+};
+
+class ImapResourceFolderFacade : public Sink::GenericFacade<Sink::ApplicationDomain::Folder>
+{
+public:
+    ImapResourceFolderFacade(const QByteArray &instanceIdentifier);
+    virtual ~ImapResourceFolderFacade();
+};
diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp
new file mode 100644
index 0000000..baa88b9
--- /dev/null
+++ b/examples/imapresource/imapresource.cpp
@@ -0,0 +1,277 @@
+/*
+ *   Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+
+#include "imapresource.h"
+#include "facade.h"
+#include "entitybuffer.h"
+#include "pipeline.h"
+#include "mail_generated.h"
+#include "createentity_generated.h"
+#include "modifyentity_generated.h"
+#include "deleteentity_generated.h"
+#include "domainadaptor.h"
+#include "resourceconfig.h"
+#include "commands.h"
+#include "index.h"
+#include "log.h"
+#include "domain/mail.h"
+#include "definitions.h"
+#include "facadefactory.h"
+#include "indexupdater.h"
+#include "inspection.h"
+#include <QDate>
+#include <QUuid>
+#include <QDir>
+#include <QDirIterator>
+
+#include "imapserverproxy.h"
+
+//This is the resources entity type, and not the domain type
+#define ENTITY_TYPE_MAIL "mail"
+#define ENTITY_TYPE_FOLDER "folder"
+
+#undef DEBUG_AREA
+#define DEBUG_AREA "resource.imap"
+
+
+ImapResource::ImapResource(const QByteArray &instanceIdentifier, const QSharedPointer<Sink::Pipeline> &pipeline)
+    : Sink::GenericResource(instanceIdentifier, pipeline),
+    mMailAdaptorFactory(QSharedPointer<ImapMailAdaptorFactory>::create()),
+    mFolderAdaptorFactory(QSharedPointer<ImapFolderAdaptorFactory>::create())
+{
+    auto config = ResourceConfig::getConfiguration(instanceIdentifier);
+    mServer = config.value("server").toString();
+    mPort = config.value("port").toInt();
+
+    // auto folderUpdater = new FolderUpdater(QByteArray());
+    addType(ENTITY_TYPE_MAIL, mMailAdaptorFactory,
+            QVector<Sink::Preprocessor*>() << new DefaultIndexUpdater<Sink::ApplicationDomain::Mail>);
+    addType(ENTITY_TYPE_FOLDER, mFolderAdaptorFactory,
+            QVector<Sink::Preprocessor*>() << new DefaultIndexUpdater<Sink::ApplicationDomain::Folder>);
+}
+
+QByteArray ImapResource::createFolder(const QString &folderPath, const QByteArray &icon, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction)
+{
+    auto remoteId = folderPath.toUtf8();
+    auto bufferType = ENTITY_TYPE_FOLDER;
+    Sink::ApplicationDomain::Folder folder;
+    folder.setProperty("name", folderPath.split('/').last());
+    folder.setProperty("icon", icon);
+
+    // if (!md.isRoot()) {
+    //     folder.setProperty("parent", resolveRemoteId(ENTITY_TYPE_FOLDER, md.parent().path().toUtf8(), synchronizationTransaction));
+    // }
+    createOrModify(transaction, synchronizationTransaction, *mFolderAdaptorFactory, bufferType, remoteId, folder);
+    return remoteId;
+}
+
+void ImapResource::synchronizeFolders(const QStringList &folderList, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction)
+{
+    const QByteArray bufferType = ENTITY_TYPE_FOLDER;
+    Trace() << "Found folders " << folderList;
+
+    scanForRemovals(transaction, synchronizationTransaction, bufferType,
+        [&bufferType, &transaction](const std::function<void(const QByteArray &)> &callback) {
+            //TODO Instead of iterating over all entries in the database, which can also pick up the same item multiple times,
+            //we should rather iterate over an index that contains every uid exactly once. The remoteId index would be such an index,
+            //but we currently fail to iterate over all entries in an index it seems.
+            // auto remoteIds = synchronizationTransaction.openDatabase("rid.mapping." + bufferType, std::function<void(const Sink::Storage::Error &)>(), true);
+            auto mainDatabase = Sink::Storage::mainDatabase(transaction, bufferType);
+            mainDatabase.scan("", [&](const QByteArray &key, const QByteArray &) {
+                callback(key);
+                return true;
+            });
+        },
+        [&folderList](const QByteArray &remoteId) -> bool {
+            return folderList.contains(remoteId);
+        }
+    );
+
+    for (const auto folderPath : folderList) {
+        createFolder(folderPath, "folder", transaction, synchronizationTransaction);
+    }
+}
+
+void ImapResource::synchronizeMails(Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction, const QString &path, const QVector<Message> &messages)
+{
+    auto time = QSharedPointer<QTime>::create();
+    time->start();
+    const QByteArray bufferType = ENTITY_TYPE_MAIL;
+
+
+    Trace() << "Importing new mail.";
+
+    // Trace() << "Looking into " << listingPath;
+
+    const auto folderLocalId = resolveRemoteId(ENTITY_TYPE_FOLDER, path.toUtf8(), synchronizationTransaction);
+
+    //This is not a full listing
+    // auto property = "folder";
+    // scanForRemovals(transaction, synchronizationTransaction, bufferType,
+    //     [&](const std::function<void(const QByteArray &)> &callback) {
+    //         Index index(bufferType + ".index." + property, transaction);
+    //         index.lookup(folderLocalId, [&](const QByteArray &sinkId) {
+    //             callback(sinkId);
+    //         },
+    //         [&](const Index::Error &error) {
+    //             Warning() << "Error in index: " <<  error.message << property;
+    //         });
+    //     },
+    //     [](const QByteArray &remoteId) -> bool {
+    //         return QFile(remoteId).exists();
+    //     }
+    // );
+
+    mSynchronizerQueue.startTransaction();
+    int count = 0;
+    for (const auto &message : messages) {
+        count++;
+        const auto remoteId = path.toUtf8() + "/" + QByteArray::number(message.uid);
+
+        Trace() << "Found a mail " << remoteId << message.msg->subject(true)->asUnicodeString() << message.flags;
+
+        Sink::ApplicationDomain::Mail mail;
+        mail.setFolder(folderLocalId);
+        //FIXME this should come from the mime message, extracted in the pipeline
+        mail.setExtractedSubject(message.msg->subject(true)->asUnicodeString());
+
+        auto filePath = Sink::resourceStorageLocation(mResourceInstanceIdentifier) + "/" + remoteId;
+        QDir().mkpath(Sink::resourceStorageLocation(mResourceInstanceIdentifier) + "/" + path.toUtf8());
+        QFile file(filePath);
+        if (!file.open(QIODevice::WriteOnly)) {
+            Warning() << "Failed to open file for writing: " << file.errorString();
+        }
+        const auto content = message.msg->encodedContent();
+        file.write(content);
+        mail.setMimeMessagePath(filePath);
+        //FIXME  Not sure if these are the actual flags
+        mail.setUnread(message.flags.contains("\\SEEN"));
+        mail.setImportant(message.flags.contains("\\FLAGGED"));
+
+        createOrModify(transaction, synchronizationTransaction, *mMailAdaptorFactory, bufferType, remoteId, mail);
+    }
+    mSynchronizerQueue.commit();
+    const auto elapsed = time->elapsed();
+    Log() << "Synchronized " << count << " mails in " << path << Sink::Log::TraceTime(elapsed) << " " << elapsed/qMax(count, 1) << " [ms/mail]";
+}
+
+KAsync::Job<void> ImapResource::synchronizeWithSource(Sink::Storage &mainStore, Sink::Storage &synchronizationStore)
+{
+    Log() << " Synchronizing";
+    return KAsync::start<void>([this, &mainStore, &synchronizationStore](KAsync::Future<void> future) {
+        ImapServerProxy imap(mServer, mPort);
+        QStringList folderList;
+        // QList<KAsync::Future<void>> waitCondition;
+        auto folderFuture = imap.fetchFolders([this, &imap, &mainStore, &synchronizationStore, &folderList](const QStringList &folders) {
+            auto transaction = mainStore.createTransaction(Sink::Storage::ReadOnly);
+            auto syncTransaction = synchronizationStore.createTransaction(Sink::Storage::ReadWrite);
+            synchronizeFolders(folders, transaction, syncTransaction);
+            transaction.commit();
+            syncTransaction.commit();
+            folderList << folders;
+
+        });
+        folderFuture.waitForFinished();
+        if (folderFuture.errorCode()) {
+            future.setError(1, "Folder list sync failed");
+            return;
+        }
+
+        for (const auto &folder : folderList) {
+            // auto transaction = mainStore.createTransaction(Sink::Storage::ReadOnly);
+            // auto syncTransaction = synchronizationStore.createTransaction(Sink::Storage::ReadOnly);
+
+            //TODO load entity to read sync settings should we have some (if the folder is existing already)
+            //Note that this will not work if we change any of those settings in the pipeline
+            //
+            // auto mainDatabase = Sink::Storage::mainDatabase(transaction, ENTITY_TYPE_FOLDER);
+            // const auto sinkId = resolveRemoteId(ENTITY_TYPE_FOLDER, folder.toUtf8(), syncTransaction);
+            // const auto found = mainDatabase.contains(sinkId);
+            // if (found) {
+            //     if (auto current = getLatest(mainDatabase, sinkId, mFolderAdaptorFactory)) {
+            //
+            //     }
+            // }
+
+            // transaction.commit();
+            // syncTransaction.commit();
+
+            auto messagesFuture = imap.fetchMessages(folder, [this, &mainStore, &synchronizationStore, folder](const QVector<Message> &messages) {
+                auto transaction = mainStore.createTransaction(Sink::Storage::ReadOnly);
+                auto syncTransaction = synchronizationStore.createTransaction(Sink::Storage::ReadWrite);
+                Trace() << "Synchronizing mails" << folder;
+                synchronizeMails(transaction, syncTransaction, folder, messages);
+                transaction.commit();
+                syncTransaction.commit();
+            });
+            messagesFuture.waitForFinished();
+            if (messagesFuture.errorCode()) {
+                future.setError(1, "Folder sync failed: " + folder);
+                return;
+            }
+        }
+
+
+        // auto transaction = mainStore.createTransaction(Sink::Storage::ReadWrite);
+        // auto mainDatabase = Sink::Storage::mainDatabase(transaction, ENTITY_TYPE_FOLDER);
+        // mainDatabase.scan("", [&](const QByteArray &key, const QByteArray &data) {
+        //     return true;
+        // });
+        //TODO now fetch all folders and iterate over them and synchronize each one
+
+        Log() << "Done Synchronizing";
+        future.setFinished();
+    });
+}
+
+KAsync::Job<void> ImapResource::replay(Sink::Storage &synchronizationStore, const QByteArray &type, const QByteArray &key, const QByteArray &value)
+{
+    //TODO implement
+    return KAsync::null<void>();
+}
+
+void ImapResource::removeFromDisk(const QByteArray &instanceIdentifier)
+{
+    GenericResource::removeFromDisk(instanceIdentifier);
+    Sink::Storage(Sink::storageLocation(), instanceIdentifier + ".synchronization", Sink::Storage::ReadWrite).removeFromDisk();
+}
+
+KAsync::Job<void> ImapResource::inspect(int inspectionType, const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue)
+{
+    //TODO
+    return KAsync::null<void>();
+}
+
+ImapResourceFactory::ImapResourceFactory(QObject *parent)
+    : Sink::ResourceFactory(parent)
+{
+
+}
+
+Sink::Resource *ImapResourceFactory::createResource(const QByteArray &instanceIdentifier)
+{
+    return new ImapResource(instanceIdentifier);
+}
+
+void ImapResourceFactory::registerFacades(Sink::FacadeFactory &factory)
+{
+    factory.registerFacade<Sink::ApplicationDomain::Mail, ImapResourceMailFacade>(PLUGIN_NAME);
+    factory.registerFacade<Sink::ApplicationDomain::Folder, ImapResourceFolderFacade>(PLUGIN_NAME);
+}
+
diff --git a/examples/imapresource/imapresource.h b/examples/imapresource/imapresource.h
new file mode 100644
index 0000000..6fe15dd
--- /dev/null
+++ b/examples/imapresource/imapresource.h
@@ -0,0 +1,71 @@
+/*
+ *   Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+
+#pragma once
+
+#include "common/genericresource.h"
+
+#include <Async/Async>
+
+#include <flatbuffers/flatbuffers.h>
+
+//TODO: a little ugly to have this in two places, once here and once in Q_PLUGIN_METADATA
+#define PLUGIN_NAME "org.kde.imap"
+
+class ImapMailAdaptorFactory;
+class ImapFolderAdaptorFactory;
+struct Message;
+
+/**
+ * An imap resource.
+ */
+class ImapResource : public Sink::GenericResource
+{
+public:
+    ImapResource(const QByteArray &instanceIdentifier, const QSharedPointer<Sink::Pipeline> &pipeline = QSharedPointer<Sink::Pipeline>());
+    KAsync::Job<void> synchronizeWithSource(Sink::Storage &mainStore, Sink::Storage &synchronizationStore) Q_DECL_OVERRIDE;
+    KAsync::Job<void> inspect(int inspectionType, const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) Q_DECL_OVERRIDE;
+    static void removeFromDisk(const QByteArray &instanceIdentifier);
+private:
+    KAsync::Job<void> replay(Sink::Storage &synchronizationStore, const QByteArray &type, const QByteArray &key, const QByteArray &value) Q_DECL_OVERRIDE;
+
+    QByteArray createFolder(const QString &folderPath, const QByteArray &icon, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction);
+    void synchronizeFolders(const QStringList &folderList, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction);
+    void synchronizeMails(Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction, const QString &path, const QVector<Message> &messages);
+
+    QSharedPointer<ImapMailAdaptorFactory> mMailAdaptorFactory;
+    QSharedPointer<ImapFolderAdaptorFactory> mFolderAdaptorFactory;
+private:
+    QString mServer;
+    int mPort;
+};
+
+class ImapResourceFactory : public Sink::ResourceFactory
+{
+    Q_OBJECT
+    Q_PLUGIN_METADATA(IID "org.kde.imap")
+    Q_INTERFACES(Sink::ResourceFactory)
+
+public:
+    ImapResourceFactory(QObject *parent = 0);
+
+    Sink::Resource *createResource(const QByteArray &instanceIdentifier) Q_DECL_OVERRIDE;
+    void registerFacades(Sink::FacadeFactory &factory) Q_DECL_OVERRIDE;
+};
+
diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp
new file mode 100644
index 0000000..9630096
--- /dev/null
+++ b/examples/imapresource/imapserverproxy.cpp
@@ -0,0 +1,209 @@
+/*
+ *   Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+#include "imapserverproxy.h"
+
+#include <QDir>
+#include <QFile>
+#include <KIMAP/KIMAP/LoginJob>
+#include <KIMAP/KIMAP/SelectJob>
+#include <KIMAP/KIMAP/AppendJob>
+
+#include <KIMAP/KIMAP/SessionUiProxy>
+#include <KCoreAddons/KJob>
+
+#include "log.h"
+
+static KAsync::Job<void> runJob(KJob *job)
+{
+    return KAsync::start<void>([job](KAsync::Future<void> &future) {
+        QObject::connect(job, &KJob::result, job, [&future](KJob *job) {
+            if (job->error()) {
+                Warning() << "Job failed: " << job->errorString();
+                future.setError(job->error(), job->errorString());
+            } else {
+                future.setFinished();
+            }
+        });
+        job->start();
+    });
+}
+
+class SessionUiProxy : public KIMAP::SessionUiProxy {
+  public:
+    bool ignoreSslError( const KSslErrorUiData &errorData ) {
+        return true;
+    }
+};
+
+ImapServerProxy::ImapServerProxy(const QString &serverUrl, int port) : mSession(new KIMAP::Session(serverUrl, port))
+{
+    mSession->setUiProxy(SessionUiProxy::Ptr(new SessionUiProxy));
+    mSession->setTimeout(10);
+}
+
+KAsync::Job<void> ImapServerProxy::login(const QString &username, const QString &password)
+{
+    if (mSession->state() == KIMAP::Session::State::Authenticated || mSession->state() == KIMAP::Session::State::Selected) {
+        return KAsync::null<void>();
+    }
+    auto loginJob = new KIMAP::LoginJob(mSession);
+    loginJob->setUserName(username);
+    loginJob->setPassword(password);
+    loginJob->setAuthenticationMode(KIMAP::LoginJob::Plain);
+    loginJob->setEncryptionMode(KIMAP::LoginJob::EncryptionMode::AnySslVersion);
+    return runJob(loginJob);
+}
+
+KAsync::Job<void> ImapServerProxy::select(const QString &mailbox)
+{
+    if (mSession->state() == KIMAP::Session::State::Disconnected) {
+        return KAsync::error<void>(1, "Not connected");
+    }
+    auto select = new KIMAP::SelectJob(mSession);
+    select->setMailBox(mailbox);
+    // select->setCondstoreEnabled(serverSupportsCondstore());
+    return runJob(select);
+}
+
+KAsync::Job<void> ImapServerProxy::append(const QString &mailbox, const QByteArray &content, const QList<QByteArray> &flags, const QDateTime &internalDate)
+{
+    if (mSession->state() == KIMAP::Session::State::Disconnected) {
+        return KAsync::error<void>(1, "Not connected");
+    }
+    auto append = new KIMAP::AppendJob(mSession);
+    append->setMailBox(mailbox);
+    append->setContent(content);
+    append->setFlags(flags);
+    append->setInternalDate(internalDate);
+    return runJob(append);
+}
+
+KAsync::Job<void> ImapServerProxy::fetch(const KIMAP::ImapSet &set, KIMAP::FetchJob::FetchScope scope, FetchCallback callback)
+{
+    if (mSession->state() == KIMAP::Session::State::Disconnected) {
+        return KAsync::error<void>(1, "Not connected");
+    }
+    auto fetch = new KIMAP::FetchJob(mSession);
+    fetch->setSequenceSet(set);
+    fetch->setUidBased(true);
+    fetch->setScope(scope);
+    QObject::connect(fetch, static_cast<void(KIMAP::FetchJob::*)(const QString &,
+                    const QMap<qint64,qint64> &,
+                    const QMap<qint64,qint64> &,
+                    const QMap<qint64,KIMAP::MessageAttribute> &,
+                    const QMap<qint64,KIMAP::MessageFlags> &,
+                    const QMap<qint64,KIMAP::MessagePtr> &)>(&KIMAP::FetchJob::headersReceived),
+            callback);
+    return runJob(fetch);
+}
+
+KAsync::Job<QList<qint64>> ImapServerProxy::fetchHeaders(const QString &mailbox)
+{
+    auto list = QSharedPointer<QList<qint64>>::create();
+    KIMAP::FetchJob::FetchScope scope;
+    scope.parts.clear();
+    scope.mode = KIMAP::FetchJob::FetchScope::Headers;
+
+    //Fetch headers of all messages
+    return fetch(KIMAP::ImapSet(1, 0), scope,
+            [list](const QString &mailbox,
+                    const QMap<qint64,qint64> &uids,
+                    const QMap<qint64,qint64> &sizes,
+                    const QMap<qint64,KIMAP::MessageAttribute> &attrs,
+                    const QMap<qint64,KIMAP::MessageFlags> &flags,
+                    const QMap<qint64,KIMAP::MessagePtr> &messages) {
+                Trace() << "Received " << uids.size() << " headers from " << mailbox;
+                Trace() << uids.size() << sizes.size() << attrs.size() << flags.size() << messages.size();
+
+                //TODO based on the data available here, figure out which messages to actually fetch
+                //(we only fetched headers and structure so far)
+                //We could i.e. build chunks to fetch based on the size
+
+                for (const auto &id : uids.keys()) {
+                    list->append(uids.value(id));
+                }
+            })
+    .then<QList<qint64>>([list](){
+        return *list;
+    });
+}
+
+KAsync::Job<void> ImapServerProxy::list(KIMAP::ListJob::Option option, const std::function<void(const QList<KIMAP::MailBoxDescriptor> &mailboxes,const QList<QList<QByteArray> > &flags)> &callback)
+{
+    auto listJob = new KIMAP::ListJob(mSession);
+    listJob->setOption(option);
+    // listJob->setQueriedNamespaces(serverNamespaces());
+    QObject::connect(listJob, &KIMAP::ListJob::mailBoxesReceived,
+            listJob, callback);
+    return runJob(listJob);
+}
+
+KAsync::Future<void> ImapServerProxy::fetchFolders(std::function<void(const QStringList &)> callback)
+{
+    Trace() << "Fetching folders";
+    auto job =  login("doe", "doe").then<void>(list(KIMAP::ListJob::IncludeUnsubscribed, [callback](const QList<KIMAP::MailBoxDescriptor> &mailboxes, const QList<QList<QByteArray> > &flags){
+        QStringList list;
+        for (const auto &mailbox : mailboxes) {
+            Trace() << "Found mailbox: " << mailbox.name;
+            list << mailbox.name;
+        }
+        callback(list);
+    }),
+    [](int errorCode, const QString &errorString) {
+        Warning() << "Failed to list folders: " << errorCode << errorString;
+    });
+    return job.exec();
+}
+
+KAsync::Future<void> ImapServerProxy::fetchMessages(const QString &folder, std::function<void(const QVector<Message> &)> callback)
+{
+    auto job = login("doe", "doe").then<void>(select(folder)).then<void, KAsync::Job<void>>([this, callback, folder]() -> KAsync::Job<void> {
+        return fetchHeaders(folder).then<void, KAsync::Job<void>, QList<qint64>>([this, callback](const QList<qint64> &uidsToFetch){
+            Trace() << "Uids to fetch: " << uidsToFetch;
+            if (uidsToFetch.isEmpty()) {
+                Trace() << "Nothing to fetch";
+                return KAsync::null<void>();
+            }
+            KIMAP::FetchJob::FetchScope scope;
+            scope.parts.clear();
+            scope.mode = KIMAP::FetchJob::FetchScope::Full;
+
+            KIMAP::ImapSet set;
+            set.add(uidsToFetch.toVector());
+            return fetch(set, scope,
+                    [callback](const QString &mailbox,
+                            const QMap<qint64,qint64> &uids,
+                            const QMap<qint64,qint64> &sizes,
+                            const QMap<qint64,KIMAP::MessageAttribute> &attrs,
+                            const QMap<qint64,KIMAP::MessageFlags> &flags,
+                            const QMap<qint64,KIMAP::MessagePtr> &messages) {
+                        Trace() << "Received " << uids.size() << " messages from " << mailbox;
+                        Trace() << uids.size() << sizes.size() << attrs.size() << flags.size() << messages.size();
+
+                        QVector<Message> list;
+                        for (const auto &id : uids.keys()) {
+                            list << Message{uids.value(id), sizes.value(id), attrs.value(id), flags.value(id), messages.value(id)};
+                        }
+                        callback(list);
+                    });
+        });
+
+    });
+    return job.exec();
+}
diff --git a/examples/imapresource/imapserverproxy.h b/examples/imapresource/imapserverproxy.h
new file mode 100644
index 0000000..2beb28c
--- /dev/null
+++ b/examples/imapresource/imapserverproxy.h
@@ -0,0 +1,62 @@
+/*
+ *   Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+
+#pragma once
+
+#include <Async/Async>
+
+#include <KMime/KMime/KMimeMessage>
+#include <KIMAP/KIMAP/ListJob>
+#include <KIMAP/KIMAP/Session>
+#include <KIMAP/KIMAP/FetchJob>
+
+struct Message {
+    qint64 uid;
+    qint64 size;
+    QPair<QByteArray, QVariant> attributes;
+    QList<QByteArray> flags;
+    KMime::Message::Ptr msg;
+};
+
+class ImapServerProxy {
+    KIMAP::Session *mSession;
+public:
+    ImapServerProxy(const QString &serverUrl, int port);
+
+    //Standard IMAP calls
+    KAsync::Job<void> login(const QString &username, const QString &password);
+    KAsync::Job<void> select(const QString &mailbox);
+    KAsync::Job<void> append(const QString &mailbox, const QByteArray &content, const QList<QByteArray> &flags = QList<QByteArray>(), const QDateTime &internalDate = QDateTime());
+
+    typedef std::function<void(const QString &,
+                        const QMap<qint64,qint64> &,
+                        const QMap<qint64,qint64> &,
+                        const QMap<qint64,KIMAP::MessageAttribute> &,
+                        const QMap<qint64,KIMAP::MessageFlags> &,
+                        const QMap<qint64,KIMAP::MessagePtr> &)> FetchCallback;
+
+    KAsync::Job<void> fetch(const KIMAP::ImapSet &set, KIMAP::FetchJob::FetchScope scope, FetchCallback callback);
+    KAsync::Job<void> list(KIMAP::ListJob::Option option, const std::function<void(const QList<KIMAP::MailBoxDescriptor> &mailboxes,const QList<QList<QByteArray> > &flags)> &callback);
+
+    //Composed calls that do login etc.
+    KAsync::Job<QList<qint64>> fetchHeaders(const QString &mailbox);
+
+    KAsync::Future<void> fetchFolders(std::function<void(const QStringList &)> callback);
+    KAsync::Future<void> fetchMessages(const QString &folder, std::function<void(const QVector<Message> &)> callback);
+};
diff --git a/examples/imapresource/tests/CMakeLists.txt b/examples/imapresource/tests/CMakeLists.txt
new file mode 100644
index 0000000..cdd5fcb
--- /dev/null
+++ b/examples/imapresource/tests/CMakeLists.txt
@@ -0,0 +1,21 @@
+set(CMAKE_AUTOMOC ON)
+include_directories(
+    ${CMAKE_CURRENT_BINARY_DIR}
+    )
+
+macro(auto_tests)
+    foreach(_testname ${ARGN})
+        add_executable(${_testname} ${_testname}.cpp)
+        # generate_flatbuffers(${_testname} calendar)
+        add_test(${_testname} ${_testname})
+        qt5_use_modules(${_testname} Core Test Concurrent)
+        target_link_libraries(${_testname} sink libhawd)
+    endforeach(_testname)
+endmacro(auto_tests)
+
+auto_tests (
+    imapresourcetest
+)
+target_link_libraries(imapresourcetest sink_resource_imap)
+
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/resetmailbox.sh DESTINATION bin PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ)
diff --git a/examples/imapresource/tests/imapresourcetest.cpp b/examples/imapresource/tests/imapresourcetest.cpp
new file mode 100644
index 0000000..27d7d6d
--- /dev/null
+++ b/examples/imapresource/tests/imapresourcetest.cpp
@@ -0,0 +1,162 @@
+#include <QtTest>
+
+#include <QString>
+#include <KMime/Message>
+
+// #include "imapresource/imapresource.h"
+#include "store.h"
+#include "resourcecontrol.h"
+#include "commands.h"
+#include "entitybuffer.h"
+#include "resourceconfig.h"
+#include "modelresult.h"
+#include "pipeline.h"
+#include "log.h"
+#include "test.h"
+#include "../imapresource.h"
+#include "../imapserverproxy.h"
+
+#define ASYNCCOMPARE(actual, expected) \
+do {\
+    if (!QTest::qCompare(actual, expected, #actual, #expected, __FILE__, __LINE__))\
+        return KAsync::error<void>(1, "Comparison failed.");\
+} while (0)
+
+#define ASYNCVERIFY(statement) \
+do {\
+    if (!QTest::qVerify((statement), #statement, "", __FILE__, __LINE__))\
+        return KAsync::error<void>(1, "Verify failed.");\
+} while (0)
+
+#define VERIFYEXEC(statement) \
+do {\
+    auto result = statement.exec(); \
+    result.waitForFinished(); \
+    if (!QTest::qVerify(!result.errorCode(), #statement, "", __FILE__, __LINE__))\
+        return;\
+} while (0)
+
+using namespace Sink;
+using namespace Sink::ApplicationDomain;
+
+/**
+ * Test of complete system using the imap resource.
+ *
+ * This test requires the imap resource installed.
+ */
+class ImapResourceTest : public QObject
+{
+    Q_OBJECT
+
+    QTemporaryDir tempDir;
+    QString targetPath;
+private slots:
+    void initTestCase()
+    {
+
+        //FIXME initTest only works for the current process,
+        //we also have to start resources in test-mode
+        // Sink::Test::initTest();
+        Sink::Log::setDebugOutputLevel(Sink::Log::Trace);
+        ::ImapResource::removeFromDisk("org.kde.imap.instance1");
+        system("resetmailbox.sh");
+        // auto resource = ApplicationDomain::ImapResource::create("account1");
+        Sink::ApplicationDomain::SinkResource resource;
+        resource.setProperty("identifier", "org.kde.imap.instance1");
+        resource.setProperty("type", "org.kde.imap");
+        resource.setProperty("server", "localhost");
+        resource.setProperty("port", 993);
+        Sink::Store::create(resource).exec().waitForFinished();
+    }
+
+    void cleanup()
+    {
+        Sink::ResourceControl::shutdown(QByteArray("org.kde.imap.instance1")).exec().waitForFinished();
+        ::ImapResource::removeFromDisk("org.kde.imap.instance1");
+    }
+
+    void init()
+    {
+        qDebug();
+        qDebug() << "-----------------------------------------";
+        qDebug();
+        Sink::ResourceControl::start(QByteArray("org.kde.imap.instance1")).exec().waitForFinished();
+    }
+
+    void testListFolders()
+    {
+        Sink::Query query;
+        query.resources << "org.kde.imap.instance1";
+        query.request<Folder::Name>();
+
+        // Ensure all local data is processed
+        VERIFYEXEC(Store::synchronize(query));
+        ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished();
+
+        auto job = Store::fetchAll<Folder>(query).then<void, QList<Folder::Ptr>>([](const QList<Folder::Ptr> &folders) {
+            QCOMPARE(folders.size(), 2);
+            QStringList names;
+            for (const auto &folder : folders) {
+                names << folder->getName();
+            }
+            QVERIFY(names.contains("INBOX"));
+            QVERIFY(names.contains("INBOX.test"));
+        });
+        VERIFYEXEC(job);
+    }
+
+    void testListMails()
+    {
+        Sink::Query query;
+        query.resources << "org.kde.imap.instance1";
+        query.request<Mail::Subject>().request<Mail::MimeMessage>();
+
+        // Ensure all local data is processed
+        VERIFYEXEC(Store::synchronize(query));
+        ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished();
+
+        auto job = Store::fetchAll<Mail>(query).then<void, QList<Mail::Ptr>>([](const QList<Mail::Ptr> &mails) {
+            QCOMPARE(mails.size(), 1);
+            QVERIFY(mails.first()->getSubject().startsWith(QString("[Nepomuk] Jenkins build is still unstable")));
+            const auto data = mails.first()->getMimeMessage();
+            QVERIFY(!data.isEmpty());
+
+            KMime::Message m;
+            m.setContent(data);
+            m.parse();
+            QCOMPARE(mails.first()->getSubject(), m.subject(true)->asUnicodeString());
+        });
+        VERIFYEXEC(job);
+    }
+
+    void testFetchNewMessages()
+    {
+        Sink::Query query;
+        query.resources << "org.kde.imap.instance1";
+        query.request<Mail::Subject>().request<Mail::MimeMessage>();
+
+        // Ensure all local data is processed
+        VERIFYEXEC(Store::synchronize(query));
+        ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished();
+
+        ImapServerProxy imap("localhost", 993);
+        imap.login("doe", "doe").exec().waitForFinished();
+
+        auto msg = KMime::Message::Ptr::create();
+        msg->subject(true)->fromUnicodeString("Foobar", "utf8");
+        msg->assemble();
+
+        VERIFYEXEC(imap.append("INBOX.test", msg->encodedContent(true)));
+
+        Store::synchronize(query).exec().waitForFinished();
+        ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished();
+
+        auto job = Store::fetchAll<Mail>(query).then<void, QList<Mail::Ptr>>([](const QList<Mail::Ptr> &mails) {
+            QCOMPARE(mails.size(), 2);
+        });
+        VERIFYEXEC(job);
+    }
+};
+
+QTEST_MAIN(ImapResourceTest)
+#include "imapresourcetest.moc"
diff --git a/examples/imapresource/tests/resetmailbox.sh b/examples/imapresource/tests/resetmailbox.sh
new file mode 100644
index 0000000..966115a
--- /dev/null
+++ b/examples/imapresource/tests/resetmailbox.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+ 
+sudo echo "dm user.doe.test" | cyradm --auth PLAIN -u cyrus -w admin localhost
+sudo echo "cm user.doe.test" | cyradm --auth PLAIN -u cyrus -w admin localhost
+sudo echo "sam user.doe cyrus c" | cyradm --auth PLAIN -u cyrus -w admin localhost
+sudo echo "sam user.doe.test cyrus c" | cyradm --auth PLAIN -u cyrus -w admin localhost
+# sudo rm -R /var/spool/imap/d/user/doe/*
+sudo cp /work/source/Sink/tests/data/maildir1/cur/1365777830.R28.localhost.localdomain\:2\,S /var/spool/imap/d/user/doe/test/1.
+sudo chown cyrus:mail /var/spool/imap/d/user/doe/test/1.
+sudo /usr/lib/cyrus-imapd/reconstruct "user.doe.test"