From 6864e4accaafa4fa90332719bff5a85a0e92b242 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sun, 22 May 2016 13:10:39 +0200 Subject: ImapResource prototype --- examples/CMakeLists.txt | 1 + examples/imapresource/CMakeLists.txt | 18 ++ examples/imapresource/domainadaptor.cpp | 35 +++ examples/imapresource/domainadaptor.h | 38 ++++ examples/imapresource/facade.cpp | 44 ++++ examples/imapresource/facade.h | 36 +++ examples/imapresource/imapresource.cpp | 277 +++++++++++++++++++++++ examples/imapresource/imapresource.h | 71 ++++++ examples/imapresource/imapserverproxy.cpp | 209 +++++++++++++++++ examples/imapresource/imapserverproxy.h | 62 +++++ examples/imapresource/tests/CMakeLists.txt | 21 ++ examples/imapresource/tests/imapresourcetest.cpp | 162 +++++++++++++ examples/imapresource/tests/resetmailbox.sh | 10 + 13 files changed, 984 insertions(+) create mode 100644 examples/imapresource/CMakeLists.txt create mode 100644 examples/imapresource/domainadaptor.cpp create mode 100644 examples/imapresource/domainadaptor.h create mode 100644 examples/imapresource/facade.cpp create mode 100644 examples/imapresource/facade.h create mode 100644 examples/imapresource/imapresource.cpp create mode 100644 examples/imapresource/imapresource.h create mode 100644 examples/imapresource/imapserverproxy.cpp create mode 100644 examples/imapresource/imapserverproxy.h create mode 100644 examples/imapresource/tests/CMakeLists.txt create mode 100644 examples/imapresource/tests/imapresourcetest.cpp create mode 100644 examples/imapresource/tests/resetmailbox.sh (limited to 'examples') diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index d5fcacf..59753bb 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -7,5 +7,6 @@ add_subdirectory(dummyresource) if (BUILD_MAILDIR) # a maildir resource implementation add_subdirectory(maildirresource) + add_subdirectory(imapresource) endif() add_subdirectory(mailtransportresource) diff --git a/examples/imapresource/CMakeLists.txt b/examples/imapresource/CMakeLists.txt new file mode 100644 index 0000000..d5320a6 --- /dev/null +++ b/examples/imapresource/CMakeLists.txt @@ -0,0 +1,18 @@ +project(sink_resource_imap) + +add_definitions(-DQT_PLUGIN) +include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +find_package(KF5 COMPONENTS REQUIRED Mime IMAP KIO) +find_package(KF5CoreAddons REQUIRED) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +add_library(${PROJECT_NAME} SHARED facade.cpp imapresource.cpp domainadaptor.cpp imapserverproxy.cpp) +qt5_use_modules(${PROJECT_NAME} Core Network) +#We need CoreAddons for KJob and KIOCore for KTcpSocket. Both used in KIMAP +target_link_libraries(${PROJECT_NAME} sink KF5::Mime KF5::IMAP KF5::CoreAddons KF5::KIOCore) + +install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${SINK_RESOURCE_PLUGINS_PATH}) + +add_subdirectory(tests) diff --git a/examples/imapresource/domainadaptor.cpp b/examples/imapresource/domainadaptor.cpp new file mode 100644 index 0000000..4e74ad2 --- /dev/null +++ b/examples/imapresource/domainadaptor.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "domainadaptor.h" + +using namespace flatbuffers; + +ImapMailAdaptorFactory::ImapMailAdaptorFactory() + : DomainTypeAdaptorFactory() +{ + +} + +ImapFolderAdaptorFactory::ImapFolderAdaptorFactory() + : DomainTypeAdaptorFactory() +{ + +} + diff --git a/examples/imapresource/domainadaptor.h b/examples/imapresource/domainadaptor.h new file mode 100644 index 0000000..06a513c --- /dev/null +++ b/examples/imapresource/domainadaptor.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include +#include "mail_generated.h" +#include "folder_generated.h" +#include "dummy_generated.h" + +class ImapMailAdaptorFactory : public DomainTypeAdaptorFactory +{ +public: + ImapMailAdaptorFactory(); + virtual ~ImapMailAdaptorFactory() {}; +}; + +class ImapFolderAdaptorFactory : public DomainTypeAdaptorFactory +{ +public: + ImapFolderAdaptorFactory(); + virtual ~ImapFolderAdaptorFactory() {}; +}; diff --git a/examples/imapresource/facade.cpp b/examples/imapresource/facade.cpp new file mode 100644 index 0000000..d338b01 --- /dev/null +++ b/examples/imapresource/facade.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "facade.h" + +#include +#include + +#include "domainadaptor.h" +#include "queryrunner.h" + +ImapResourceMailFacade::ImapResourceMailFacade(const QByteArray &instanceIdentifier) + : Sink::GenericFacade(instanceIdentifier, QSharedPointer::create()) +{ +} + +ImapResourceMailFacade::~ImapResourceMailFacade() +{ +} + +ImapResourceFolderFacade::ImapResourceFolderFacade(const QByteArray &instanceIdentifier) + : Sink::GenericFacade(instanceIdentifier, QSharedPointer::create()) +{ +} + +ImapResourceFolderFacade::~ImapResourceFolderFacade() +{ +} diff --git a/examples/imapresource/facade.h b/examples/imapresource/facade.h new file mode 100644 index 0000000..479ad96 --- /dev/null +++ b/examples/imapresource/facade.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2014 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include "common/facade.h" + +class ImapResourceMailFacade : public Sink::GenericFacade +{ +public: + ImapResourceMailFacade(const QByteArray &instanceIdentifier); + virtual ~ImapResourceMailFacade(); +}; + +class ImapResourceFolderFacade : public Sink::GenericFacade +{ +public: + ImapResourceFolderFacade(const QByteArray &instanceIdentifier); + virtual ~ImapResourceFolderFacade(); +}; diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp new file mode 100644 index 0000000..baa88b9 --- /dev/null +++ b/examples/imapresource/imapresource.cpp @@ -0,0 +1,277 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "imapresource.h" +#include "facade.h" +#include "entitybuffer.h" +#include "pipeline.h" +#include "mail_generated.h" +#include "createentity_generated.h" +#include "modifyentity_generated.h" +#include "deleteentity_generated.h" +#include "domainadaptor.h" +#include "resourceconfig.h" +#include "commands.h" +#include "index.h" +#include "log.h" +#include "domain/mail.h" +#include "definitions.h" +#include "facadefactory.h" +#include "indexupdater.h" +#include "inspection.h" +#include +#include +#include +#include + +#include "imapserverproxy.h" + +//This is the resources entity type, and not the domain type +#define ENTITY_TYPE_MAIL "mail" +#define ENTITY_TYPE_FOLDER "folder" + +#undef DEBUG_AREA +#define DEBUG_AREA "resource.imap" + + +ImapResource::ImapResource(const QByteArray &instanceIdentifier, const QSharedPointer &pipeline) + : Sink::GenericResource(instanceIdentifier, pipeline), + mMailAdaptorFactory(QSharedPointer::create()), + mFolderAdaptorFactory(QSharedPointer::create()) +{ + auto config = ResourceConfig::getConfiguration(instanceIdentifier); + mServer = config.value("server").toString(); + mPort = config.value("port").toInt(); + + // auto folderUpdater = new FolderUpdater(QByteArray()); + addType(ENTITY_TYPE_MAIL, mMailAdaptorFactory, + QVector() << new DefaultIndexUpdater); + addType(ENTITY_TYPE_FOLDER, mFolderAdaptorFactory, + QVector() << new DefaultIndexUpdater); +} + +QByteArray ImapResource::createFolder(const QString &folderPath, const QByteArray &icon, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction) +{ + auto remoteId = folderPath.toUtf8(); + auto bufferType = ENTITY_TYPE_FOLDER; + Sink::ApplicationDomain::Folder folder; + folder.setProperty("name", folderPath.split('/').last()); + folder.setProperty("icon", icon); + + // if (!md.isRoot()) { + // folder.setProperty("parent", resolveRemoteId(ENTITY_TYPE_FOLDER, md.parent().path().toUtf8(), synchronizationTransaction)); + // } + createOrModify(transaction, synchronizationTransaction, *mFolderAdaptorFactory, bufferType, remoteId, folder); + return remoteId; +} + +void ImapResource::synchronizeFolders(const QStringList &folderList, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction) +{ + const QByteArray bufferType = ENTITY_TYPE_FOLDER; + Trace() << "Found folders " << folderList; + + scanForRemovals(transaction, synchronizationTransaction, bufferType, + [&bufferType, &transaction](const std::function &callback) { + //TODO Instead of iterating over all entries in the database, which can also pick up the same item multiple times, + //we should rather iterate over an index that contains every uid exactly once. The remoteId index would be such an index, + //but we currently fail to iterate over all entries in an index it seems. + // auto remoteIds = synchronizationTransaction.openDatabase("rid.mapping." + bufferType, std::function(), true); + auto mainDatabase = Sink::Storage::mainDatabase(transaction, bufferType); + mainDatabase.scan("", [&](const QByteArray &key, const QByteArray &) { + callback(key); + return true; + }); + }, + [&folderList](const QByteArray &remoteId) -> bool { + return folderList.contains(remoteId); + } + ); + + for (const auto folderPath : folderList) { + createFolder(folderPath, "folder", transaction, synchronizationTransaction); + } +} + +void ImapResource::synchronizeMails(Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction, const QString &path, const QVector &messages) +{ + auto time = QSharedPointer::create(); + time->start(); + const QByteArray bufferType = ENTITY_TYPE_MAIL; + + + Trace() << "Importing new mail."; + + // Trace() << "Looking into " << listingPath; + + const auto folderLocalId = resolveRemoteId(ENTITY_TYPE_FOLDER, path.toUtf8(), synchronizationTransaction); + + //This is not a full listing + // auto property = "folder"; + // scanForRemovals(transaction, synchronizationTransaction, bufferType, + // [&](const std::function &callback) { + // Index index(bufferType + ".index." + property, transaction); + // index.lookup(folderLocalId, [&](const QByteArray &sinkId) { + // callback(sinkId); + // }, + // [&](const Index::Error &error) { + // Warning() << "Error in index: " << error.message << property; + // }); + // }, + // [](const QByteArray &remoteId) -> bool { + // return QFile(remoteId).exists(); + // } + // ); + + mSynchronizerQueue.startTransaction(); + int count = 0; + for (const auto &message : messages) { + count++; + const auto remoteId = path.toUtf8() + "/" + QByteArray::number(message.uid); + + Trace() << "Found a mail " << remoteId << message.msg->subject(true)->asUnicodeString() << message.flags; + + Sink::ApplicationDomain::Mail mail; + mail.setFolder(folderLocalId); + //FIXME this should come from the mime message, extracted in the pipeline + mail.setExtractedSubject(message.msg->subject(true)->asUnicodeString()); + + auto filePath = Sink::resourceStorageLocation(mResourceInstanceIdentifier) + "/" + remoteId; + QDir().mkpath(Sink::resourceStorageLocation(mResourceInstanceIdentifier) + "/" + path.toUtf8()); + QFile file(filePath); + if (!file.open(QIODevice::WriteOnly)) { + Warning() << "Failed to open file for writing: " << file.errorString(); + } + const auto content = message.msg->encodedContent(); + file.write(content); + mail.setMimeMessagePath(filePath); + //FIXME Not sure if these are the actual flags + mail.setUnread(message.flags.contains("\\SEEN")); + mail.setImportant(message.flags.contains("\\FLAGGED")); + + createOrModify(transaction, synchronizationTransaction, *mMailAdaptorFactory, bufferType, remoteId, mail); + } + mSynchronizerQueue.commit(); + const auto elapsed = time->elapsed(); + Log() << "Synchronized " << count << " mails in " << path << Sink::Log::TraceTime(elapsed) << " " << elapsed/qMax(count, 1) << " [ms/mail]"; +} + +KAsync::Job ImapResource::synchronizeWithSource(Sink::Storage &mainStore, Sink::Storage &synchronizationStore) +{ + Log() << " Synchronizing"; + return KAsync::start([this, &mainStore, &synchronizationStore](KAsync::Future future) { + ImapServerProxy imap(mServer, mPort); + QStringList folderList; + // QList> waitCondition; + auto folderFuture = imap.fetchFolders([this, &imap, &mainStore, &synchronizationStore, &folderList](const QStringList &folders) { + auto transaction = mainStore.createTransaction(Sink::Storage::ReadOnly); + auto syncTransaction = synchronizationStore.createTransaction(Sink::Storage::ReadWrite); + synchronizeFolders(folders, transaction, syncTransaction); + transaction.commit(); + syncTransaction.commit(); + folderList << folders; + + }); + folderFuture.waitForFinished(); + if (folderFuture.errorCode()) { + future.setError(1, "Folder list sync failed"); + return; + } + + for (const auto &folder : folderList) { + // auto transaction = mainStore.createTransaction(Sink::Storage::ReadOnly); + // auto syncTransaction = synchronizationStore.createTransaction(Sink::Storage::ReadOnly); + + //TODO load entity to read sync settings should we have some (if the folder is existing already) + //Note that this will not work if we change any of those settings in the pipeline + // + // auto mainDatabase = Sink::Storage::mainDatabase(transaction, ENTITY_TYPE_FOLDER); + // const auto sinkId = resolveRemoteId(ENTITY_TYPE_FOLDER, folder.toUtf8(), syncTransaction); + // const auto found = mainDatabase.contains(sinkId); + // if (found) { + // if (auto current = getLatest(mainDatabase, sinkId, mFolderAdaptorFactory)) { + // + // } + // } + + // transaction.commit(); + // syncTransaction.commit(); + + auto messagesFuture = imap.fetchMessages(folder, [this, &mainStore, &synchronizationStore, folder](const QVector &messages) { + auto transaction = mainStore.createTransaction(Sink::Storage::ReadOnly); + auto syncTransaction = synchronizationStore.createTransaction(Sink::Storage::ReadWrite); + Trace() << "Synchronizing mails" << folder; + synchronizeMails(transaction, syncTransaction, folder, messages); + transaction.commit(); + syncTransaction.commit(); + }); + messagesFuture.waitForFinished(); + if (messagesFuture.errorCode()) { + future.setError(1, "Folder sync failed: " + folder); + return; + } + } + + + // auto transaction = mainStore.createTransaction(Sink::Storage::ReadWrite); + // auto mainDatabase = Sink::Storage::mainDatabase(transaction, ENTITY_TYPE_FOLDER); + // mainDatabase.scan("", [&](const QByteArray &key, const QByteArray &data) { + // return true; + // }); + //TODO now fetch all folders and iterate over them and synchronize each one + + Log() << "Done Synchronizing"; + future.setFinished(); + }); +} + +KAsync::Job ImapResource::replay(Sink::Storage &synchronizationStore, const QByteArray &type, const QByteArray &key, const QByteArray &value) +{ + //TODO implement + return KAsync::null(); +} + +void ImapResource::removeFromDisk(const QByteArray &instanceIdentifier) +{ + GenericResource::removeFromDisk(instanceIdentifier); + Sink::Storage(Sink::storageLocation(), instanceIdentifier + ".synchronization", Sink::Storage::ReadWrite).removeFromDisk(); +} + +KAsync::Job ImapResource::inspect(int inspectionType, const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) +{ + //TODO + return KAsync::null(); +} + +ImapResourceFactory::ImapResourceFactory(QObject *parent) + : Sink::ResourceFactory(parent) +{ + +} + +Sink::Resource *ImapResourceFactory::createResource(const QByteArray &instanceIdentifier) +{ + return new ImapResource(instanceIdentifier); +} + +void ImapResourceFactory::registerFacades(Sink::FacadeFactory &factory) +{ + factory.registerFacade(PLUGIN_NAME); + factory.registerFacade(PLUGIN_NAME); +} + diff --git a/examples/imapresource/imapresource.h b/examples/imapresource/imapresource.h new file mode 100644 index 0000000..6fe15dd --- /dev/null +++ b/examples/imapresource/imapresource.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include "common/genericresource.h" + +#include + +#include + +//TODO: a little ugly to have this in two places, once here and once in Q_PLUGIN_METADATA +#define PLUGIN_NAME "org.kde.imap" + +class ImapMailAdaptorFactory; +class ImapFolderAdaptorFactory; +struct Message; + +/** + * An imap resource. + */ +class ImapResource : public Sink::GenericResource +{ +public: + ImapResource(const QByteArray &instanceIdentifier, const QSharedPointer &pipeline = QSharedPointer()); + KAsync::Job synchronizeWithSource(Sink::Storage &mainStore, Sink::Storage &synchronizationStore) Q_DECL_OVERRIDE; + KAsync::Job inspect(int inspectionType, const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) Q_DECL_OVERRIDE; + static void removeFromDisk(const QByteArray &instanceIdentifier); +private: + KAsync::Job replay(Sink::Storage &synchronizationStore, const QByteArray &type, const QByteArray &key, const QByteArray &value) Q_DECL_OVERRIDE; + + QByteArray createFolder(const QString &folderPath, const QByteArray &icon, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction); + void synchronizeFolders(const QStringList &folderList, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction); + void synchronizeMails(Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction, const QString &path, const QVector &messages); + + QSharedPointer mMailAdaptorFactory; + QSharedPointer mFolderAdaptorFactory; +private: + QString mServer; + int mPort; +}; + +class ImapResourceFactory : public Sink::ResourceFactory +{ + Q_OBJECT + Q_PLUGIN_METADATA(IID "org.kde.imap") + Q_INTERFACES(Sink::ResourceFactory) + +public: + ImapResourceFactory(QObject *parent = 0); + + Sink::Resource *createResource(const QByteArray &instanceIdentifier) Q_DECL_OVERRIDE; + void registerFacades(Sink::FacadeFactory &factory) Q_DECL_OVERRIDE; +}; + diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp new file mode 100644 index 0000000..9630096 --- /dev/null +++ b/examples/imapresource/imapserverproxy.cpp @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "imapserverproxy.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include "log.h" + +static KAsync::Job runJob(KJob *job) +{ + return KAsync::start([job](KAsync::Future &future) { + QObject::connect(job, &KJob::result, job, [&future](KJob *job) { + if (job->error()) { + Warning() << "Job failed: " << job->errorString(); + future.setError(job->error(), job->errorString()); + } else { + future.setFinished(); + } + }); + job->start(); + }); +} + +class SessionUiProxy : public KIMAP::SessionUiProxy { + public: + bool ignoreSslError( const KSslErrorUiData &errorData ) { + return true; + } +}; + +ImapServerProxy::ImapServerProxy(const QString &serverUrl, int port) : mSession(new KIMAP::Session(serverUrl, port)) +{ + mSession->setUiProxy(SessionUiProxy::Ptr(new SessionUiProxy)); + mSession->setTimeout(10); +} + +KAsync::Job ImapServerProxy::login(const QString &username, const QString &password) +{ + if (mSession->state() == KIMAP::Session::State::Authenticated || mSession->state() == KIMAP::Session::State::Selected) { + return KAsync::null(); + } + auto loginJob = new KIMAP::LoginJob(mSession); + loginJob->setUserName(username); + loginJob->setPassword(password); + loginJob->setAuthenticationMode(KIMAP::LoginJob::Plain); + loginJob->setEncryptionMode(KIMAP::LoginJob::EncryptionMode::AnySslVersion); + return runJob(loginJob); +} + +KAsync::Job ImapServerProxy::select(const QString &mailbox) +{ + if (mSession->state() == KIMAP::Session::State::Disconnected) { + return KAsync::error(1, "Not connected"); + } + auto select = new KIMAP::SelectJob(mSession); + select->setMailBox(mailbox); + // select->setCondstoreEnabled(serverSupportsCondstore()); + return runJob(select); +} + +KAsync::Job ImapServerProxy::append(const QString &mailbox, const QByteArray &content, const QList &flags, const QDateTime &internalDate) +{ + if (mSession->state() == KIMAP::Session::State::Disconnected) { + return KAsync::error(1, "Not connected"); + } + auto append = new KIMAP::AppendJob(mSession); + append->setMailBox(mailbox); + append->setContent(content); + append->setFlags(flags); + append->setInternalDate(internalDate); + return runJob(append); +} + +KAsync::Job ImapServerProxy::fetch(const KIMAP::ImapSet &set, KIMAP::FetchJob::FetchScope scope, FetchCallback callback) +{ + if (mSession->state() == KIMAP::Session::State::Disconnected) { + return KAsync::error(1, "Not connected"); + } + auto fetch = new KIMAP::FetchJob(mSession); + fetch->setSequenceSet(set); + fetch->setUidBased(true); + fetch->setScope(scope); + QObject::connect(fetch, static_cast &, + const QMap &, + const QMap &, + const QMap &, + const QMap &)>(&KIMAP::FetchJob::headersReceived), + callback); + return runJob(fetch); +} + +KAsync::Job> ImapServerProxy::fetchHeaders(const QString &mailbox) +{ + auto list = QSharedPointer>::create(); + KIMAP::FetchJob::FetchScope scope; + scope.parts.clear(); + scope.mode = KIMAP::FetchJob::FetchScope::Headers; + + //Fetch headers of all messages + return fetch(KIMAP::ImapSet(1, 0), scope, + [list](const QString &mailbox, + const QMap &uids, + const QMap &sizes, + const QMap &attrs, + const QMap &flags, + const QMap &messages) { + Trace() << "Received " << uids.size() << " headers from " << mailbox; + Trace() << uids.size() << sizes.size() << attrs.size() << flags.size() << messages.size(); + + //TODO based on the data available here, figure out which messages to actually fetch + //(we only fetched headers and structure so far) + //We could i.e. build chunks to fetch based on the size + + for (const auto &id : uids.keys()) { + list->append(uids.value(id)); + } + }) + .then>([list](){ + return *list; + }); +} + +KAsync::Job ImapServerProxy::list(KIMAP::ListJob::Option option, const std::function &mailboxes,const QList > &flags)> &callback) +{ + auto listJob = new KIMAP::ListJob(mSession); + listJob->setOption(option); + // listJob->setQueriedNamespaces(serverNamespaces()); + QObject::connect(listJob, &KIMAP::ListJob::mailBoxesReceived, + listJob, callback); + return runJob(listJob); +} + +KAsync::Future ImapServerProxy::fetchFolders(std::function callback) +{ + Trace() << "Fetching folders"; + auto job = login("doe", "doe").then(list(KIMAP::ListJob::IncludeUnsubscribed, [callback](const QList &mailboxes, const QList > &flags){ + QStringList list; + for (const auto &mailbox : mailboxes) { + Trace() << "Found mailbox: " << mailbox.name; + list << mailbox.name; + } + callback(list); + }), + [](int errorCode, const QString &errorString) { + Warning() << "Failed to list folders: " << errorCode << errorString; + }); + return job.exec(); +} + +KAsync::Future ImapServerProxy::fetchMessages(const QString &folder, std::function &)> callback) +{ + auto job = login("doe", "doe").then(select(folder)).then>([this, callback, folder]() -> KAsync::Job { + return fetchHeaders(folder).then, QList>([this, callback](const QList &uidsToFetch){ + Trace() << "Uids to fetch: " << uidsToFetch; + if (uidsToFetch.isEmpty()) { + Trace() << "Nothing to fetch"; + return KAsync::null(); + } + KIMAP::FetchJob::FetchScope scope; + scope.parts.clear(); + scope.mode = KIMAP::FetchJob::FetchScope::Full; + + KIMAP::ImapSet set; + set.add(uidsToFetch.toVector()); + return fetch(set, scope, + [callback](const QString &mailbox, + const QMap &uids, + const QMap &sizes, + const QMap &attrs, + const QMap &flags, + const QMap &messages) { + Trace() << "Received " << uids.size() << " messages from " << mailbox; + Trace() << uids.size() << sizes.size() << attrs.size() << flags.size() << messages.size(); + + QVector list; + for (const auto &id : uids.keys()) { + list << Message{uids.value(id), sizes.value(id), attrs.value(id), flags.value(id), messages.value(id)}; + } + callback(list); + }); + }); + + }); + return job.exec(); +} diff --git a/examples/imapresource/imapserverproxy.h b/examples/imapresource/imapserverproxy.h new file mode 100644 index 0000000..2beb28c --- /dev/null +++ b/examples/imapresource/imapserverproxy.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include + +#include +#include +#include +#include + +struct Message { + qint64 uid; + qint64 size; + QPair attributes; + QList flags; + KMime::Message::Ptr msg; +}; + +class ImapServerProxy { + KIMAP::Session *mSession; +public: + ImapServerProxy(const QString &serverUrl, int port); + + //Standard IMAP calls + KAsync::Job login(const QString &username, const QString &password); + KAsync::Job select(const QString &mailbox); + KAsync::Job append(const QString &mailbox, const QByteArray &content, const QList &flags = QList(), const QDateTime &internalDate = QDateTime()); + + typedef std::function &, + const QMap &, + const QMap &, + const QMap &, + const QMap &)> FetchCallback; + + KAsync::Job fetch(const KIMAP::ImapSet &set, KIMAP::FetchJob::FetchScope scope, FetchCallback callback); + KAsync::Job list(KIMAP::ListJob::Option option, const std::function &mailboxes,const QList > &flags)> &callback); + + //Composed calls that do login etc. + KAsync::Job> fetchHeaders(const QString &mailbox); + + KAsync::Future fetchFolders(std::function callback); + KAsync::Future fetchMessages(const QString &folder, std::function &)> callback); +}; diff --git a/examples/imapresource/tests/CMakeLists.txt b/examples/imapresource/tests/CMakeLists.txt new file mode 100644 index 0000000..cdd5fcb --- /dev/null +++ b/examples/imapresource/tests/CMakeLists.txt @@ -0,0 +1,21 @@ +set(CMAKE_AUTOMOC ON) +include_directories( + ${CMAKE_CURRENT_BINARY_DIR} + ) + +macro(auto_tests) + foreach(_testname ${ARGN}) + add_executable(${_testname} ${_testname}.cpp) + # generate_flatbuffers(${_testname} calendar) + add_test(${_testname} ${_testname}) + qt5_use_modules(${_testname} Core Test Concurrent) + target_link_libraries(${_testname} sink libhawd) + endforeach(_testname) +endmacro(auto_tests) + +auto_tests ( + imapresourcetest +) +target_link_libraries(imapresourcetest sink_resource_imap) + +install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/resetmailbox.sh DESTINATION bin PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ) diff --git a/examples/imapresource/tests/imapresourcetest.cpp b/examples/imapresource/tests/imapresourcetest.cpp new file mode 100644 index 0000000..27d7d6d --- /dev/null +++ b/examples/imapresource/tests/imapresourcetest.cpp @@ -0,0 +1,162 @@ +#include + +#include +#include + +// #include "imapresource/imapresource.h" +#include "store.h" +#include "resourcecontrol.h" +#include "commands.h" +#include "entitybuffer.h" +#include "resourceconfig.h" +#include "modelresult.h" +#include "pipeline.h" +#include "log.h" +#include "test.h" +#include "../imapresource.h" +#include "../imapserverproxy.h" + +#define ASYNCCOMPARE(actual, expected) \ +do {\ + if (!QTest::qCompare(actual, expected, #actual, #expected, __FILE__, __LINE__))\ + return KAsync::error(1, "Comparison failed.");\ +} while (0) + +#define ASYNCVERIFY(statement) \ +do {\ + if (!QTest::qVerify((statement), #statement, "", __FILE__, __LINE__))\ + return KAsync::error(1, "Verify failed.");\ +} while (0) + +#define VERIFYEXEC(statement) \ +do {\ + auto result = statement.exec(); \ + result.waitForFinished(); \ + if (!QTest::qVerify(!result.errorCode(), #statement, "", __FILE__, __LINE__))\ + return;\ +} while (0) + +using namespace Sink; +using namespace Sink::ApplicationDomain; + +/** + * Test of complete system using the imap resource. + * + * This test requires the imap resource installed. + */ +class ImapResourceTest : public QObject +{ + Q_OBJECT + + QTemporaryDir tempDir; + QString targetPath; +private slots: + void initTestCase() + { + + //FIXME initTest only works for the current process, + //we also have to start resources in test-mode + // Sink::Test::initTest(); + Sink::Log::setDebugOutputLevel(Sink::Log::Trace); + ::ImapResource::removeFromDisk("org.kde.imap.instance1"); + system("resetmailbox.sh"); + // auto resource = ApplicationDomain::ImapResource::create("account1"); + Sink::ApplicationDomain::SinkResource resource; + resource.setProperty("identifier", "org.kde.imap.instance1"); + resource.setProperty("type", "org.kde.imap"); + resource.setProperty("server", "localhost"); + resource.setProperty("port", 993); + Sink::Store::create(resource).exec().waitForFinished(); + } + + void cleanup() + { + Sink::ResourceControl::shutdown(QByteArray("org.kde.imap.instance1")).exec().waitForFinished(); + ::ImapResource::removeFromDisk("org.kde.imap.instance1"); + } + + void init() + { + qDebug(); + qDebug() << "-----------------------------------------"; + qDebug(); + Sink::ResourceControl::start(QByteArray("org.kde.imap.instance1")).exec().waitForFinished(); + } + + void testListFolders() + { + Sink::Query query; + query.resources << "org.kde.imap.instance1"; + query.request(); + + // Ensure all local data is processed + VERIFYEXEC(Store::synchronize(query)); + ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished(); + + auto job = Store::fetchAll(query).then>([](const QList &folders) { + QCOMPARE(folders.size(), 2); + QStringList names; + for (const auto &folder : folders) { + names << folder->getName(); + } + QVERIFY(names.contains("INBOX")); + QVERIFY(names.contains("INBOX.test")); + }); + VERIFYEXEC(job); + } + + void testListMails() + { + Sink::Query query; + query.resources << "org.kde.imap.instance1"; + query.request().request(); + + // Ensure all local data is processed + VERIFYEXEC(Store::synchronize(query)); + ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished(); + + auto job = Store::fetchAll(query).then>([](const QList &mails) { + QCOMPARE(mails.size(), 1); + QVERIFY(mails.first()->getSubject().startsWith(QString("[Nepomuk] Jenkins build is still unstable"))); + const auto data = mails.first()->getMimeMessage(); + QVERIFY(!data.isEmpty()); + + KMime::Message m; + m.setContent(data); + m.parse(); + QCOMPARE(mails.first()->getSubject(), m.subject(true)->asUnicodeString()); + }); + VERIFYEXEC(job); + } + + void testFetchNewMessages() + { + Sink::Query query; + query.resources << "org.kde.imap.instance1"; + query.request().request(); + + // Ensure all local data is processed + VERIFYEXEC(Store::synchronize(query)); + ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished(); + + ImapServerProxy imap("localhost", 993); + imap.login("doe", "doe").exec().waitForFinished(); + + auto msg = KMime::Message::Ptr::create(); + msg->subject(true)->fromUnicodeString("Foobar", "utf8"); + msg->assemble(); + + VERIFYEXEC(imap.append("INBOX.test", msg->encodedContent(true))); + + Store::synchronize(query).exec().waitForFinished(); + ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished(); + + auto job = Store::fetchAll(query).then>([](const QList &mails) { + QCOMPARE(mails.size(), 2); + }); + VERIFYEXEC(job); + } +}; + +QTEST_MAIN(ImapResourceTest) +#include "imapresourcetest.moc" diff --git a/examples/imapresource/tests/resetmailbox.sh b/examples/imapresource/tests/resetmailbox.sh new file mode 100644 index 0000000..966115a --- /dev/null +++ b/examples/imapresource/tests/resetmailbox.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +sudo echo "dm user.doe.test" | cyradm --auth PLAIN -u cyrus -w admin localhost +sudo echo "cm user.doe.test" | cyradm --auth PLAIN -u cyrus -w admin localhost +sudo echo "sam user.doe cyrus c" | cyradm --auth PLAIN -u cyrus -w admin localhost +sudo echo "sam user.doe.test cyrus c" | cyradm --auth PLAIN -u cyrus -w admin localhost +# sudo rm -R /var/spool/imap/d/user/doe/* +sudo cp /work/source/Sink/tests/data/maildir1/cur/1365777830.R28.localhost.localdomain\:2\,S /var/spool/imap/d/user/doe/test/1. +sudo chown cyrus:mail /var/spool/imap/d/user/doe/test/1. +sudo /usr/lib/cyrus-imapd/reconstruct "user.doe.test" -- cgit v1.2.3