From a853cb1a32ae9207501fc4378357107013ebce21 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 23 May 2016 13:55:36 +0200 Subject: Message removals --- examples/imapresource/imapresource.cpp | 108 +++++++++++++++------ examples/imapresource/imapresource.h | 3 + examples/imapresource/imapserverproxy.cpp | 76 ++++++++++----- examples/imapresource/imapserverproxy.h | 22 +++++ examples/imapresource/tests/imapresourcetest.cpp | 30 +++++- .../imapresource/tests/imapserverproxytest.cpp | 50 ++++++++++ 6 files changed, 232 insertions(+), 57 deletions(-) (limited to 'examples') diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 49cbb20..6adfeb3 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -59,6 +59,8 @@ ImapResource::ImapResource(const QByteArray &instanceIdentifier, const QSharedPo auto config = ResourceConfig::getConfiguration(instanceIdentifier); mServer = config.value("server").toString(); mPort = config.value("port").toInt(); + mUser = config.value("user").toString(); + mPassword = config.value("password").toString(); // auto folderUpdater = new FolderUpdater(QByteArray()); addType(ENTITY_TYPE_MAIL, mMailAdaptorFactory, @@ -117,6 +119,15 @@ void ImapResource::synchronizeFolders(const QVector &folderList, Sink::S } } +static QByteArray remoteIdForMessage(const QString &path, qint64 uid) +{ + return path.toUtf8() + "/" + QByteArray::number(uid); +} + +static qint64 uidFromMessageRemoteId(const QByteArray &remoteId) +{ + return remoteId.split('/').last().toLongLong(); +} void ImapResource::synchronizeMails(Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction, const QString &path, const QVector &messages) { auto time = QSharedPointer::create(); @@ -130,23 +141,6 @@ void ImapResource::synchronizeMails(Sink::Storage::Transaction &transaction, Sin const auto folderLocalId = resolveRemoteId(ENTITY_TYPE_FOLDER, path.toUtf8(), synchronizationTransaction); - //This is not a full listing - // auto property = "folder"; - // scanForRemovals(transaction, synchronizationTransaction, bufferType, - // [&](const std::function &callback) { - // Index index(bufferType + ".index." + property, transaction); - // index.lookup(folderLocalId, [&](const QByteArray &sinkId) { - // callback(sinkId); - // }, - // [&](const Index::Error &error) { - // Warning() << "Error in index: " << error.message << property; - // }); - // }, - // [](const QByteArray &remoteId) -> bool { - // return QFile(remoteId).exists(); - // } - // ); - mSynchronizerQueue.startTransaction(); int count = 0; for (const auto &message : messages) { @@ -170,8 +164,8 @@ void ImapResource::synchronizeMails(Sink::Storage::Transaction &transaction, Sin file.write(content); mail.setMimeMessagePath(filePath); //FIXME Not sure if these are the actual flags - mail.setUnread(message.flags.contains("\\SEEN")); - mail.setImportant(message.flags.contains("\\FLAGGED")); + mail.setUnread(!message.flags.contains(Imap::Flags::Seen)); + mail.setImportant(message.flags.contains(Imap::Flags::Flagged)); createOrModify(transaction, synchronizationTransaction, *mMailAdaptorFactory, bufferType, remoteId, mail); } @@ -180,11 +174,56 @@ void ImapResource::synchronizeMails(Sink::Storage::Transaction &transaction, Sin Log() << "Synchronized " << count << " mails in " << path << Sink::Log::TraceTime(elapsed) << " " << elapsed/qMax(count, 1) << " [ms/mail]"; } +void ImapResource::synchronizeRemovals(Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction, const QString &path, const QSet &messages) +{ + auto time = QSharedPointer::create(); + time->start(); + const QByteArray bufferType = ENTITY_TYPE_MAIL; + + Trace() << "Finding removed mail."; + + const auto folderLocalId = resolveRemoteId(ENTITY_TYPE_FOLDER, path.toUtf8(), synchronizationTransaction); + + int count = 0; + auto property = Sink::ApplicationDomain::Mail::Folder::name; + scanForRemovals(transaction, synchronizationTransaction, bufferType, + [&](const std::function &callback) { + Index index(bufferType + ".index." + property, transaction); + index.lookup(folderLocalId, [&](const QByteArray &sinkId) { + callback(sinkId); + }, + [&](const Index::Error &error) { + Warning() << "Error in index: " << error.message << property; + }); + }, + [messages, path, &count](const QByteArray &remoteId) -> bool { + if (messages.contains(uidFromMessageRemoteId(remoteId))) { + return true; + } + count++; + return false; + } + ); + + const auto elapsed = time->elapsed(); + Log() << "Removed " << count << " mails in " << path << Sink::Log::TraceTime(elapsed) << " " << elapsed/qMax(count, 1) << " [ms/mail]"; +} + KAsync::Job ImapResource::synchronizeWithSource(Sink::Storage &mainStore, Sink::Storage &synchronizationStore) { Log() << " Synchronizing"; return KAsync::start([this, &mainStore, &synchronizationStore](KAsync::Future future) { ImapServerProxy imap(mServer, mPort); + auto loginFuture = imap.login(mUser, mPassword).exec(); + loginFuture.waitForFinished(); + if (loginFuture.errorCode()) { + Warning() << "Login failed."; + future.setError(1, "Login failed"); + return; + } else { + Trace() << "Login was successful"; + } + QVector folderList; auto folderFuture = imap.fetchFolders([this, &imap, &mainStore, &synchronizationStore, &folderList](const QVector &folders) { auto transaction = mainStore.createTransaction(Sink::Storage::ReadOnly); @@ -197,8 +236,11 @@ KAsync::Job ImapResource::synchronizeWithSource(Sink::Storage &mainStore, }); folderFuture.waitForFinished(); if (folderFuture.errorCode()) { + Warning() << "Folder sync failed."; future.setError(1, "Folder list sync failed"); return; + } else { + Trace() << "Folder sync was successful"; } for (const auto &folder : folderList) { @@ -220,30 +262,32 @@ KAsync::Job ImapResource::synchronizeWithSource(Sink::Storage &mainStore, // transaction.commit(); // syncTransaction.commit(); - auto messagesFuture = imap.fetchMessages(folder, [this, &mainStore, &synchronizationStore, folder](const QVector &messages) { + QSet uids; + auto messagesFuture = imap.fetchMessages(folder, [this, &mainStore, &synchronizationStore, folder, &uids](const QVector &messages) { auto transaction = mainStore.createTransaction(Sink::Storage::ReadOnly); auto syncTransaction = synchronizationStore.createTransaction(Sink::Storage::ReadWrite); - Trace() << "Synchronizing mails" << folder.pathParts.join('/'); - synchronizeMails(transaction, syncTransaction, folder.pathParts.join('/'), messages); + Trace() << "Synchronizing mails" << folder.normalizedPath(); + for (const auto &msg : messages) { + uids << msg.uid; + } + synchronizeMails(transaction, syncTransaction, folder.normalizedPath(), messages); transaction.commit(); syncTransaction.commit(); }); messagesFuture.waitForFinished(); if (messagesFuture.errorCode()) { - future.setError(1, "Folder sync failed: " + folder.pathParts.join('/')); + future.setError(1, "Folder sync failed: " + folder.normalizedPath()); return; } - Trace() << "Folder synchronized: " << folder.pathParts.join('/'); + //Remove what there is to remove + auto transaction = mainStore.createTransaction(Sink::Storage::ReadOnly); + auto syncTransaction = synchronizationStore.createTransaction(Sink::Storage::ReadWrite); + synchronizeRemovals(transaction, syncTransaction, folder.normalizedPath(), uids); + transaction.commit(); + syncTransaction.commit(); + Trace() << "Folder synchronized: " << folder.normalizedPath(); } - - // auto transaction = mainStore.createTransaction(Sink::Storage::ReadWrite); - // auto mainDatabase = Sink::Storage::mainDatabase(transaction, ENTITY_TYPE_FOLDER); - // mainDatabase.scan("", [&](const QByteArray &key, const QByteArray &data) { - // return true; - // }); - //TODO now fetch all folders and iterate over them and synchronize each one - Log() << "Done Synchronizing"; future.setFinished(); }); diff --git a/examples/imapresource/imapresource.h b/examples/imapresource/imapresource.h index 23b7e1a..82f96a4 100644 --- a/examples/imapresource/imapresource.h +++ b/examples/imapresource/imapresource.h @@ -52,12 +52,15 @@ private: QByteArray createFolder(const QString &folderPath, const QByteArray &icon, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction); void synchronizeFolders(const QVector &folderList, Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction); void synchronizeMails(Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction, const QString &path, const QVector &messages); + void synchronizeRemovals(Sink::Storage::Transaction &transaction, Sink::Storage::Transaction &synchronizationTransaction, const QString &path, const QSet &messages); QSharedPointer mMailAdaptorFactory; QSharedPointer mFolderAdaptorFactory; private: QString mServer; int mPort; + QString mUser; + QString mPassword; }; class ImapResourceFactory : public Sink::ResourceFactory diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp index 836a9bc..dfb134c 100644 --- a/examples/imapresource/imapserverproxy.cpp +++ b/examples/imapresource/imapserverproxy.cpp @@ -24,6 +24,9 @@ #include #include #include +#include +#include +#include #include #include @@ -32,10 +35,15 @@ using namespace Imap; +const char* Imap::Flags::Seen = "\\Seen"; +const char* Imap::Flags::Deleted = "\\Deleted"; +const char* Imap::Flags::Answered = "\\Answered"; +const char* Imap::Flags::Flagged = "\\Flagged"; + static KAsync::Job runJob(KJob *job) { return KAsync::start([job](KAsync::Future &future) { - QObject::connect(job, &KJob::result, job, [&future](KJob *job) { + QObject::connect(job, &KJob::result, [&future](KJob *job) { if (job->error()) { Warning() << "Job failed: " << job->errorString(); future.setError(job->error(), job->errorString()); @@ -62,9 +70,6 @@ ImapServerProxy::ImapServerProxy(const QString &serverUrl, int port) : mSession( KAsync::Job ImapServerProxy::login(const QString &username, const QString &password) { - if (mSession->state() == KIMAP::Session::State::Authenticated || mSession->state() == KIMAP::Session::State::Selected) { - return KAsync::null(); - } auto loginJob = new KIMAP::LoginJob(mSession); loginJob->setUserName(username); loginJob->setPassword(password); @@ -75,9 +80,6 @@ KAsync::Job ImapServerProxy::login(const QString &username, const QString KAsync::Job ImapServerProxy::select(const QString &mailbox) { - if (mSession->state() == KIMAP::Session::State::Disconnected) { - return KAsync::error(1, "Not connected"); - } auto select = new KIMAP::SelectJob(mSession); select->setMailBox(mailbox); // select->setCondstoreEnabled(serverSupportsCondstore()); @@ -86,9 +88,6 @@ KAsync::Job ImapServerProxy::select(const QString &mailbox) KAsync::Job ImapServerProxy::append(const QString &mailbox, const QByteArray &content, const QList &flags, const QDateTime &internalDate) { - if (mSession->state() == KIMAP::Session::State::Disconnected) { - return KAsync::error(1, "Not connected"); - } auto append = new KIMAP::AppendJob(mSession); append->setMailBox(mailbox); append->setContent(content); @@ -97,21 +96,38 @@ KAsync::Job ImapServerProxy::append(const QString &mailbox, const QByteArr return runJob(append); } +KAsync::Job ImapServerProxy::store(const KIMAP::ImapSet &set, const QList &flags) +{ + auto store = new KIMAP::StoreJob(mSession); + store->setUidBased(true); + store->setSequenceSet(set); + store->setFlags(flags); + store->setMode(KIMAP::StoreJob::AppendFlags); + return runJob(store); +} + KAsync::Job ImapServerProxy::create(const QString &mailbox) { - if (mSession->state() == KIMAP::Session::State::Disconnected) { - return KAsync::error(1, "Not connected"); - } auto create = new KIMAP::CreateJob(mSession); create->setMailBox(mailbox); return runJob(create); } +KAsync::Job ImapServerProxy::remove(const QString &mailbox) +{ + auto job = new KIMAP::DeleteJob(mSession); + job->setMailBox(mailbox); + return runJob(job); +} + +KAsync::Job ImapServerProxy::expunge() +{ + auto job = new KIMAP::ExpungeJob(mSession); + return runJob(job); +} + KAsync::Job ImapServerProxy::fetch(const KIMAP::ImapSet &set, KIMAP::FetchJob::FetchScope scope, FetchCallback callback) { - if (mSession->state() == KIMAP::Session::State::Disconnected) { - return KAsync::error(1, "Not connected"); - } auto fetch = new KIMAP::FetchJob(mSession); fetch->setSequenceSet(set); fetch->setUidBased(true); @@ -164,34 +180,48 @@ KAsync::Job ImapServerProxy::list(KIMAP::ListJob::Option option, const std // listJob->setQueriedNamespaces(serverNamespaces()); QObject::connect(listJob, &KIMAP::ListJob::mailBoxesReceived, listJob, callback); + //Figure out the separator character on the first list issued. + if (mSeparatorCharacter.isNull()) { + QObject::connect(listJob, &KIMAP::ListJob::mailBoxesReceived, + listJob, [this](const QList &mailboxes,const QList > &flags) { + if (!mailboxes.isEmpty() && mSeparatorCharacter.isNull()) { + mSeparatorCharacter = mailboxes.first().separator; + } + } + ); + } return runJob(listJob); } +KAsync::Job ImapServerProxy::remove(const QString &mailbox, const QByteArray &imapSet) +{ + const auto set = KIMAP::ImapSet::fromImapSequenceSet(imapSet); + return select(mailbox).then(store(set, QByteArrayList() << Flags::Deleted)).then(expunge()); +} + KAsync::Future ImapServerProxy::fetchFolders(std::function &)> callback) { Trace() << "Fetching folders"; - auto job = login("doe", "doe").then(list(KIMAP::ListJob::IncludeUnsubscribed, [callback](const QList &mailboxes, const QList > &flags){ + auto job = list(KIMAP::ListJob::IncludeUnsubscribed, [callback](const QList &mailboxes, const QList > &flags){ QVector list; for (const auto &mailbox : mailboxes) { Trace() << "Found mailbox: " << mailbox.name; list << Folder{mailbox.name.split(mailbox.separator)}; } callback(list); - }), - [](int errorCode, const QString &errorString) { - Warning() << "Failed to list folders: " << errorCode << errorString; }); return job.exec(); } KAsync::Future ImapServerProxy::fetchMessages(const Folder &folder, std::function &)> callback) { - //TODO use the right separator - auto job = login("doe", "doe").then(select(folder.pathParts.join('.'))).then>([this, callback, folder]() -> KAsync::Job { - return fetchHeaders(folder.pathParts.join('.')).then, QList>([this, callback](const QList &uidsToFetch){ + Q_ASSERT(!mSeparatorCharacter.isNull()); + auto job = select(folder.pathParts.join(mSeparatorCharacter)).then>([this, callback, folder]() -> KAsync::Job { + return fetchHeaders(folder.pathParts.join(mSeparatorCharacter)).then, QList>([this, callback](const QList &uidsToFetch){ Trace() << "Uids to fetch: " << uidsToFetch; if (uidsToFetch.isEmpty()) { Trace() << "Nothing to fetch"; + callback(QVector()); return KAsync::null(); } KIMAP::FetchJob::FetchScope scope; diff --git a/examples/imapresource/imapserverproxy.h b/examples/imapresource/imapserverproxy.h index 475a45d..7000c67 100644 --- a/examples/imapresource/imapserverproxy.h +++ b/examples/imapresource/imapserverproxy.h @@ -28,6 +28,18 @@ namespace Imap { +namespace Flags +{ + /// The flag for a message being seen (i.e. opened by user). + extern const char* Seen; + /// The flag for a message being deleted by the user. + extern const char* Deleted; + /// The flag for a message being replied to by the user. + extern const char* Answered; + /// The flag for a message being marked as flagged. + extern const char* Flagged; +} + struct Message { qint64 uid; qint64 size; @@ -37,11 +49,17 @@ struct Message { }; struct Folder { + QString normalizedPath() const + { + return pathParts.join('/'); + } + QList pathParts; }; class ImapServerProxy { KIMAP::Session *mSession; + QChar mSeparatorCharacter; public: ImapServerProxy(const QString &serverUrl, int port); @@ -49,7 +67,10 @@ public: KAsync::Job login(const QString &username, const QString &password); KAsync::Job select(const QString &mailbox); KAsync::Job append(const QString &mailbox, const QByteArray &content, const QList &flags = QList(), const QDateTime &internalDate = QDateTime()); + KAsync::Job store(const KIMAP::ImapSet &set, const QList &flags); KAsync::Job create(const QString &mailbox); + KAsync::Job remove(const QString &mailbox); + KAsync::Job expunge(); typedef std::function &, @@ -63,6 +84,7 @@ public: //Composed calls that do login etc. KAsync::Job> fetchHeaders(const QString &mailbox); + KAsync::Job remove(const QString &mailbox, const QByteArray &imapSet); KAsync::Future fetchFolders(std::function &)> callback); KAsync::Future fetchMessages(const Folder &folder, std::function &)> callback); diff --git a/examples/imapresource/tests/imapresourcetest.cpp b/examples/imapresource/tests/imapresourcetest.cpp index 8ce32c2..30bb953 100644 --- a/examples/imapresource/tests/imapresourcetest.cpp +++ b/examples/imapresource/tests/imapresourcetest.cpp @@ -65,6 +65,8 @@ private slots: resource.setProperty("identifier", "org.kde.imap.instance1"); resource.setProperty("type", "org.kde.imap"); resource.setProperty("server", "localhost"); + resource.setProperty("user", "doe"); + resource.setProperty("password", "doe"); resource.setProperty("port", 993); Sink::Store::create(resource).exec().waitForFinished(); } @@ -164,8 +166,8 @@ private slots: VERIFYEXEC(Store::synchronize(query)); ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished(); - ImapServerProxy imap("localhost", 993); - imap.login("doe", "doe").exec().waitForFinished(); + Imap::ImapServerProxy imap("localhost", 993); + VERIFYEXEC(imap.login("doe", "doe")); auto msg = KMime::Message::Ptr::create(); msg->subject(true)->fromUnicodeString("Foobar", "utf8"); @@ -182,6 +184,30 @@ private slots: VERIFYEXEC(job); } + void testFetchRemovedMessages() + { + Sink::Query query; + query.resources << "org.kde.imap.instance1"; + query.request().request(); + + // Ensure all local data is processed + VERIFYEXEC(Store::synchronize(query)); + ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished(); + + Imap::ImapServerProxy imap("localhost", 993); + VERIFYEXEC(imap.login("doe", "doe")); + + VERIFYEXEC(imap.remove("INBOX.test", "2:*")); + + Store::synchronize(query).exec().waitForFinished(); + ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished(); + + auto job = Store::fetchAll(query).then>([](const QList &mails) { + QCOMPARE(mails.size(), 1); + }); + VERIFYEXEC(job); + } + void testFailingSync() { auto resource = ApplicationDomain::ImapResource::create("account1"); diff --git a/examples/imapresource/tests/imapserverproxytest.cpp b/examples/imapresource/tests/imapserverproxytest.cpp index 139597a..b30cc33 100644 --- a/examples/imapresource/tests/imapserverproxytest.cpp +++ b/examples/imapresource/tests/imapserverproxytest.cpp @@ -79,6 +79,7 @@ private slots: void testFetchFolders() { ImapServerProxy imap("localhost", 993); + VERIFYEXEC(imap.login("doe", "doe")); auto future = imap.fetchFolders([](const QVector &){}); future.waitForFinished(); QVERIFY(!future.errorCode()); @@ -93,6 +94,55 @@ private slots: QVERIFY(future2.errorCode()); } + void testFetchMail() + { + ImapServerProxy imap("localhost", 993); + VERIFYEXEC(imap.login("doe", "doe")); + + KIMAP::FetchJob::FetchScope scope; + scope.mode = KIMAP::FetchJob::FetchScope::Headers; + int count = 0; + auto job = imap.select("INBOX.test").then(imap.fetch(KIMAP::ImapSet::fromImapSequenceSet("1:*"), scope, + [&count](const QString &mailbox, + const QMap &uids, + const QMap &sizes, + const QMap &attrs, + const QMap &flags, + const QMap &messages) { + Trace() << "Received " << uids.size() << " messages from " << mailbox; + Trace() << uids.size() << sizes.size() << attrs.size() << flags.size() << messages.size(); + count += uids.size(); + })); + + VERIFYEXEC(job); + QCOMPARE(count, 1); + } + + void testRemoveMail() + { + ImapServerProxy imap("localhost", 993); + VERIFYEXEC(imap.login("doe", "doe")); + VERIFYEXEC(imap.remove("INBOX.test", "1:*")); + + KIMAP::FetchJob::FetchScope scope; + scope.mode = KIMAP::FetchJob::FetchScope::Headers; + int count = 0; + auto job = imap.select("INBOX.test").then(imap.fetch(KIMAP::ImapSet::fromImapSequenceSet("1:*"), scope, + [&count](const QString &mailbox, + const QMap &uids, + const QMap &sizes, + const QMap &attrs, + const QMap &flags, + const QMap &messages) { + Trace() << "Received " << uids.size() << " messages from " << mailbox; + Trace() << uids.size() << sizes.size() << attrs.size() << flags.size() << messages.size(); + count += uids.size(); + })); + + VERIFYEXEC(job); + QCOMPARE(count, 0); + } + }; QTEST_MAIN(ImapServerProxyTest) -- cgit v1.2.3