From 6d5be4fb7b8cbc450e2780905eaac9a18b486c5c Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 14 Sep 2016 16:16:42 +0200 Subject: New synchronization algorithm that only fetches the last 14 days. --- examples/imapresource/imapresource.cpp | 129 ++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 58 deletions(-) (limited to 'examples/imapresource/imapresource.cpp') diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 63ae07b..aa0fb94 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -40,10 +40,14 @@ #include "entitystore.h" #include "remoteidmap.h" #include "query.h" + +#include #include #include #include #include +#include +#include #include "imapserverproxy.h" #include "entityreader.h" @@ -149,7 +153,7 @@ public: } } - void synchronizeMails(const QString &path, const QVector &messages) + void synchronizeMails(const QString &path, const Message &message) { auto time = QSharedPointer::create(); time->start(); @@ -159,23 +163,20 @@ public: const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, path.toUtf8()); - int count = 0; - for (const auto &message : messages) { - count++; - const auto remoteId = assembleMailRid(folderLocalId, message.uid); + const auto remoteId = assembleMailRid(folderLocalId, message.uid); - SinkTrace() << "Found a mail " << remoteId << message.msg->subject(true)->asUnicodeString() << message.flags; + Q_ASSERT(message.msg); + SinkTrace() << "Found a mail " << remoteId << message.msg->subject(true)->asUnicodeString() << message.flags; - auto mail = Sink::ApplicationDomain::Mail::create(mResourceInstanceIdentifier); - mail.setFolder(folderLocalId); - mail.setMimeMessage(message.msg->encodedContent()); - mail.setUnread(!message.flags.contains(Imap::Flags::Seen)); - mail.setImportant(message.flags.contains(Imap::Flags::Flagged)); + auto mail = Sink::ApplicationDomain::Mail::create(mResourceInstanceIdentifier); + mail.setFolder(folderLocalId); + mail.setMimeMessage(message.msg->encodedContent()); + mail.setUnread(!message.flags.contains(Imap::Flags::Seen)); + mail.setImportant(message.flags.contains(Imap::Flags::Flagged)); - createOrModify(bufferType, remoteId, mail); - } - const auto elapsed = time->elapsed(); - SinkTrace() << "Synchronized " << count << " mails in " << path << Sink::Log::TraceTime(elapsed) << " " << elapsed/qMax(count, 1) << " [ms/mail]"; + createOrModify(bufferType, remoteId, mail); + // const auto elapsed = time->elapsed(); + // SinkTrace() << "Synchronized " << count << " mails in " << path << Sink::Log::TraceTime(elapsed) << " " << elapsed/qMax(count, 1) << " [ms/mail]"; } void synchronizeRemovals(const QString &path, const QSet &messages) @@ -220,29 +221,71 @@ public: auto capabilities = imap->getCapabilities(); bool canDoIncrementalRemovals = false; return KAsync::start([=]() { + //First we fetch flag changes for all messages. Since we don't know which messages are locally available we just get everything and only apply to what we have. + SinkLog() << "About to update flags" << folder.normalizedPath(); auto uidNext = syncStore().readValue(folder.normalizedPath().toUtf8() + "uidnext").toLongLong(); const auto changedsince = syncStore().readValue(folder.normalizedPath().toUtf8() + "changedsince").toLongLong(); - return imap->fetchFlags(folder, KIMAP2::ImapSet(1, qMax(uidNext, qint64(1))), changedsince, [this, folder](const QVector &messages) { - // synchronizeMails(folder.normalizedPath(), messages); + return imap->fetchFlags(folder, KIMAP2::ImapSet(1, qMax(uidNext, qint64(1))), changedsince, [this, folder](const Message &message) { const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folder.normalizedPath().toUtf8()); - for (const auto &message : messages) { - const auto remoteId = assembleMailRid(folderLocalId, message.uid); + const auto remoteId = assembleMailRid(folderLocalId, message.uid); + SinkLog() << "Updating mail flags " << remoteId << message.flags; - auto mail = Sink::ApplicationDomain::Mail::create(mResourceInstanceIdentifier); - mail.setUnread(!message.flags.contains(Imap::Flags::Seen)); - mail.setImportant(message.flags.contains(Imap::Flags::Flagged)); + auto mail = Sink::ApplicationDomain::Mail::create(mResourceInstanceIdentifier); + mail.setUnread(!message.flags.contains(Imap::Flags::Seen)); + mail.setImportant(message.flags.contains(Imap::Flags::Flagged)); - modify(ENTITY_TYPE_MAIL, remoteId, mail); - } + modify(ENTITY_TYPE_MAIL, remoteId, mail); }) .syncThen([this, folder](const SelectResult &selectResult) { syncStore().writeValue(folder.normalizedPath().toUtf8() + "changedsince", QByteArray::number(selectResult.highestModSequence)); }); }) + .then([=]() { + //Get the range we're interested in. This is what we're going to download. + return imap->fetchUidsSince(imap->mailboxFromFolder(folder), QDate::currentDate().addDays(-14)) + .then>([this, folder, imap](const QVector &uidsToFetch) { + SinkTrace() << "Received result set " << uidsToFetch; + SinkTrace() << "About to fetch mail" << folder.normalizedPath(); + const auto uidNext = syncStore().readValue(folder.normalizedPath().toUtf8() + "uidnext").toLongLong(); + QVector filteredAndSorted = uidsToFetch; + qSort(filteredAndSorted.begin(), filteredAndSorted.end(), qGreater()); + auto lowerBound = qLowerBound(filteredAndSorted.begin(), filteredAndSorted.end(), uidNext, qGreater()); + if (lowerBound != filteredAndSorted.end()) { + filteredAndSorted.erase(lowerBound, filteredAndSorted.end()); + } + + auto maxUid = QSharedPointer::create(0); + if (!filteredAndSorted.isEmpty()) { + *maxUid = filteredAndSorted.first(); + } + SinkTrace() << "Uids to fetch: " << filteredAndSorted; + return imap->fetchMessages(folder, filteredAndSorted, [this, folder, maxUid](const Message &m) { + if (*maxUid < m.uid) { + *maxUid = m.uid; + } + synchronizeMails(folder.normalizedPath(), m); + }, + [this, maxUid, folder](int progress, int total) { + SinkLog() << "Progress: " << progress << " out of " << total; + //commit every 10 messages + if ((progress % 10) == 0) { + commit(); + } + }) + .syncThen([this, maxUid, folder]() { + SinkLog() << "UIDMAX: " << *maxUid << folder.normalizedPath(); + if (*maxUid > 0) { + syncStore().writeValue(folder.normalizedPath().toUtf8() + "uidnext", QByteArray::number(*maxUid)); + } + commit(); + }); + }); + }) .then([=]() { //TODO Remove what's no longer existing if (canDoIncrementalRemovals) { + //TODO do an examine with QRESYNC and remove VANISHED messages } else { return imap->fetchUids(folder).syncThen>([this, folder](const QVector &uids) { SinkTrace() << "Syncing removals"; @@ -251,32 +294,6 @@ public: }); } return KAsync::null(); - }) - .then([this, folder, imap]() { - SinkTrace() << "About to fetch mail"; - const auto uidNext = syncStore().readValue(folder.normalizedPath().toUtf8() + "uidnext").toLongLong(); - auto maxUid = QSharedPointer::create(0); - return imap->fetchMessages(folder, uidNext, [this, folder, maxUid](const QVector &messages) { - SinkTrace() << "Got mail"; - for (const auto &m : messages) { - if (*maxUid < m.uid) { - *maxUid = m.uid; - } - } - synchronizeMails(folder.normalizedPath(), messages); - }, - [this, maxUid, folder](int progress, int total) { - SinkLog() << "Progress: " << progress << " out of " << total; - //commit every 10 messages - if ((progress % 10) == 0) { - commit(); - } - }) - .syncThen([this, maxUid, folder]() { - syncStore().writeValue(folder.normalizedPath().toUtf8() + "uidnext", QByteArray::number(*maxUid)); - SinkLog() << "UIDMAX: " << *maxUid << folder.normalizedPath(); - commit(); - }); }); @@ -578,10 +595,8 @@ KAsync::Job ImapResource::inspect(int inspectionType, const QByteArray &in auto inspectionJob = imap->login(mUser, mPassword) .then(imap->select(folderRemoteId)) .syncThen([](Imap::SelectResult){}) - .then(imap->fetch(set, scope, [imap, messageByUid](const QVector &messages) { - for (const auto &m : messages) { - messageByUid->insert(m.uid, m); - } + .then(imap->fetch(set, scope, [imap, messageByUid](const Imap::Message &message) { + messageByUid->insert(message.uid, message); })); if (inspectionType == Sink::ResourceControl::Inspection::PropertyInspectionType) { @@ -641,10 +656,8 @@ KAsync::Job ImapResource::inspect(int inspectionType, const QByteArray &in auto messageByUid = QSharedPointer>::create(); return imap->login(mUser, mPassword) .then(imap->select(remoteId).syncThen([](){})) - .then(imap->fetch(set, scope, [=](const QVector &messages) { - for (const auto &m : messages) { - messageByUid->insert(m.uid, m); - } + .then(imap->fetch(set, scope, [=](const Imap::Message message) { + messageByUid->insert(message.uid, message); })) .then([imap, messageByUid, expectedCount]() { if (messageByUid->size() != expectedCount) { -- cgit v1.2.3