Generalized the sync algorithms and applied them to mail.

Not necessarily the smartest algorithms, but at least they work and
are generally applicable.

3 files changed, 155 insertions, 86 deletions

diff --git a/examples/maildirresource/maildirresource.cpp b/examples/maildirresource/maildirresource.cpp
index 10e9046..90b4e55 100644
--- a/examples/maildirresource/maildirresource.cpp
+++ b/examples/maildirresource/maildirresource.cpp
@@ -114,7 +114,7 @@ QStringList MaildirResource::listAvailableFolders()
     return folderList;
 }
 
-static void createEntity(const QByteArray &akonadiId, const QByteArray &bufferType, Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback)
+static void createEntity(const QByteArray &akonadiId, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback)
 {
     flatbuffers::FlatBufferBuilder entityFbb;
     adaptorFactory.createBuffer(domainObject, entityFbb);
@@ -128,7 +128,7 @@ static void createEntity(const QByteArray &akonadiId, const QByteArray &bufferTy
     callback(QByteArray::fromRawData(reinterpret_cast<char const *>(fbb.GetBufferPointer()), fbb.GetSize()));
 }
 
-static void modifyEntity(const QByteArray &akonadiId, qint64 revision, const QByteArray &bufferType, Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback)
+static void modifyEntity(const QByteArray &akonadiId, qint64 revision, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback)
 {
     flatbuffers::FlatBufferBuilder entityFbb;
     adaptorFactory.createBuffer(domainObject, entityFbb);
@@ -172,40 +172,79 @@ static QSharedPointer<Akonadi2::ApplicationDomain::BufferAdaptor> getLatest(cons
     return current;
 }
 
-void MaildirResource::synchronizeFolders(Akonadi2::Storage::Transaction &transaction)
+void MaildirResource::scanForRemovals(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, const QByteArray &bufferType, std::function<bool(const QByteArray &remoteId)> exists)
 {
-    const QByteArray bufferType = ENTITY_TYPE_FOLDER;
-    QStringList folderList = listAvailableFolders();
-    Trace() << "Found folders " << folderList;
-
-    Akonadi2::Storage store(Akonadi2::storageLocation(), mResourceInstanceIdentifier + ".synchronization", Akonadi2::Storage::ReadWrite);
-    auto synchronizationTransaction = store.createTransaction(Akonadi2::Storage::ReadWrite);
     auto mainDatabase = transaction.openDatabase(bufferType + ".main");
-
     //TODO Instead of iterating over all entries in the database, which can also pick up the same item multiple times,
     //we should rather iterate over an index that contains every uid exactly once. The remoteId index would be such an index,
     //but we currently fail to iterate over all entries in an index it seems.
     // auto remoteIds = synchronizationTransaction.openDatabase("rid.mapping." + bufferType, std::function<void(const Akonadi2::Storage::Error &)>(), true);
-    mainDatabase.scan("", [&folderList, this, &transaction, bufferType, &synchronizationTransaction](const QByteArray &key, const QByteArray &) {
+    mainDatabase.scan("", [this, &transaction, bufferType, &synchronizationTransaction, &exists](const QByteArray &key, const QByteArray &) {
         auto akonadiId = Akonadi2::Storage::uidFromKey(key);
+        Trace() << "Checking for removal " << key;
         const auto remoteId = resolveLocalId(bufferType, akonadiId, synchronizationTransaction);
-        if (!folderList.contains(remoteId)) {
-            Trace() << "Found a removed entity: " << akonadiId;
-            deleteEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, [this](const QByteArray &buffer) {
-                enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::DeleteEntityCommand, buffer);
-            });
+        if (!remoteId.isEmpty()) {
+            if (!exists(remoteId.toLatin1())) {
+                Trace() << "Found a removed entity: " << akonadiId;
+                deleteEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, [this](const QByteArray &buffer) {
+                    enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::DeleteEntityCommand, buffer);
+                });
+            }
         }
         return true;
     },
     [](const Akonadi2::Storage::Error &error) {
     });
 
+}
+
+void MaildirResource::createOrModify(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, DomainTypeAdaptorFactoryInterface &adaptorFactory, const QByteArray &bufferType, const QByteArray &remoteId, const Akonadi2::ApplicationDomain::ApplicationDomainType &entity)
+{
+    auto mainDatabase = transaction.openDatabase(bufferType + ".main");
+    const auto akonadiId = resolveRemoteId(bufferType, remoteId, synchronizationTransaction).toLatin1();
+    const auto found = mainDatabase.contains(akonadiId);
+    if (!found) {
+        Trace() << "Found a new entity: " << remoteId;
+        createEntity(akonadiId, bufferType, entity, adaptorFactory, [this](const QByteArray &buffer) {
+            enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::CreateEntityCommand, buffer);
+        });
+    } else { //modification
+        if (auto current = getLatest(mainDatabase, akonadiId, adaptorFactory)) {
+            bool changed = false;
+            for (const auto &property : entity.changedProperties()) {
+                if (entity.getProperty(property) != current->getProperty(property)) {
+                    Trace() << "Property changed " << akonadiId << property;
+                    changed = true;
+                }
+            }
+            if (changed) {
+                Trace() << "Found a modified entity: " << remoteId;
+                modifyEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, entity, adaptorFactory, [this](const QByteArray &buffer) {
+                    enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::ModifyEntityCommand, buffer);
+                });
+            }
+        } else {
+            Warning() << "Failed to get current entity";
+        }
+    }
+}
+
+void MaildirResource::synchronizeFolders(Akonadi2::Storage::Transaction &transaction)
+{
+    const QByteArray bufferType = ENTITY_TYPE_FOLDER;
+    QStringList folderList = listAvailableFolders();
+    Trace() << "Found folders " << folderList;
+
+    Akonadi2::Storage store(Akonadi2::storageLocation(), mResourceInstanceIdentifier + ".synchronization", Akonadi2::Storage::ReadWrite);
+    auto synchronizationTransaction = store.createTransaction(Akonadi2::Storage::ReadWrite);
+    auto mainDatabase = transaction.openDatabase(bufferType + ".main");
+    scanForRemovals(transaction, synchronizationTransaction, bufferType, [&folderList](const QByteArray &remoteId) -> bool {
+        return folderList.contains(remoteId);
+    });
+
     for (const auto folderPath : folderList) {
         const auto remoteId = folderPath.toUtf8();
         Trace() << "Processing folder " << remoteId;
-        const auto akonadiId = resolveRemoteId(bufferType, remoteId, synchronizationTransaction).toLatin1();
-        const auto found = mainDatabase.contains(akonadiId);
-
         KPIM::Maildir md(folderPath, folderPath == mMaildirPath);
 
         Akonadi2::ApplicationDomain::Folder folder;
@@ -214,31 +253,7 @@ void MaildirResource::synchronizeFolders(Akonadi2::Storage::Transaction &transac
         if (!md.isRoot()) {
             folder.setProperty("parent", resolveRemoteId(ENTITY_TYPE_FOLDER, md.parent().path(), synchronizationTransaction).toLatin1());
         }
-
-        if (!found) {
-            Trace() << "Found a new entity: " << remoteId;
-            createEntity(akonadiId, bufferType, folder, *mFolderAdaptorFactory, [this](const QByteArray &buffer) {
-                enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::CreateEntityCommand, buffer);
-            });
-        } else { //modification
-            if (auto current = getLatest(mainDatabase, akonadiId, *mFolderAdaptorFactory)) {
-                bool changed = false;
-                for (const auto &property : folder.changedProperties()) {
-                    if (folder.getProperty(property) != current->getProperty(property)) {
-                        Trace() << "Property changed " << akonadiId << property;
-                        changed = true;
-                    }
-                }
-                if (changed) {
-                    Trace() << "Found a modified entity: " << remoteId;
-                    modifyEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, folder, *mFolderAdaptorFactory, [this](const QByteArray &buffer) {
-                        enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::ModifyEntityCommand, buffer);
-                    });
-                }
-            } else {
-                Warning() << "Failed to get current entity";
-            }
-        }
+        createOrModify(transaction, synchronizationTransaction, *mFolderAdaptorFactory, bufferType, remoteId, folder);
     }
 }
 
@@ -262,53 +277,36 @@ void MaildirResource::synchronizeMails(Akonadi2::Storage::Transaction &transacti
     Akonadi2::Storage store(Akonadi2::storageLocation(), mResourceInstanceIdentifier + ".synchronization", Akonadi2::Storage::ReadWrite);
     auto synchronizationTransaction = store.createTransaction(Akonadi2::Storage::ReadWrite);
 
+    scanForRemovals(transaction, synchronizationTransaction, bufferType, [&listingPath](const QByteArray &remoteId) -> bool {
+        return QFile(listingPath + QDir::separator() + remoteId).exists();
+    });
+
     while (entryIterator->hasNext()) {
         QString filePath = entryIterator->next();
         QString fileName = entryIterator->fileName();
-
         const auto remoteId = fileName.toUtf8();
-        auto akonadiId = resolveRemoteId(bufferType, remoteId, synchronizationTransaction);
-
-        bool found = false;
-        transaction.openDatabase(bufferType + ".main").scan(akonadiId.toUtf8(), [&found](const QByteArray &, const QByteArray &) -> bool {
-            found = true;
-            return false;
-        }, [this](const Akonadi2::Storage::Error &error) {
-        }, true);
-
-        if (!found) { //A new entity
-            KMime::Message *msg = new KMime::Message;
-            auto filepath = listingPath + QDir::separator() + fileName;
-            msg->setHead(KMime::CRLFtoLF(maildir.readEntryHeadersFromFile(filepath)));
-            msg->parse();
-
-            const auto flags = maildir.readEntryFlags(fileName);
-
-            Trace() << "Found a mail " << filePath << fileName << msg->subject(true)->asUnicodeString();
-
-            Akonadi2::ApplicationDomain::Mail mail;
-            mail.setProperty("subject", msg->subject(true)->asUnicodeString());
-            mail.setProperty("sender", msg->from(true)->asUnicodeString());
-            mail.setProperty("senderName", msg->from(true)->asUnicodeString());
-            mail.setProperty("date", msg->date(true)->dateTime().toString());
-            mail.setProperty("folder", resolveRemoteId(ENTITY_TYPE_FOLDER, path, synchronizationTransaction));
-            mail.setProperty("mimeMessage", filepath);
-            mail.setProperty("unread", !flags.testFlag(KPIM::Maildir::Seen));
-            mail.setProperty("important", flags.testFlag(KPIM::Maildir::Flagged));
-
-            flatbuffers::FlatBufferBuilder entityFbb;
-            mMailAdaptorFactory->createBuffer(mail, entityFbb);
-
-            Trace() << "Found a new entity: " << remoteId;
-            createEntity(akonadiId.toLatin1(), bufferType, mail, *mMailAdaptorFactory, [this](const QByteArray &buffer) {
-                enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::CreateEntityCommand, buffer);
-            });
-        } else { //modification
-            Trace() << "Found a modified entity: " << remoteId;
-            //TODO diff and create modification if necessary
-        }
+
+        KMime::Message *msg = new KMime::Message;
+        auto filepath = listingPath + QDir::separator() + fileName;
+        msg->setHead(KMime::CRLFtoLF(maildir.readEntryHeadersFromFile(filepath)));
+        msg->parse();
+
+        const auto flags = maildir.readEntryFlags(fileName);
+
+        Trace() << "Found a mail " << filePath << fileName << msg->subject(true)->asUnicodeString();
+
+        Akonadi2::ApplicationDomain::Mail mail;
+        mail.setProperty("subject", msg->subject(true)->asUnicodeString());
+        mail.setProperty("sender", msg->from(true)->asUnicodeString());
+        mail.setProperty("senderName", msg->from(true)->asUnicodeString());
+        mail.setProperty("date", msg->date(true)->dateTime().toString());
+        mail.setProperty("folder", resolveRemoteId(ENTITY_TYPE_FOLDER, path, synchronizationTransaction));
+        mail.setProperty("mimeMessage", filepath);
+        mail.setProperty("unread", !flags.testFlag(KPIM::Maildir::Seen));
+        mail.setProperty("important", flags.testFlag(KPIM::Maildir::Flagged));
+
+        createOrModify(transaction, synchronizationTransaction, *mMailAdaptorFactory, bufferType, remoteId, mail);
     }
-    //TODO find items to remove
 }
 
 KAsync::Job<void> MaildirResource::synchronizeWithSource()
diff --git a/examples/maildirresource/maildirresource.h b/examples/maildirresource/maildirresource.h
index e1eecc1..eec1e97 100644
--- a/examples/maildirresource/maildirresource.h
+++ b/examples/maildirresource/maildirresource.h
@@ -39,8 +39,36 @@ public:
     static void removeFromDisk(const QByteArray &instanceIdentifier);
 private:
     KAsync::Job<void> replay(const QByteArray &type, const QByteArray &key, const QByteArray &value) Q_DECL_OVERRIDE;
+
+    /**
+     * Tries to find a local id for the remote id, and creates a new local id otherwise.
+     * 
+     * The new local id is recorded in the local to remote id mapping.
+     */
     QString resolveRemoteId(const QByteArray &type, const QString &remoteId, Akonadi2::Storage::Transaction &transaction);
+
+    /**
+     * Tries to find a remote id for a local id.
+     * 
+     * This can fail if the entity hasn't been written back to the server yet.
+     */
     QString resolveLocalId(const QByteArray &bufferType, const QByteArray &localId, Akonadi2::Storage::Transaction &transaction);
+
+    /**
+    * An algorithm to remove entities that are no longer existing.
+    * 
+    * This algorithm calls @param exists for every entity of type @param type, with its remoteId. For every entity where @param exists returns false,
+    * an entity delete command is enqueued.
+    */
+    void scanForRemovals(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, const QByteArray &bufferType, std::function<bool(const QByteArray &remoteId)> exists);
+
+    /**
+     * An algorithm to create or modify the entity.
+     *
+     * Depending on whether the entity is locally available, or has changed.
+     */
+    void createOrModify(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, DomainTypeAdaptorFactoryInterface &adaptorFactory, const QByteArray &bufferType, const QByteArray &remoteId, const Akonadi2::ApplicationDomain::ApplicationDomainType &entity);
+
     void synchronizeFolders(Akonadi2::Storage::Transaction &transaction);
     void synchronizeMails(Akonadi2::Storage::Transaction &transaction, const QString &folder);
     QStringList listAvailableFolders();
diff --git a/tests/maildirresourcetest.cpp b/tests/maildirresourcetest.cpp
index e6b6849..a4856ce 100644
--- a/tests/maildirresourcetest.cpp
+++ b/tests/maildirresourcetest.cpp
@@ -149,6 +149,26 @@ private Q_SLOTS:
         QVERIFY(mailModel->rowCount(QModelIndex()) >= 1);
     }
 
+    void testMailContent()
+    {
+        Akonadi2::Query query;
+        query.resources << "org.kde.maildir.instance1";
+        query.requestedProperties << "folder" << "subject" << "mimeMessage";
+        query.syncOnDemand = true;
+        query.processAll = true;
+
+        //Ensure all local data is processed
+        Akonadi2::Store::synchronize(query).exec().waitForFinished();
+
+        auto mailModel = Akonadi2::Store::loadModel<Akonadi2::ApplicationDomain::Mail>(query);
+        QTRY_VERIFY(mailModel->data(QModelIndex(), Akonadi2::Store::ChildrenFetchedRole).toBool());
+        QVERIFY(mailModel->rowCount(QModelIndex()) >= 1);
+        auto mail = mailModel->index(0, 0, QModelIndex()).data(Akonadi2::Store::DomainObjectRole).value<Akonadi2::ApplicationDomain::Mail::Ptr>();
+        QVERIFY(!mail->getProperty("subject").toString().isEmpty());
+        QVERIFY(!mail->getProperty("mimeMessage").toString().isEmpty());
+    }
+
+
     void testSyncFolderMove()
     {
         Akonadi2::Query query;
@@ -173,6 +193,29 @@ private Q_SLOTS:
         QCOMPARE(model->match(model->index(0, 0, QModelIndex()), Qt::DisplayRole, QStringLiteral("newbox"), 1).size(), 1);
     }
 
+    void testSyncMailMove()
+    {
+        Akonadi2::Query query;
+        query.resources << "org.kde.maildir.instance1";
+        query.syncOnDemand = true;
+        query.processAll = true;
+        query.requestedProperties << "folder" << "summary";
+
+        //Ensure all local data is processed
+        Akonadi2::Store::synchronize(query).exec().waitForFinished();
+
+        auto targetPath = tempDir.path() + QDir::separator() + "maildir1/cur/1365777830.R28.localhost.localdomain:2,S";
+        QFile file(targetPath);
+        QVERIFY(file.remove());
+
+        //Ensure all local data is processed
+        Akonadi2::Store::synchronize(query).exec().waitForFinished();
+
+        auto mailModel = Akonadi2::Store::loadModel<Akonadi2::ApplicationDomain::Mail>(query);
+        QTRY_VERIFY(mailModel->data(QModelIndex(), Akonadi2::Store::ChildrenFetchedRole).toBool());
+        QCOMPARE(mailModel->rowCount(QModelIndex()), 1);
+    }
+
 };
 
 QTEST_MAIN(MaildirResourceTest)