4 files changed, 239 insertions, 223 deletions

diff --git a/common/genericresource.cpp b/common/genericresource.cpp
index 9fbcaaa..42153ec 100644
--- a/common/genericresource.cpp
+++ b/common/genericresource.cpp
@@ -4,12 +4,16 @@
 #include "pipeline.h"
 #include "queuedcommand_generated.h"
 #include "createentity_generated.h"
+#include "modifyentity_generated.h"
+#include "deleteentity_generated.h"
 #include "domainadaptor.h"
 #include "commands.h"
 #include "index.h"
 #include "log.h"
 #include "definitions.h"
 
+#include <QUuid>
+
 static int sBatchSize = 100;
 
 using namespace Akonadi2;
@@ -52,6 +56,7 @@ public:
     {
         const qint64 topRevision = Storage::maxRevision(mStorage.createTransaction(Storage::ReadOnly));
         const qint64 lastReplayedRevision = getLastReplayedRevision();
+        Trace() << "All changes replayed " << topRevision << lastReplayedRevision;
         return (lastReplayedRevision >= topRevision);
     }
 
@@ -444,4 +449,159 @@ void GenericResource::setLowerBoundRevision(qint64 revision)
     updateLowerBoundRevision();
 }
 
+void GenericResource::createEntity(const QByteArray &akonadiId, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback)
+{
+    //These changes are coming from the source
+    const auto replayToSource = false;
+    flatbuffers::FlatBufferBuilder entityFbb;
+    adaptorFactory.createBuffer(domainObject, entityFbb);
+    flatbuffers::FlatBufferBuilder fbb;
+    //This is the resource type and not the domain type
+    auto entityId = fbb.CreateString(akonadiId.toStdString());
+    auto type = fbb.CreateString(bufferType.toStdString());
+    auto delta = Akonadi2::EntityBuffer::appendAsVector(fbb, entityFbb.GetBufferPointer(), entityFbb.GetSize());
+    auto location = Akonadi2::Commands::CreateCreateEntity(fbb, entityId, type, delta, replayToSource);
+    Akonadi2::Commands::FinishCreateEntityBuffer(fbb, location);
+    callback(QByteArray::fromRawData(reinterpret_cast<char const *>(fbb.GetBufferPointer()), fbb.GetSize()));
+}
+
+void GenericResource::modifyEntity(const QByteArray &akonadiId, qint64 revision, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback)
+{
+    //These changes are coming from the source
+    const auto replayToSource = false;
+    flatbuffers::FlatBufferBuilder entityFbb;
+    adaptorFactory.createBuffer(domainObject, entityFbb);
+    flatbuffers::FlatBufferBuilder fbb;
+    auto entityId = fbb.CreateString(akonadiId.toStdString());
+    //This is the resource type and not the domain type
+    auto type = fbb.CreateString(bufferType.toStdString());
+    auto delta = Akonadi2::EntityBuffer::appendAsVector(fbb, entityFbb.GetBufferPointer(), entityFbb.GetSize());
+    //TODO removals
+    auto location = Akonadi2::Commands::CreateModifyEntity(fbb, revision, entityId, 0, type, delta, replayToSource);
+    Akonadi2::Commands::FinishModifyEntityBuffer(fbb, location);
+    callback(QByteArray::fromRawData(reinterpret_cast<char const *>(fbb.GetBufferPointer()), fbb.GetSize()));
+}
+
+void GenericResource::deleteEntity(const QByteArray &akonadiId, qint64 revision, const QByteArray &bufferType, std::function<void(const QByteArray &)> callback)
+{
+    //These changes are coming from the source
+    const auto replayToSource = false;
+    flatbuffers::FlatBufferBuilder fbb;
+    auto entityId = fbb.CreateString(akonadiId.toStdString());
+    //This is the resource type and not the domain type
+    auto type = fbb.CreateString(bufferType.toStdString());
+    auto location = Akonadi2::Commands::CreateDeleteEntity(fbb, revision, entityId, type, replayToSource);
+    Akonadi2::Commands::FinishDeleteEntityBuffer(fbb, location);
+    callback(QByteArray::fromRawData(reinterpret_cast<char const *>(fbb.GetBufferPointer()), fbb.GetSize()));
+}
+
+void GenericResource::recordRemoteId(const QByteArray &bufferType, const QByteArray &localId, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction)
+{
+    Index index("rid.mapping." + bufferType, transaction);
+    Index localIndex("localid.mapping." + bufferType, transaction);
+    index.add(remoteId, localId);
+    localIndex.add(localId, remoteId);
+}
+
+void GenericResource::removeRemoteId(const QByteArray &bufferType, const QByteArray &localId, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction)
+{
+    Index index("rid.mapping." + bufferType, transaction);
+    Index localIndex("localid.mapping." + bufferType, transaction);
+    index.remove(remoteId, localId);
+    localIndex.remove(localId, remoteId);
+}
+
+QByteArray GenericResource::resolveRemoteId(const QByteArray &bufferType, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction)
+{
+    //Lookup local id for remote id, or insert a new pair otherwise
+    Index index("rid.mapping." + bufferType, transaction);
+    Index localIndex("localid.mapping." + bufferType, transaction);
+    QByteArray akonadiId = index.lookup(remoteId);
+    if (akonadiId.isEmpty()) {
+        akonadiId = QUuid::createUuid().toString().toUtf8();
+        index.add(remoteId, akonadiId);
+        localIndex.add(akonadiId, remoteId);
+    }
+    return akonadiId;
+}
+
+QByteArray GenericResource::resolveLocalId(const QByteArray &bufferType, const QByteArray &localId, Akonadi2::Storage::Transaction &transaction)
+{
+    Index index("localid.mapping." + bufferType, transaction);
+    QByteArray remoteId = index.lookup(localId);
+    if (remoteId.isEmpty()) {
+        Warning() << "Couldn't find the remote id for " << localId;
+        return QByteArray();
+    }
+    return remoteId;
+}
+
+void GenericResource::scanForRemovals(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, const QByteArray &bufferType, const std::function<void(const std::function<void(const QByteArray &key)> &callback)> &entryGenerator, std::function<bool(const QByteArray &remoteId)> exists)
+{
+    entryGenerator([this, &transaction, bufferType, &synchronizationTransaction, &exists](const QByteArray &key) {
+        auto akonadiId = Akonadi2::Storage::uidFromKey(key);
+        Trace() << "Checking for removal " << key;
+        const auto remoteId = resolveLocalId(bufferType, akonadiId, synchronizationTransaction);
+        //If we have no remoteId, the entity hasn't been replayed to the source yet
+        if (!remoteId.isEmpty()) {
+            if (!exists(remoteId)) {
+                Trace() << "Found a removed entity: " << akonadiId;
+                deleteEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, [this](const QByteArray &buffer) {
+                    enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::DeleteEntityCommand, buffer);
+                });
+            }
+        }
+    });
+}
+
+static QSharedPointer<Akonadi2::ApplicationDomain::BufferAdaptor> getLatest(const Akonadi2::Storage::NamedDatabase &db, const QByteArray &uid, DomainTypeAdaptorFactoryInterface &adaptorFactory)
+{
+    QSharedPointer<Akonadi2::ApplicationDomain::BufferAdaptor> current;
+    db.findLatest(uid, [&current, &adaptorFactory](const QByteArray &key, const QByteArray &data) -> bool {
+        Akonadi2::EntityBuffer buffer(const_cast<const char *>(data.data()), data.size());
+        if (!buffer.isValid()) {
+            Warning() << "Read invalid buffer from disk";
+        } else {
+            current = adaptorFactory.createAdaptor(buffer.entity());
+        }
+        return false;
+    },
+    [](const Akonadi2::Storage::Error &error) {
+        Warning() << "Failed to read current value from storage: " << error.message;
+    });
+    return current;
+}
+
+void GenericResource::createOrModify(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, DomainTypeAdaptorFactoryInterface &adaptorFactory, const QByteArray &bufferType, const QByteArray &remoteId, const Akonadi2::ApplicationDomain::ApplicationDomainType &entity)
+{
+    auto mainDatabase = transaction.openDatabase(bufferType + ".main");
+    const auto akonadiId = resolveRemoteId(bufferType, remoteId, synchronizationTransaction);
+    const auto found = mainDatabase.contains(akonadiId);
+    if (!found) {
+        Trace() << "Found a new entity: " << remoteId;
+        createEntity(akonadiId, bufferType, entity, adaptorFactory, [this](const QByteArray &buffer) {
+            enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::CreateEntityCommand, buffer);
+        });
+    } else { //modification
+        if (auto current = getLatest(mainDatabase, akonadiId, adaptorFactory)) {
+            bool changed = false;
+            for (const auto &property : entity.changedProperties()) {
+                if (entity.getProperty(property) != current->getProperty(property)) {
+                    Trace() << "Property changed " << akonadiId << property;
+                    changed = true;
+                }
+            }
+            if (changed) {
+                Trace() << "Found a modified entity: " << remoteId;
+                modifyEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, entity, adaptorFactory, [this](const QByteArray &buffer) {
+                    enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::ModifyEntityCommand, buffer);
+                });
+            }
+        } else {
+            Warning() << "Failed to get current entity";
+        }
+    }
+}
+
+
 #include "genericresource.moc"
diff --git a/common/genericresource.h b/common/genericresource.h
index ea68a25..c12c631 100644
--- a/common/genericresource.h
+++ b/common/genericresource.h
@@ -63,6 +63,50 @@ protected:
     virtual KAsync::Job<void> replay(const QByteArray &type, const QByteArray &key, const QByteArray &value);
     void onProcessorError(int errorCode, const QString &errorMessage);
     void enqueueCommand(MessageQueue &mq, int commandId, const QByteArray &data);
+
+    static void createEntity(const QByteArray &localId, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback);
+    static void modifyEntity(const QByteArray &localId, qint64 revision, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback);
+    static void deleteEntity(const QByteArray &localId, qint64 revision, const QByteArray &bufferType, std::function<void(const QByteArray &)> callback);
+
+    /**
+     * Records a localId to remoteId mapping
+     */
+    void recordRemoteId(const QByteArray &bufferType, const QByteArray &localId, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction);
+    void removeRemoteId(const QByteArray &bufferType, const QByteArray &localId, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction);
+
+    /**
+     * Tries to find a local id for the remote id, and creates a new local id otherwise.
+     * 
+     * The new local id is recorded in the local to remote id mapping.
+     */
+    QByteArray resolveRemoteId(const QByteArray &type, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction);
+
+    /**
+     * Tries to find a remote id for a local id.
+     * 
+     * This can fail if the entity hasn't been written back to the server yet.
+     */
+    QByteArray resolveLocalId(const QByteArray &bufferType, const QByteArray &localId, Akonadi2::Storage::Transaction &transaction);
+
+    /**
+    * A synchronous algorithm to remove entities that are no longer existing.
+    * 
+    * A list of entities is generated by @param entryGenerator.
+    * The entiry Generator typically iterates over an index to produce all existing entries.
+    * This algorithm calls @param exists for every entity of type @param type, with its remoteId. For every entity where @param exists returns false,
+    * an entity delete command is enqueued.
+    * 
+    * All functions are called synchronously, and both @param entryGenerator and @param exists need to be synchronous.
+    */
+    void scanForRemovals(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, const QByteArray &bufferType, const std::function<void(const std::function<void(const QByteArray &key)> &callback)> &entryGenerator, std::function<bool(const QByteArray &remoteId)> exists);
+
+    /**
+     * An algorithm to create or modify the entity.
+     *
+     * Depending on whether the entity is locally available, or has changed.
+     */
+    void createOrModify(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, DomainTypeAdaptorFactoryInterface &adaptorFactory, const QByteArray &bufferType, const QByteArray &remoteId, const Akonadi2::ApplicationDomain::ApplicationDomainType &entity);
+
     MessageQueue mUserQueue;
     MessageQueue mSynchronizerQueue;
     QByteArray mResourceInstanceIdentifier;
diff --git a/examples/maildirresource/maildirresource.cpp b/examples/maildirresource/maildirresource.cpp
index 6c6c5aa..273b996 100644
--- a/examples/maildirresource/maildirresource.cpp
+++ b/examples/maildirresource/maildirresource.cpp
@@ -63,46 +63,6 @@ MaildirResource::MaildirResource(const QByteArray &instanceIdentifier, const QSh
     Trace() << "Started maildir resource for maildir: " << mMaildirPath;
 }
 
-void MaildirResource::recordRemoteId(const QByteArray &bufferType, const QByteArray &localId, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction)
-{
-    Index index("rid.mapping." + bufferType, transaction);
-    Index localIndex("localid.mapping." + bufferType, transaction);
-    index.add(remoteId, localId);
-    localIndex.add(localId, remoteId);
-}
-
-void MaildirResource::removeRemoteId(const QByteArray &bufferType, const QByteArray &localId, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction)
-{
-    Index index("rid.mapping." + bufferType, transaction);
-    Index localIndex("localid.mapping." + bufferType, transaction);
-    index.remove(remoteId, localId);
-    localIndex.remove(localId, remoteId);
-}
-
-QString MaildirResource::resolveRemoteId(const QByteArray &bufferType, const QString &remoteId, Akonadi2::Storage::Transaction &transaction)
-{
-    //Lookup local id for remote id, or insert a new pair otherwise
-    Index index("rid.mapping." + bufferType, transaction);
-    Index localIndex("localid.mapping." + bufferType, transaction);
-    QByteArray akonadiId = index.lookup(remoteId.toUtf8());
-    if (akonadiId.isEmpty()) {
-        akonadiId = QUuid::createUuid().toString().toUtf8();
-        index.add(remoteId.toUtf8(), akonadiId);
-        localIndex.add(akonadiId, remoteId.toUtf8());
-    }
-    return akonadiId;
-}
-
-QString MaildirResource::resolveLocalId(const QByteArray &bufferType, const QByteArray &localId, Akonadi2::Storage::Transaction &transaction)
-{
-    Index index("localid.mapping." + bufferType, transaction);
-    QByteArray remoteId = index.lookup(localId);
-    if (remoteId.isEmpty()) {
-        Warning() << "Couldn't find the remote id for " << localId;
-    }
-    return remoteId;
-}
-
 static QStringList listRecursive( const QString &root, const KPIM::Maildir &dir )
 {
     QStringList list;
@@ -130,136 +90,28 @@ QStringList MaildirResource::listAvailableFolders()
     return folderList;
 }
 
-static void createEntity(const QByteArray &akonadiId, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback)
-{
-    //These changes are coming from the source
-    const auto replayToSource = false;
-    flatbuffers::FlatBufferBuilder entityFbb;
-    adaptorFactory.createBuffer(domainObject, entityFbb);
-    flatbuffers::FlatBufferBuilder fbb;
-    //This is the resource type and not the domain type
-    auto entityId = fbb.CreateString(akonadiId.toStdString());
-    auto type = fbb.CreateString(bufferType.toStdString());
-    auto delta = Akonadi2::EntityBuffer::appendAsVector(fbb, entityFbb.GetBufferPointer(), entityFbb.GetSize());
-    auto location = Akonadi2::Commands::CreateCreateEntity(fbb, entityId, type, delta, replayToSource);
-    Akonadi2::Commands::FinishCreateEntityBuffer(fbb, location);
-    callback(QByteArray::fromRawData(reinterpret_cast<char const *>(fbb.GetBufferPointer()), fbb.GetSize()));
-}
-
-static void modifyEntity(const QByteArray &akonadiId, qint64 revision, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function<void(const QByteArray &)> callback)
-{
-    //These changes are coming from the source
-    const auto replayToSource = false;
-    flatbuffers::FlatBufferBuilder entityFbb;
-    adaptorFactory.createBuffer(domainObject, entityFbb);
-    flatbuffers::FlatBufferBuilder fbb;
-    auto entityId = fbb.CreateString(akonadiId.toStdString());
-    //This is the resource type and not the domain type
-    auto type = fbb.CreateString(bufferType.toStdString());
-    auto delta = Akonadi2::EntityBuffer::appendAsVector(fbb, entityFbb.GetBufferPointer(), entityFbb.GetSize());
-    //TODO removals
-    auto location = Akonadi2::Commands::CreateModifyEntity(fbb, revision, entityId, 0, type, delta, replayToSource);
-    Akonadi2::Commands::FinishModifyEntityBuffer(fbb, location);
-    callback(QByteArray::fromRawData(reinterpret_cast<char const *>(fbb.GetBufferPointer()), fbb.GetSize()));
-}
-
-static void deleteEntity(const QByteArray &akonadiId, qint64 revision, const QByteArray &bufferType, std::function<void(const QByteArray &)> callback)
-{
-    //These changes are coming from the source
-    const auto replayToSource = false;
-    flatbuffers::FlatBufferBuilder fbb;
-    auto entityId = fbb.CreateString(akonadiId.toStdString());
-    //This is the resource type and not the domain type
-    auto type = fbb.CreateString(bufferType.toStdString());
-    auto location = Akonadi2::Commands::CreateDeleteEntity(fbb, revision, entityId, type, replayToSource);
-    Akonadi2::Commands::FinishDeleteEntityBuffer(fbb, location);
-    callback(QByteArray::fromRawData(reinterpret_cast<char const *>(fbb.GetBufferPointer()), fbb.GetSize()));
-}
-
-static QSharedPointer<Akonadi2::ApplicationDomain::BufferAdaptor> getLatest(const Akonadi2::Storage::NamedDatabase &db, const QByteArray &uid, DomainTypeAdaptorFactoryInterface &adaptorFactory)
-{
-    QSharedPointer<Akonadi2::ApplicationDomain::BufferAdaptor> current;
-    db.findLatest(uid, [&current, &adaptorFactory](const QByteArray &key, const QByteArray &data) -> bool {
-        Akonadi2::EntityBuffer buffer(const_cast<const char *>(data.data()), data.size());
-        if (!buffer.isValid()) {
-            Warning() << "Read invalid buffer from disk";
-        } else {
-            current = adaptorFactory.createAdaptor(buffer.entity());
-        }
-        return false;
-    },
-    [](const Akonadi2::Storage::Error &error) {
-        Warning() << "Failed to read current value from storage: " << error.message;
-    });
-    return current;
-}
-
-void MaildirResource::scanForRemovals(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, const QByteArray &bufferType, std::function<bool(const QByteArray &remoteId)> exists)
-{
-    auto mainDatabase = transaction.openDatabase(bufferType + ".main");
-    //TODO Instead of iterating over all entries in the database, which can also pick up the same item multiple times,
-    //we should rather iterate over an index that contains every uid exactly once. The remoteId index would be such an index,
-    //but we currently fail to iterate over all entries in an index it seems.
-    // auto remoteIds = synchronizationTransaction.openDatabase("rid.mapping." + bufferType, std::function<void(const Akonadi2::Storage::Error &)>(), true);
-    mainDatabase.scan("", [this, &transaction, bufferType, &synchronizationTransaction, &exists](const QByteArray &key, const QByteArray &) {
-        auto akonadiId = Akonadi2::Storage::uidFromKey(key);
-        Trace() << "Checking for removal " << key;
-        const auto remoteId = resolveLocalId(bufferType, akonadiId, synchronizationTransaction);
-        if (!remoteId.isEmpty()) {
-            if (!exists(remoteId.toLatin1())) {
-                Trace() << "Found a removed entity: " << akonadiId;
-                deleteEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, [this](const QByteArray &buffer) {
-                    enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::DeleteEntityCommand, buffer);
-                });
-            }
-        }
-        return true;
-    },
-    [](const Akonadi2::Storage::Error &error) {
-    });
-
-}
-
-void MaildirResource::createOrModify(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, DomainTypeAdaptorFactoryInterface &adaptorFactory, const QByteArray &bufferType, const QByteArray &remoteId, const Akonadi2::ApplicationDomain::ApplicationDomainType &entity)
-{
-    auto mainDatabase = transaction.openDatabase(bufferType + ".main");
-    const auto akonadiId = resolveRemoteId(bufferType, remoteId, synchronizationTransaction).toLatin1();
-    const auto found = mainDatabase.contains(akonadiId);
-    if (!found) {
-        Trace() << "Found a new entity: " << remoteId;
-        createEntity(akonadiId, bufferType, entity, adaptorFactory, [this](const QByteArray &buffer) {
-            enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::CreateEntityCommand, buffer);
-        });
-    } else { //modification
-        if (auto current = getLatest(mainDatabase, akonadiId, adaptorFactory)) {
-            bool changed = false;
-            for (const auto &property : entity.changedProperties()) {
-                if (entity.getProperty(property) != current->getProperty(property)) {
-                    Trace() << "Property changed " << akonadiId << property;
-                    changed = true;
-                }
-            }
-            if (changed) {
-                Trace() << "Found a modified entity: " << remoteId;
-                modifyEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, entity, adaptorFactory, [this](const QByteArray &buffer) {
-                    enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::ModifyEntityCommand, buffer);
-                });
-            }
-        } else {
-            Warning() << "Failed to get current entity";
-        }
-    }
-}
-
 void MaildirResource::synchronizeFolders(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction)
 {
     const QByteArray bufferType = ENTITY_TYPE_FOLDER;
     QStringList folderList = listAvailableFolders();
     Trace() << "Found folders " << folderList;
 
-    scanForRemovals(transaction, synchronizationTransaction, bufferType, [&folderList](const QByteArray &remoteId) -> bool {
-        return folderList.contains(remoteId);
-    });
+    scanForRemovals(transaction, synchronizationTransaction, bufferType,
+        [&bufferType, &transaction](const std::function<void(const QByteArray &)> &callback) {
+            //TODO Instead of iterating over all entries in the database, which can also pick up the same item multiple times,
+            //we should rather iterate over an index that contains every uid exactly once. The remoteId index would be such an index,
+            //but we currently fail to iterate over all entries in an index it seems.
+            // auto remoteIds = synchronizationTransaction.openDatabase("rid.mapping." + bufferType, std::function<void(const Akonadi2::Storage::Error &)>(), true);
+            auto mainDatabase = transaction.openDatabase(bufferType + ".main");
+            mainDatabase.scan("", [&](const QByteArray &key, const QByteArray &) {
+                callback(key);
+                return true;
+            });
+        },
+        [&folderList](const QByteArray &remoteId) -> bool {
+            return folderList.contains(remoteId);
+        }
+    );
 
     for (const auto folderPath : folderList) {
         const auto remoteId = folderPath.toUtf8();
@@ -270,7 +122,7 @@ void MaildirResource::synchronizeFolders(Akonadi2::Storage::Transaction &transac
         folder.setProperty("name", md.name());
         folder.setProperty("icon", "folder");
         if (!md.isRoot()) {
-            folder.setProperty("parent", resolveRemoteId(ENTITY_TYPE_FOLDER, md.parent().path(), synchronizationTransaction).toLatin1());
+            folder.setProperty("parent", resolveRemoteId(ENTITY_TYPE_FOLDER, md.parent().path().toUtf8(), synchronizationTransaction));
         }
         createOrModify(transaction, synchronizationTransaction, *mFolderAdaptorFactory, bufferType, remoteId, folder);
     }
@@ -293,28 +145,23 @@ void MaildirResource::synchronizeMails(Akonadi2::Storage::Transaction &transacti
 
     QFileInfo entryInfo;
 
-    const auto folderLocalId = resolveRemoteId(ENTITY_TYPE_FOLDER, path, synchronizationTransaction);
-
-    auto exists = [&listingPath](const QByteArray &remoteId) -> bool {
-        return QFile(listingPath + "/" + remoteId).exists();
-    };
+    const auto folderLocalId = resolveRemoteId(ENTITY_TYPE_FOLDER, path.toUtf8(), synchronizationTransaction);
 
     auto property = "folder";
-    Index index(bufferType + ".index." + property, transaction);
-    index.lookup(folderLocalId.toLatin1(), [&](const QByteArray &akonadiId) {
-        const auto remoteId = resolveLocalId(bufferType, akonadiId, synchronizationTransaction);
-        if (!remoteId.isEmpty()) {
-            if (!exists(remoteId.toLatin1())) {
-                Trace() << "Found a removed entity: " << akonadiId;
-                deleteEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, [this](const QByteArray &buffer) {
-                    enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::DeleteEntityCommand, buffer);
-                });
-            }
+    scanForRemovals(transaction, synchronizationTransaction, bufferType,
+        [&](const std::function<void(const QByteArray &)> &callback) {
+            Index index(bufferType + ".index." + property, transaction);
+            index.lookup(folderLocalId, [&](const QByteArray &akonadiId) {
+                callback(akonadiId);
+            },
+            [&](const Index::Error &error) {
+                Warning() << "Error in index: " <<  error.message << property;
+            });
+        },
+        [&listingPath](const QByteArray &remoteId) -> bool {
+            return QFile(listingPath + "/" + remoteId).exists();
         }
-    },
-    [property](const Index::Error &error) {
-        Warning() << "Error in index: " <<  error.message << property;
-    });
+    );
 
     while (entryIterator->hasNext()) {
         QString filePath = entryIterator->next();
@@ -398,7 +245,7 @@ KAsync::Job<void> MaildirResource::replay(const QByteArray &type, const QByteArr
             Trace() << "Removing a folder: " << path;
             KPIM::Maildir maildir(path, false);
             maildir.remove();
-            removeRemoteId(ENTITY_TYPE_FOLDER, uid, remoteId.toUtf8(), synchronizationTransaction);
+            removeRemoteId(ENTITY_TYPE_FOLDER, uid, remoteId, synchronizationTransaction);
         } else if (operation == Akonadi2::Operation_Modification) {
             Warning() << "Folder modifications are not implemented";
         } else {
@@ -419,9 +266,9 @@ KAsync::Job<void> MaildirResource::replay(const QByteArray &type, const QByteArr
             auto parentFolder = mail.getProperty("folder").toByteArray();
             QByteArray parentFolderRemoteId;
             if (!parentFolder.isEmpty()) {
-                parentFolderRemoteId = resolveLocalId(ENTITY_TYPE_FOLDER, parentFolder, synchronizationTransaction).toLatin1();
+                parentFolderRemoteId = resolveLocalId(ENTITY_TYPE_FOLDER, parentFolder, synchronizationTransaction);
             } else {
-                parentFolderRemoteId = mMaildirPath.toLatin1();
+                parentFolderRemoteId = mMaildirPath.toUtf8();
             }
             const auto parentFolderPath = parentFolderRemoteId;
             KPIM::Maildir maildir(parentFolderPath, false);
@@ -439,7 +286,7 @@ KAsync::Job<void> MaildirResource::replay(const QByteArray &type, const QByteArr
             KPIM::Maildir maildir(parentFolderPath, false);
             Trace() << "Removing a mail: " << remoteId;
             maildir.removeEntry(remoteId);
-            removeRemoteId(ENTITY_TYPE_MAIL, uid, remoteId.toUtf8(), synchronizationTransaction);
+            removeRemoteId(ENTITY_TYPE_MAIL, uid, remoteId, synchronizationTransaction);
         } else if (operation == Akonadi2::Operation_Modification) {
             Warning() << "Mail modifications are not implemented";
         } else {
diff --git a/examples/maildirresource/maildirresource.h b/examples/maildirresource/maildirresource.h
index 9c205c8..48eac67 100644
--- a/examples/maildirresource/maildirresource.h
+++ b/examples/maildirresource/maildirresource.h
@@ -40,41 +40,6 @@ public:
 private:
     KAsync::Job<void> replay(const QByteArray &type, const QByteArray &key, const QByteArray &value) Q_DECL_OVERRIDE;
 
-    /**
-     * Records a localId to remoteId mapping
-     */
-    void recordRemoteId(const QByteArray &bufferType, const QByteArray &localId, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction);
-    void removeRemoteId(const QByteArray &bufferType, const QByteArray &localId, const QByteArray &remoteId, Akonadi2::Storage::Transaction &transaction);
-
-    /**
-     * Tries to find a local id for the remote id, and creates a new local id otherwise.
-     * 
-     * The new local id is recorded in the local to remote id mapping.
-     */
-    QString resolveRemoteId(const QByteArray &type, const QString &remoteId, Akonadi2::Storage::Transaction &transaction);
-
-    /**
-     * Tries to find a remote id for a local id.
-     * 
-     * This can fail if the entity hasn't been written back to the server yet.
-     */
-    QString resolveLocalId(const QByteArray &bufferType, const QByteArray &localId, Akonadi2::Storage::Transaction &transaction);
-
-    /**
-    * An algorithm to remove entities that are no longer existing.
-    * 
-    * This algorithm calls @param exists for every entity of type @param type, with its remoteId. For every entity where @param exists returns false,
-    * an entity delete command is enqueued.
-    */
-    void scanForRemovals(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, const QByteArray &bufferType, std::function<bool(const QByteArray &remoteId)> exists);
-
-    /**
-     * An algorithm to create or modify the entity.
-     *
-     * Depending on whether the entity is locally available, or has changed.
-     */
-    void createOrModify(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, DomainTypeAdaptorFactoryInterface &adaptorFactory, const QByteArray &bufferType, const QByteArray &remoteId, const Akonadi2::ApplicationDomain::ApplicationDomainType &entity);
-
     void synchronizeFolders(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction);
     void synchronizeMails(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, const QString &folder);
     QStringList listAvailableFolders();