From b1e5525be34850ef4d11cccbf23e118c93e93506 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 19 Dec 2015 19:39:53 +0100 Subject: Generalized the sync algorithms and applied them to mail. Not necessarily the smartest algorithms, but at least they work and are generally applicable. --- examples/maildirresource/maildirresource.cpp | 170 +++++++++++++-------------- examples/maildirresource/maildirresource.h | 28 +++++ 2 files changed, 112 insertions(+), 86 deletions(-) (limited to 'examples') diff --git a/examples/maildirresource/maildirresource.cpp b/examples/maildirresource/maildirresource.cpp index 10e9046..90b4e55 100644 --- a/examples/maildirresource/maildirresource.cpp +++ b/examples/maildirresource/maildirresource.cpp @@ -114,7 +114,7 @@ QStringList MaildirResource::listAvailableFolders() return folderList; } -static void createEntity(const QByteArray &akonadiId, const QByteArray &bufferType, Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function callback) +static void createEntity(const QByteArray &akonadiId, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function callback) { flatbuffers::FlatBufferBuilder entityFbb; adaptorFactory.createBuffer(domainObject, entityFbb); @@ -128,7 +128,7 @@ static void createEntity(const QByteArray &akonadiId, const QByteArray &bufferTy callback(QByteArray::fromRawData(reinterpret_cast(fbb.GetBufferPointer()), fbb.GetSize())); } -static void modifyEntity(const QByteArray &akonadiId, qint64 revision, const QByteArray &bufferType, Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function callback) +static void modifyEntity(const QByteArray &akonadiId, qint64 revision, const QByteArray &bufferType, const Akonadi2::ApplicationDomain::ApplicationDomainType &domainObject, DomainTypeAdaptorFactoryInterface &adaptorFactory, std::function callback) { flatbuffers::FlatBufferBuilder entityFbb; adaptorFactory.createBuffer(domainObject, entityFbb); @@ -172,40 +172,79 @@ static QSharedPointer getLatest(cons return current; } -void MaildirResource::synchronizeFolders(Akonadi2::Storage::Transaction &transaction) +void MaildirResource::scanForRemovals(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, const QByteArray &bufferType, std::function exists) { - const QByteArray bufferType = ENTITY_TYPE_FOLDER; - QStringList folderList = listAvailableFolders(); - Trace() << "Found folders " << folderList; - - Akonadi2::Storage store(Akonadi2::storageLocation(), mResourceInstanceIdentifier + ".synchronization", Akonadi2::Storage::ReadWrite); - auto synchronizationTransaction = store.createTransaction(Akonadi2::Storage::ReadWrite); auto mainDatabase = transaction.openDatabase(bufferType + ".main"); - //TODO Instead of iterating over all entries in the database, which can also pick up the same item multiple times, //we should rather iterate over an index that contains every uid exactly once. The remoteId index would be such an index, //but we currently fail to iterate over all entries in an index it seems. // auto remoteIds = synchronizationTransaction.openDatabase("rid.mapping." + bufferType, std::function(), true); - mainDatabase.scan("", [&folderList, this, &transaction, bufferType, &synchronizationTransaction](const QByteArray &key, const QByteArray &) { + mainDatabase.scan("", [this, &transaction, bufferType, &synchronizationTransaction, &exists](const QByteArray &key, const QByteArray &) { auto akonadiId = Akonadi2::Storage::uidFromKey(key); + Trace() << "Checking for removal " << key; const auto remoteId = resolveLocalId(bufferType, akonadiId, synchronizationTransaction); - if (!folderList.contains(remoteId)) { - Trace() << "Found a removed entity: " << akonadiId; - deleteEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, [this](const QByteArray &buffer) { - enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::DeleteEntityCommand, buffer); - }); + if (!remoteId.isEmpty()) { + if (!exists(remoteId.toLatin1())) { + Trace() << "Found a removed entity: " << akonadiId; + deleteEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, [this](const QByteArray &buffer) { + enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::DeleteEntityCommand, buffer); + }); + } } return true; }, [](const Akonadi2::Storage::Error &error) { }); +} + +void MaildirResource::createOrModify(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, DomainTypeAdaptorFactoryInterface &adaptorFactory, const QByteArray &bufferType, const QByteArray &remoteId, const Akonadi2::ApplicationDomain::ApplicationDomainType &entity) +{ + auto mainDatabase = transaction.openDatabase(bufferType + ".main"); + const auto akonadiId = resolveRemoteId(bufferType, remoteId, synchronizationTransaction).toLatin1(); + const auto found = mainDatabase.contains(akonadiId); + if (!found) { + Trace() << "Found a new entity: " << remoteId; + createEntity(akonadiId, bufferType, entity, adaptorFactory, [this](const QByteArray &buffer) { + enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::CreateEntityCommand, buffer); + }); + } else { //modification + if (auto current = getLatest(mainDatabase, akonadiId, adaptorFactory)) { + bool changed = false; + for (const auto &property : entity.changedProperties()) { + if (entity.getProperty(property) != current->getProperty(property)) { + Trace() << "Property changed " << akonadiId << property; + changed = true; + } + } + if (changed) { + Trace() << "Found a modified entity: " << remoteId; + modifyEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, entity, adaptorFactory, [this](const QByteArray &buffer) { + enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::ModifyEntityCommand, buffer); + }); + } + } else { + Warning() << "Failed to get current entity"; + } + } +} + +void MaildirResource::synchronizeFolders(Akonadi2::Storage::Transaction &transaction) +{ + const QByteArray bufferType = ENTITY_TYPE_FOLDER; + QStringList folderList = listAvailableFolders(); + Trace() << "Found folders " << folderList; + + Akonadi2::Storage store(Akonadi2::storageLocation(), mResourceInstanceIdentifier + ".synchronization", Akonadi2::Storage::ReadWrite); + auto synchronizationTransaction = store.createTransaction(Akonadi2::Storage::ReadWrite); + auto mainDatabase = transaction.openDatabase(bufferType + ".main"); + scanForRemovals(transaction, synchronizationTransaction, bufferType, [&folderList](const QByteArray &remoteId) -> bool { + return folderList.contains(remoteId); + }); + for (const auto folderPath : folderList) { const auto remoteId = folderPath.toUtf8(); Trace() << "Processing folder " << remoteId; - const auto akonadiId = resolveRemoteId(bufferType, remoteId, synchronizationTransaction).toLatin1(); - const auto found = mainDatabase.contains(akonadiId); - KPIM::Maildir md(folderPath, folderPath == mMaildirPath); Akonadi2::ApplicationDomain::Folder folder; @@ -214,31 +253,7 @@ void MaildirResource::synchronizeFolders(Akonadi2::Storage::Transaction &transac if (!md.isRoot()) { folder.setProperty("parent", resolveRemoteId(ENTITY_TYPE_FOLDER, md.parent().path(), synchronizationTransaction).toLatin1()); } - - if (!found) { - Trace() << "Found a new entity: " << remoteId; - createEntity(akonadiId, bufferType, folder, *mFolderAdaptorFactory, [this](const QByteArray &buffer) { - enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::CreateEntityCommand, buffer); - }); - } else { //modification - if (auto current = getLatest(mainDatabase, akonadiId, *mFolderAdaptorFactory)) { - bool changed = false; - for (const auto &property : folder.changedProperties()) { - if (folder.getProperty(property) != current->getProperty(property)) { - Trace() << "Property changed " << akonadiId << property; - changed = true; - } - } - if (changed) { - Trace() << "Found a modified entity: " << remoteId; - modifyEntity(akonadiId, Akonadi2::Storage::maxRevision(transaction), bufferType, folder, *mFolderAdaptorFactory, [this](const QByteArray &buffer) { - enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::ModifyEntityCommand, buffer); - }); - } - } else { - Warning() << "Failed to get current entity"; - } - } + createOrModify(transaction, synchronizationTransaction, *mFolderAdaptorFactory, bufferType, remoteId, folder); } } @@ -262,53 +277,36 @@ void MaildirResource::synchronizeMails(Akonadi2::Storage::Transaction &transacti Akonadi2::Storage store(Akonadi2::storageLocation(), mResourceInstanceIdentifier + ".synchronization", Akonadi2::Storage::ReadWrite); auto synchronizationTransaction = store.createTransaction(Akonadi2::Storage::ReadWrite); + scanForRemovals(transaction, synchronizationTransaction, bufferType, [&listingPath](const QByteArray &remoteId) -> bool { + return QFile(listingPath + QDir::separator() + remoteId).exists(); + }); + while (entryIterator->hasNext()) { QString filePath = entryIterator->next(); QString fileName = entryIterator->fileName(); - const auto remoteId = fileName.toUtf8(); - auto akonadiId = resolveRemoteId(bufferType, remoteId, synchronizationTransaction); - - bool found = false; - transaction.openDatabase(bufferType + ".main").scan(akonadiId.toUtf8(), [&found](const QByteArray &, const QByteArray &) -> bool { - found = true; - return false; - }, [this](const Akonadi2::Storage::Error &error) { - }, true); - - if (!found) { //A new entity - KMime::Message *msg = new KMime::Message; - auto filepath = listingPath + QDir::separator() + fileName; - msg->setHead(KMime::CRLFtoLF(maildir.readEntryHeadersFromFile(filepath))); - msg->parse(); - - const auto flags = maildir.readEntryFlags(fileName); - - Trace() << "Found a mail " << filePath << fileName << msg->subject(true)->asUnicodeString(); - - Akonadi2::ApplicationDomain::Mail mail; - mail.setProperty("subject", msg->subject(true)->asUnicodeString()); - mail.setProperty("sender", msg->from(true)->asUnicodeString()); - mail.setProperty("senderName", msg->from(true)->asUnicodeString()); - mail.setProperty("date", msg->date(true)->dateTime().toString()); - mail.setProperty("folder", resolveRemoteId(ENTITY_TYPE_FOLDER, path, synchronizationTransaction)); - mail.setProperty("mimeMessage", filepath); - mail.setProperty("unread", !flags.testFlag(KPIM::Maildir::Seen)); - mail.setProperty("important", flags.testFlag(KPIM::Maildir::Flagged)); - - flatbuffers::FlatBufferBuilder entityFbb; - mMailAdaptorFactory->createBuffer(mail, entityFbb); - - Trace() << "Found a new entity: " << remoteId; - createEntity(akonadiId.toLatin1(), bufferType, mail, *mMailAdaptorFactory, [this](const QByteArray &buffer) { - enqueueCommand(mSynchronizerQueue, Akonadi2::Commands::CreateEntityCommand, buffer); - }); - } else { //modification - Trace() << "Found a modified entity: " << remoteId; - //TODO diff and create modification if necessary - } + + KMime::Message *msg = new KMime::Message; + auto filepath = listingPath + QDir::separator() + fileName; + msg->setHead(KMime::CRLFtoLF(maildir.readEntryHeadersFromFile(filepath))); + msg->parse(); + + const auto flags = maildir.readEntryFlags(fileName); + + Trace() << "Found a mail " << filePath << fileName << msg->subject(true)->asUnicodeString(); + + Akonadi2::ApplicationDomain::Mail mail; + mail.setProperty("subject", msg->subject(true)->asUnicodeString()); + mail.setProperty("sender", msg->from(true)->asUnicodeString()); + mail.setProperty("senderName", msg->from(true)->asUnicodeString()); + mail.setProperty("date", msg->date(true)->dateTime().toString()); + mail.setProperty("folder", resolveRemoteId(ENTITY_TYPE_FOLDER, path, synchronizationTransaction)); + mail.setProperty("mimeMessage", filepath); + mail.setProperty("unread", !flags.testFlag(KPIM::Maildir::Seen)); + mail.setProperty("important", flags.testFlag(KPIM::Maildir::Flagged)); + + createOrModify(transaction, synchronizationTransaction, *mMailAdaptorFactory, bufferType, remoteId, mail); } - //TODO find items to remove } KAsync::Job MaildirResource::synchronizeWithSource() diff --git a/examples/maildirresource/maildirresource.h b/examples/maildirresource/maildirresource.h index e1eecc1..eec1e97 100644 --- a/examples/maildirresource/maildirresource.h +++ b/examples/maildirresource/maildirresource.h @@ -39,8 +39,36 @@ public: static void removeFromDisk(const QByteArray &instanceIdentifier); private: KAsync::Job replay(const QByteArray &type, const QByteArray &key, const QByteArray &value) Q_DECL_OVERRIDE; + + /** + * Tries to find a local id for the remote id, and creates a new local id otherwise. + * + * The new local id is recorded in the local to remote id mapping. + */ QString resolveRemoteId(const QByteArray &type, const QString &remoteId, Akonadi2::Storage::Transaction &transaction); + + /** + * Tries to find a remote id for a local id. + * + * This can fail if the entity hasn't been written back to the server yet. + */ QString resolveLocalId(const QByteArray &bufferType, const QByteArray &localId, Akonadi2::Storage::Transaction &transaction); + + /** + * An algorithm to remove entities that are no longer existing. + * + * This algorithm calls @param exists for every entity of type @param type, with its remoteId. For every entity where @param exists returns false, + * an entity delete command is enqueued. + */ + void scanForRemovals(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, const QByteArray &bufferType, std::function exists); + + /** + * An algorithm to create or modify the entity. + * + * Depending on whether the entity is locally available, or has changed. + */ + void createOrModify(Akonadi2::Storage::Transaction &transaction, Akonadi2::Storage::Transaction &synchronizationTransaction, DomainTypeAdaptorFactoryInterface &adaptorFactory, const QByteArray &bufferType, const QByteArray &remoteId, const Akonadi2::ApplicationDomain::ApplicationDomainType &entity); + void synchronizeFolders(Akonadi2::Storage::Transaction &transaction); void synchronizeMails(Akonadi2::Storage::Transaction &transaction, const QString &folder); QStringList listAvailableFolders(); -- cgit v1.2.3