From 237b9ae4113e7a9f489632296941becb71afdb45 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sun, 16 Oct 2016 14:55:20 +0200 Subject: Refactor how the storage is used. This is the initial refactoring to improve how we deal with the storage. It does a couple of things: * Rename Sink::Storage to Sink::Storage::DataStore to free up the Sink::Storage namespace * Introduce a Sink::ResourceContext to have a single object that can be passed around containing everything that is necessary to operate on a resource. This is a lot better than the multiple separate parameters that we used to pass around all over the place, while still allowing for dependency injection for tests. * Tie storage access together using the new EntityStore that directly works with ApplicationDomainTypes. This gives us a central place where main storage, indexes and buffer adaptors are tied together, which will also give us a place to implement external indexes, such as a fulltextindex using xapian. * Use ApplicationDomainTypes as the default way to pass around entities. Instead of using various ways to pass around entities (buffers, buffer adaptors, ApplicationDomainTypes), only use a single way. The old approach was confusing, and was only done as: * optimization; really shouldn't be necessary and otherwise I'm sure we can find better ways to optimize ApplicationDomainType itself. * a way to account for entities that have multiple buffers, a concept that I no longer deem relevant. While this commit does the bulk of the work to get there, the following commits will refactor more stuff to get things back to normal. --- examples/maildirresource/facade.cpp | 11 ++-- examples/maildirresource/facade.h | 4 +- examples/maildirresource/maildirresource.cpp | 77 +++++++++++----------------- examples/maildirresource/maildirresource.h | 4 +- 4 files changed, 38 insertions(+), 58 deletions(-) (limited to 'examples/maildirresource') diff --git a/examples/maildirresource/facade.cpp b/examples/maildirresource/facade.cpp index 256b255..ba53c5f 100644 --- a/examples/maildirresource/facade.cpp +++ b/examples/maildirresource/facade.cpp @@ -22,11 +22,10 @@ #include #include -#include "domainadaptor.h" -#include "queryrunner.h" +#include "query.h" -MaildirResourceMailFacade::MaildirResourceMailFacade(const QByteArray &instanceIdentifier) - : Sink::GenericFacade(instanceIdentifier, QSharedPointer::create()) +MaildirResourceMailFacade::MaildirResourceMailFacade(const Sink::ResourceContext &context) + : Sink::GenericFacade(context) { mResultTransformation = [](Sink::ApplicationDomain::ApplicationDomainType &value) { if (value.hasProperty("mimeMessage")) { @@ -62,8 +61,8 @@ QPair, Sink::ResultEmitter } -MaildirResourceFolderFacade::MaildirResourceFolderFacade(const QByteArray &instanceIdentifier) - : Sink::GenericFacade(instanceIdentifier, QSharedPointer::create()) +MaildirResourceFolderFacade::MaildirResourceFolderFacade(const Sink::ResourceContext &context) + : Sink::GenericFacade(context) { } diff --git a/examples/maildirresource/facade.h b/examples/maildirresource/facade.h index 38981d0..fdb693e 100644 --- a/examples/maildirresource/facade.h +++ b/examples/maildirresource/facade.h @@ -24,7 +24,7 @@ class MaildirResourceMailFacade : public Sink::GenericFacade { public: - MaildirResourceMailFacade(const QByteArray &instanceIdentifier); + MaildirResourceMailFacade(const Sink::ResourceContext &context); virtual ~MaildirResourceMailFacade(); QPair, Sink::ResultEmitter::Ptr> load(const Sink::Query &query) Q_DECL_OVERRIDE; }; @@ -32,6 +32,6 @@ public: class MaildirResourceFolderFacade : public Sink::GenericFacade { public: - MaildirResourceFolderFacade(const QByteArray &instanceIdentifier); + MaildirResourceFolderFacade(const Sink::ResourceContext &context); virtual ~MaildirResourceFolderFacade(); }; diff --git a/examples/maildirresource/maildirresource.cpp b/examples/maildirresource/maildirresource.cpp index 1ed7fc8..b89d78c 100644 --- a/examples/maildirresource/maildirresource.cpp +++ b/examples/maildirresource/maildirresource.cpp @@ -85,13 +85,13 @@ class MaildirMimeMessageMover : public Sink::Preprocessor public: MaildirMimeMessageMover(const QByteArray &resourceInstanceIdentifier, const QString &maildirPath) : mResourceInstanceIdentifier(resourceInstanceIdentifier), mMaildirPath(maildirPath) {} - QString getPath(const QByteArray &folderIdentifier, Sink::Storage::Transaction &transaction) + QString getPath(const QByteArray &folderIdentifier, Sink::Storage::DataStore::Transaction &transaction) { if (folderIdentifier.isEmpty()) { return mMaildirPath; } QString folderPath; - auto db = Sink::Storage::mainDatabase(transaction, ENTITY_TYPE_FOLDER); + auto db = Sink::Storage::DataStore::mainDatabase(transaction, ENTITY_TYPE_FOLDER); db.findLatest(folderIdentifier, [&](const QByteArray &, const QByteArray &value) { Sink::EntityBuffer buffer(value); const Sink::Entity &entity = buffer.entity(); @@ -108,7 +108,7 @@ public: return folderPath; } - QString moveMessage(const QString &oldPath, const QByteArray &folder, Sink::Storage::Transaction &transaction) + QString moveMessage(const QString &oldPath, const QByteArray &folder, Sink::Storage::DataStore::Transaction &transaction) { if (oldPath.startsWith(Sink::temporaryFileLocation())) { const auto path = getPath(folder, transaction); @@ -141,7 +141,7 @@ public: } } - void newEntity(const QByteArray &uid, qint64 revision, Sink::ApplicationDomain::BufferAdaptor &newEntity, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE + void newEntity(const QByteArray &uid, qint64 revision, Sink::ApplicationDomain::BufferAdaptor &newEntity, Sink::Storage::DataStore::Transaction &transaction) Q_DECL_OVERRIDE { const auto mimeMessage = newEntity.getProperty("mimeMessage"); if (mimeMessage.isValid()) { @@ -150,7 +150,7 @@ public: } void modifiedEntity(const QByteArray &uid, qint64 revision, const Sink::ApplicationDomain::BufferAdaptor &oldEntity, Sink::ApplicationDomain::BufferAdaptor &newEntity, - Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE + Sink::Storage::DataStore::Transaction &transaction) Q_DECL_OVERRIDE { const auto mimeMessage = newEntity.getProperty("mimeMessage"); const auto newFolder = newEntity.getProperty("folder"); @@ -185,7 +185,7 @@ public: maildir.changeEntryFlags(identifier, flags); } - void deletedEntity(const QByteArray &uid, qint64 revision, const Sink::ApplicationDomain::BufferAdaptor &oldEntity, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE + void deletedEntity(const QByteArray &uid, qint64 revision, const Sink::ApplicationDomain::BufferAdaptor &oldEntity, Sink::Storage::DataStore::Transaction &transaction) Q_DECL_OVERRIDE { const auto filePath = getFilePathFromMimeMessagePath(oldEntity.getProperty("mimeMessage").toString()); QFile::remove(filePath); @@ -199,7 +199,7 @@ class FolderPreprocessor : public Sink::Preprocessor public: FolderPreprocessor(const QString maildirPath) : mMaildirPath(maildirPath) {} - void newEntity(const QByteArray &uid, qint64 revision, Sink::ApplicationDomain::BufferAdaptor &newEntity, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE + void newEntity(const QByteArray &uid, qint64 revision, Sink::ApplicationDomain::BufferAdaptor &newEntity, Sink::Storage::DataStore::Transaction &transaction) Q_DECL_OVERRIDE { auto folderName = newEntity.getProperty("name").toString(); const auto path = mMaildirPath + "/" + folderName; @@ -208,11 +208,11 @@ public: } void modifiedEntity(const QByteArray &uid, qint64 revision, const Sink::ApplicationDomain::BufferAdaptor &oldEntity, Sink::ApplicationDomain::BufferAdaptor &newEntity, - Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE + Sink::Storage::DataStore::Transaction &transaction) Q_DECL_OVERRIDE { } - void deletedEntity(const QByteArray &uid, qint64 revision, const Sink::ApplicationDomain::BufferAdaptor &oldEntity, Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE + void deletedEntity(const QByteArray &uid, qint64 revision, const Sink::ApplicationDomain::BufferAdaptor &oldEntity, Sink::Storage::DataStore::Transaction &transaction) Q_DECL_OVERRIDE { } QString mMaildirPath; @@ -221,8 +221,8 @@ public: class MaildirSynchronizer : public Sink::Synchronizer { public: - MaildirSynchronizer(const QByteArray &resourceType, const QByteArray &resourceInstanceIdentifier) - : Sink::Synchronizer(resourceType, resourceInstanceIdentifier) + MaildirSynchronizer(const Sink::ResourceContext &resourceContext) + : Sink::Synchronizer(resourceContext) { } @@ -278,19 +278,7 @@ public: const QByteArray bufferType = ENTITY_TYPE_FOLDER; QStringList folderList = listAvailableFolders(); SinkTrace() << "Found folders " << folderList; - scanForRemovals(bufferType, - [this, &bufferType](const std::function &callback) { - //TODO Instead of iterating over all entries in the database, which can also pick up the same item multiple times, - //we should rather iterate over an index that contains every uid exactly once. The remoteId index would be such an index, - //but we currently fail to iterate over all entries in an index it seems. - // auto remoteIds = synchronizationTransaction.openDatabase("rid.mapping." + bufferType, std::function(), true); - auto mainDatabase = Sink::Storage::mainDatabase(transaction(), bufferType); - mainDatabase.scan("", [&](const QByteArray &key, const QByteArray &) { - callback(key); - return true; - }); - }, [&folderList](const QByteArray &remoteId) -> bool { return folderList.contains(remoteId); } @@ -323,16 +311,9 @@ public: const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, path.toUtf8()); - auto property = "folder"; scanForRemovals(bufferType, [&](const std::function &callback) { - Index index(bufferType + ".index." + property, transaction()); - index.lookup(folderLocalId, [&](const QByteArray &sinkId) { - callback(sinkId); - }, - [&](const Index::Error &error) { - SinkWarning() << "Error in index: " << error.message << property; - }); + store().indexLookup(folderLocalId, callback); }, [](const QByteArray &remoteId) -> bool { return QFile(remoteId).exists(); @@ -392,7 +373,7 @@ public: class MaildirWriteback : public Sink::SourceWriteBack { public: - MaildirWriteback(const QByteArray &resourceType, const QByteArray &resourceInstanceIdentifier) : Sink::SourceWriteBack(resourceType, resourceInstanceIdentifier) + MaildirWriteback(const Sink::ResourceContext &resourceContext) : Sink::SourceWriteBack(resourceContext) { } @@ -442,24 +423,24 @@ public: }; -MaildirResource::MaildirResource(const QByteArray &instanceIdentifier, const QSharedPointer &pipeline) - : Sink::GenericResource(PLUGIN_NAME, instanceIdentifier, pipeline) +MaildirResource::MaildirResource(const Sink::ResourceContext &resourceContext, const QSharedPointer &pipeline) + : Sink::GenericResource(resourceContext, pipeline) { - auto config = ResourceConfig::getConfiguration(instanceIdentifier); + auto config = ResourceConfig::getConfiguration(resourceContext.instanceId()); mMaildirPath = QDir::cleanPath(QDir::fromNativeSeparators(config.value("path").toString())); //Chop a trailing slash if necessary if (mMaildirPath.endsWith("/")) { mMaildirPath.chop(1); } - auto synchronizer = QSharedPointer::create(PLUGIN_NAME, instanceIdentifier); + auto synchronizer = QSharedPointer::create(resourceContext); synchronizer->mMaildirPath = mMaildirPath; setupSynchronizer(synchronizer); - auto changereplay = QSharedPointer::create(PLUGIN_NAME, instanceIdentifier); + auto changereplay = QSharedPointer::create(resourceContext); changereplay->mMaildirPath = mMaildirPath; setupChangereplay(changereplay); - setupPreprocessors(ENTITY_TYPE_MAIL, QVector() << new SpecialPurposeProcessor(mResourceType, mResourceInstanceIdentifier) << new MaildirMimeMessageMover(mResourceInstanceIdentifier, mMaildirPath) << new MaildirMailPropertyExtractor << new DefaultIndexUpdater); + setupPreprocessors(ENTITY_TYPE_MAIL, QVector() << new SpecialPurposeProcessor(resourceContext.resourceType, resourceContext.instanceId()) << new MaildirMimeMessageMover(resourceContext.instanceId(), mMaildirPath) << new MaildirMailPropertyExtractor << new DefaultIndexUpdater); setupPreprocessors(ENTITY_TYPE_FOLDER, QVector() << new FolderPreprocessor(mMaildirPath) << new DefaultIndexUpdater); KPIM::Maildir dir(mMaildirPath, true); @@ -480,24 +461,24 @@ MaildirResource::MaildirResource(const QByteArray &instanceIdentifier, const QSh void MaildirResource::removeFromDisk(const QByteArray &instanceIdentifier) { GenericResource::removeFromDisk(instanceIdentifier); - Sink::Storage(Sink::storageLocation(), instanceIdentifier + ".synchronization", Sink::Storage::ReadWrite).removeFromDisk(); + Sink::Storage::DataStore(Sink::storageLocation(), instanceIdentifier + ".synchronization", Sink::Storage::DataStore::ReadWrite).removeFromDisk(); } KAsync::Job MaildirResource::inspect(int inspectionType, const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) { - auto synchronizationStore = QSharedPointer::create(Sink::storageLocation(), mResourceInstanceIdentifier + ".synchronization", Sink::Storage::ReadOnly); - auto synchronizationTransaction = synchronizationStore->createTransaction(Sink::Storage::ReadOnly); + auto synchronizationStore = QSharedPointer::create(Sink::storageLocation(), mResourceContext.instanceId() + ".synchronization", Sink::Storage::DataStore::ReadOnly); + auto synchronizationTransaction = synchronizationStore->createTransaction(Sink::Storage::DataStore::ReadOnly); - auto mainStore = QSharedPointer::create(Sink::storageLocation(), mResourceInstanceIdentifier, Sink::Storage::ReadOnly); - auto transaction = mainStore->createTransaction(Sink::Storage::ReadOnly); + auto mainStore = QSharedPointer::create(Sink::storageLocation(), mResourceContext.instanceId(), Sink::Storage::DataStore::ReadOnly); + auto transaction = mainStore->createTransaction(Sink::Storage::DataStore::ReadOnly); - auto entityStore = QSharedPointer::create(mResourceType, mResourceInstanceIdentifier, transaction); + Sink::Storage::EntityStore entityStore(mResourceContext); auto syncStore = QSharedPointer::create(synchronizationTransaction); SinkTrace() << "Inspecting " << inspectionType << domainType << entityId << property << expectedValue; if (domainType == ENTITY_TYPE_MAIL) { - auto mail = entityStore->read(entityId); + auto mail = entityStore.readLatest(entityId); const auto filePath = getFilePathFromMimeMessagePath(mail.getMimeMessagePath()); if (inspectionType == Sink::ResourceControl::Inspection::PropertyInspectionType) { @@ -530,7 +511,7 @@ KAsync::Job MaildirResource::inspect(int inspectionType, const QByteArray } if (domainType == ENTITY_TYPE_FOLDER) { const auto remoteId = syncStore->resolveLocalId(ENTITY_TYPE_FOLDER, entityId); - auto folder = entityStore->read(entityId); + auto folder = entityStore.readLatest(entityId); if (inspectionType == Sink::ResourceControl::Inspection::CacheIntegrityInspectionType) { SinkTrace() << "Inspecting cache integrity" << remoteId; @@ -577,9 +558,9 @@ MaildirResourceFactory::MaildirResourceFactory(QObject *parent) } -Sink::Resource *MaildirResourceFactory::createResource(const QByteArray &instanceIdentifier) +Sink::Resource *MaildirResourceFactory::createResource(const ResourceContext &context) { - return new MaildirResource(instanceIdentifier); + return new MaildirResource(context); } void MaildirResourceFactory::registerFacades(Sink::FacadeFactory &factory) diff --git a/examples/maildirresource/maildirresource.h b/examples/maildirresource/maildirresource.h index 490e1e6..6265819 100644 --- a/examples/maildirresource/maildirresource.h +++ b/examples/maildirresource/maildirresource.h @@ -45,7 +45,7 @@ class MaildirFolderAdaptorFactory; class MaildirResource : public Sink::GenericResource { public: - MaildirResource(const QByteArray &instanceIdentifier, const QSharedPointer &pipeline = QSharedPointer()); + MaildirResource(const Sink::ResourceContext &resourceContext, const QSharedPointer &pipeline = QSharedPointer()); KAsync::Job inspect(int inspectionType, const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) Q_DECL_OVERRIDE; static void removeFromDisk(const QByteArray &instanceIdentifier); private: @@ -64,7 +64,7 @@ class MaildirResourceFactory : public Sink::ResourceFactory public: MaildirResourceFactory(QObject *parent = 0); - Sink::Resource *createResource(const QByteArray &instanceIdentifier) Q_DECL_OVERRIDE; + Sink::Resource *createResource(const Sink::ResourceContext &context) Q_DECL_OVERRIDE; void registerFacades(Sink::FacadeFactory &factory) Q_DECL_OVERRIDE; void registerAdaptorFactories(Sink::AdaptorFactoryRegistry ®istry) Q_DECL_OVERRIDE; void removeDataFromDisk(const QByteArray &instanceIdentifier) Q_DECL_OVERRIDE; -- cgit v1.2.3