From 9b744da32e64d8a6cd342faba8fc3232884d60f2 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 24 Jan 2015 23:38:12 +0100 Subject: Uid index + query using that index. ...and a couple of other fixes. --- dummyresource/domainadaptor.cpp | 26 ++++--- dummyresource/facade.cpp | 145 ++++++++++++++++++++++++-------------- dummyresource/facade.h | 1 + dummyresource/resourcefactory.cpp | 26 ++++++- 4 files changed, 138 insertions(+), 60 deletions(-) (limited to 'dummyresource') diff --git a/dummyresource/domainadaptor.cpp b/dummyresource/domainadaptor.cpp index ec5e2be..d902052 100644 --- a/dummyresource/domainadaptor.cpp +++ b/dummyresource/domainadaptor.cpp @@ -40,9 +40,10 @@ public: { if (mResourceBuffer && mResourceMapper->mReadAccessors.contains(key)) { return mResourceMapper->getProperty(key, mResourceBuffer); - } else if (mLocalBuffer) { + } else if (mLocalBuffer && mLocalMapper->mReadAccessors.contains(key)) { return mLocalMapper->getProperty(key, mLocalBuffer); } + qWarning() << "no mapping available for key " << key; return QVariant(); } @@ -67,14 +68,23 @@ DummyEventAdaptorFactory::DummyEventAdaptorFactory() { mResourceMapper = QSharedPointer >::create(); mResourceMapper->mReadAccessors.insert("summary", [](DummyEvent const *buffer) -> QVariant { - return QString::fromStdString(buffer->summary()->c_str()); + if (buffer->summary()) { + return QString::fromStdString(buffer->summary()->c_str()); + } + return QVariant(); }); mLocalMapper = QSharedPointer >::create(); mLocalMapper->mReadAccessors.insert("summary", [](Akonadi2::Domain::Buffer::Event const *buffer) -> QVariant { - return QString::fromStdString(buffer->summary()->c_str()); + if (buffer->summary()) { + return QString::fromStdString(buffer->summary()->c_str()); + } + return QVariant(); }); mLocalMapper->mReadAccessors.insert("uid", [](Akonadi2::Domain::Buffer::Event const *buffer) -> QVariant { - return QString::fromStdString(buffer->uid()->c_str()); + if (buffer->uid()) { + return QString::fromStdString(buffer->uid()->c_str()); + } + return QVariant(); }); } @@ -94,7 +104,7 @@ QSharedPointer DummyEventAdaptorFactory::create if (auto metadataData = entity.metadata()) { flatbuffers::Verifier verifyer(metadataData->Data(), metadataData->size()); if (Akonadi2::VerifyMetadataBuffer(verifyer)) { - metadataBuffer = Akonadi2::GetMetadata(metadataData); + metadataBuffer = Akonadi2::GetMetadata(metadataData->Data()); } } @@ -102,15 +112,15 @@ QSharedPointer DummyEventAdaptorFactory::create if (auto localData = entity.local()) { flatbuffers::Verifier verifyer(localData->Data(), localData->size()); if (Akonadi2::Domain::Buffer::VerifyEventBuffer(verifyer)) { - localBuffer = Akonadi2::Domain::Buffer::GetEvent(localData); + localBuffer = Akonadi2::Domain::Buffer::GetEvent(localData->Data()); } } auto adaptor = QSharedPointer::create(); adaptor->mLocalBuffer = localBuffer; + adaptor->mLocalMapper = mLocalMapper; adaptor->mResourceBuffer = resourceBuffer; adaptor->mResourceMapper = mResourceMapper; - adaptor->mLocalMapper = mLocalMapper; return adaptor; } @@ -135,6 +145,6 @@ void DummyEventAdaptorFactory::createBuffer(const Akonadi2::Domain::Event &event Akonadi2::Domain::Buffer::FinishEventBuffer(localFbb, location); } - Akonadi2::EntityBuffer::assembleEntityBuffer(fbb, localFbb.GetBufferPointer(), localFbb.GetSize(), eventFbb.GetBufferPointer(), eventFbb.GetSize(), 0, 0); + Akonadi2::EntityBuffer::assembleEntityBuffer(fbb, 0, 0, eventFbb.GetBufferPointer(), eventFbb.GetSize(), localFbb.GetBufferPointer(), localFbb.GetSize()); } diff --git a/dummyresource/facade.cpp b/dummyresource/facade.cpp index cd930f6..1aaec68 100644 --- a/dummyresource/facade.cpp +++ b/dummyresource/facade.cpp @@ -31,6 +31,7 @@ #include "createentity_generated.h" #include "domainadaptor.h" #include +#include using namespace DummyCalendar; using namespace flatbuffers; @@ -73,11 +74,11 @@ Async::Job DummyResourceFacade::remove(const Akonadi2::Domain::Event &doma return Async::null(); } -static std::function prepareQuery(const Akonadi2::Query &query) +static std::function prepareQuery(const Akonadi2::Query &query) { //Compose some functions to make query matching fast. //This way we can process the query once, and convert all values into something that can be compared quickly - std::function preparedQuery; + std::function preparedQuery; if (!query.ids.isEmpty()) { //Match by id //TODO: for id's a direct lookup would be way faster @@ -88,15 +89,26 @@ static std::function pre for (const auto &id : query.ids) { ids << id.toStdString(); } - preparedQuery = [ids](const std::string &key, DummyEvent const *buffer) { + preparedQuery = [ids](const std::string &key, DummyEvent const *buffer, Akonadi2::Domain::Buffer::Event const *local) { if (ids.contains(key)) { return true; } return false; }; + } else if (!query.propertyFilter.isEmpty()) { + if (query.propertyFilter.contains("uid")) { + const QByteArray uid = query.propertyFilter.value("uid").toByteArray(); + preparedQuery = [uid](const std::string &key, DummyEvent const *buffer, Akonadi2::Domain::Buffer::Event const *local) { + if (local && local->uid() && (QByteArray::fromRawData(local->uid()->c_str(), local->uid()->size()) == uid)) { + qDebug() << "uid match"; + return true; + } + return false; + }; + } } else { //Match everything - preparedQuery = [](const std::string &key, DummyEvent const *buffer) { + preparedQuery = [](const std::string &key, DummyEvent const *buffer, Akonadi2::Domain::Buffer::Event const *local) { return true; }; } @@ -121,65 +133,96 @@ Async::Job DummyResourceFacade::synchronizeResource(bool sync) return Async::null(); } -Async::Job DummyResourceFacade::load(const Akonadi2::Query &query, const std::function &resultCallback) +void DummyResourceFacade::readValue(QSharedPointer storage, const QByteArray &key, const std::function &resultCallback, std::function preparedQuery) { - qDebug() << "load called"; + storage->scan(key.data(), key.size(), [=](void *keyValue, int keySize, void *dataValue, int dataSize) -> bool { - return synchronizeResource(query.syncOnDemand).then([=](Async::Future &future) { - qDebug() << "sync complete"; - //Now that the sync is complete we can execute the query - const auto preparedQuery = prepareQuery(query); - - auto storage = QSharedPointer::create(Akonadi2::Store::storageLocation(), "org.kde.dummy"); + //Skip internals + if (QByteArray::fromRawData(static_cast(keyValue), keySize).startsWith("__internal")) { + return true; + } - qDebug() << "executing query"; - //We start a transaction explicitly that we'll leave open so the values can be read. - //The transaction will be closed automatically once the storage object is destroyed. - storage->startTransaction(Akonadi2::Storage::ReadOnly); - //Because we have no indexes yet, we always do a full scan - storage->scan("", [=](void *keyValue, int keySize, void *dataValue, int dataSize) -> bool { + //Extract buffers + Akonadi2::EntityBuffer buffer(dataValue, dataSize); - //Skip internals - if (QByteArray::fromRawData(static_cast(keyValue), keySize).startsWith("__internal")) { - return true; + DummyEvent const *resourceBuffer = 0; + if (auto resourceData = buffer.entity().resource()) { + flatbuffers::Verifier verifyer(resourceData->Data(), resourceData->size()); + if (VerifyDummyEventBuffer(verifyer)) { + resourceBuffer = GetDummyEvent(resourceData->Data()); } + } - //Extract buffers - Akonadi2::EntityBuffer buffer(dataValue, dataSize); - - DummyEvent const *resourceBuffer = 0; - if (auto resourceData = buffer.entity().resource()) { - flatbuffers::Verifier verifyer(resourceData->Data(), resourceData->size()); - if (VerifyDummyEventBuffer(verifyer)) { - resourceBuffer = GetDummyEvent(resourceData); - } + Akonadi2::Domain::Buffer::Event const *localBuffer = 0; + if (auto localData = buffer.entity().local()) { + flatbuffers::Verifier verifyer(localData->Data(), localData->size()); + if (Akonadi2::Domain::Buffer::VerifyEventBuffer(verifyer)) { + localBuffer = Akonadi2::Domain::Buffer::GetEvent(localData->Data()); } + } - Akonadi2::Metadata const *metadataBuffer = 0; - if (auto metadataData = buffer.entity().metadata()) { - flatbuffers::Verifier verifyer(metadataData->Data(), metadataData->size()); - if (Akonadi2::VerifyMetadataBuffer(verifyer)) { - metadataBuffer = Akonadi2::GetMetadata(metadataData); - } + Akonadi2::Metadata const *metadataBuffer = 0; + if (auto metadataData = buffer.entity().metadata()) { + flatbuffers::Verifier verifyer(metadataData->Data(), metadataData->size()); + if (Akonadi2::VerifyMetadataBuffer(verifyer)) { + metadataBuffer = Akonadi2::GetMetadata(metadataData->Data()); } + } - if (!resourceBuffer || !metadataBuffer) { - qWarning() << "invalid buffer " << QString::fromStdString(std::string(static_cast(keyValue), keySize)); - return true; - } + if (!resourceBuffer || !metadataBuffer) { + qWarning() << "invalid buffer " << QString::fromStdString(std::string(static_cast(keyValue), keySize)); + return true; + } + + //We probably only want to create all buffers after the scan + //TODO use adapter for query and scan? + if (preparedQuery && preparedQuery(std::string(static_cast(keyValue), keySize), resourceBuffer, localBuffer)) { + qint64 revision = metadataBuffer ? metadataBuffer->revision() : -1; + //This only works for a 1:1 mapping of resource to domain types. + //Not i.e. for tags that are stored as flags in each entity of an imap store. + auto adaptor = mFactory->createAdaptor(buffer.entity()); + //TODO only copy requested properties + auto memoryAdaptor = QSharedPointer::create(*adaptor); + auto event = QSharedPointer::create("org.kde.dummy", QString::fromUtf8(static_cast(keyValue), keySize), revision, memoryAdaptor); + resultCallback(event); + } + return true; + }, + [](const Akonadi2::Storage::Error &error) { + qDebug() << QString::fromStdString(error.message); + }); +} + +Async::Job DummyResourceFacade::load(const Akonadi2::Query &query, const std::function &resultCallback) +{ + return synchronizeResource(query.syncOnDemand).then([=](Async::Future &future) { + //Now that the sync is complete we can execute the query + const auto preparedQuery = prepareQuery(query); + + auto storage = QSharedPointer::create(Akonadi2::Store::storageLocation(), "org.kde.dummy"); - //We probably only want to create all buffers after the scan - //TODO use adapter for query and scan? - if (preparedQuery && preparedQuery(std::string(static_cast(keyValue), keySize), resourceBuffer)) { - qint64 revision = metadataBuffer ? metadataBuffer->revision() : -1; - auto adaptor = mFactory->createAdaptor(buffer.entity()); - //TODO only copy requested properties - auto memoryAdaptor = QSharedPointer::create(*adaptor); - auto event = QSharedPointer::create("org.kde.dummy", QString::fromUtf8(static_cast(keyValue), keySize), revision, memoryAdaptor); - resultCallback(event); + QVector keys; + if (query.propertyFilter.contains("uid")) { + static Index uidIndex(Akonadi2::Store::storageLocation(), "org.kde.dummy.index.uid", Akonadi2::Storage::ReadOnly); + uidIndex.lookup(query.propertyFilter.value("uid").toByteArray(), [&](const QByteArray &value) { + keys << value; + }, + [](const Index::Error &error) { + qWarning() << "Error in index: " << QString::fromStdString(error.message); + }); + } + + //We start a transaction explicitly that we'll leave open so the values can be read. + //The transaction will be closed automatically once the storage object is destroyed. + storage->startTransaction(Akonadi2::Storage::ReadOnly); + if (keys.isEmpty()) { + qDebug() << "full scan"; + readValue(storage, QByteArray(), resultCallback, preparedQuery); + } else { + for (const auto &key : keys) { + readValue(storage, key, resultCallback, preparedQuery); } - return true; - }); + } future.setFinished(); }); } diff --git a/dummyresource/facade.h b/dummyresource/facade.h index da0b1d6..c9c8047 100644 --- a/dummyresource/facade.h +++ b/dummyresource/facade.h @@ -43,6 +43,7 @@ public: virtual Async::Job load(const Akonadi2::Query &query, const std::function &resultCallback); private: + void readValue(QSharedPointer storage, const QByteArray &key, const std::function &resultCallback, std::function); Async::Job synchronizeResource(bool sync); QSharedPointer mResourceAccess; QSharedPointer > mFactory; diff --git a/dummyresource/resourcefactory.cpp b/dummyresource/resourcefactory.cpp index 60a9cf6..18083cb 100644 --- a/dummyresource/resourcefactory.cpp +++ b/dummyresource/resourcefactory.cpp @@ -28,6 +28,7 @@ #include "domainadaptor.h" #include "commands.h" #include "clientapi.h" +#include "index.h" #include #include @@ -261,8 +262,31 @@ void DummyResource::configurePipeline(Akonadi2::Pipeline *pipeline) auto adaptor = eventFactory->createAdaptor(entity); qDebug() << "Summary preprocessor: " << adaptor->getProperty("summary").toString(); }); + + auto uidIndexer = new SimpleProcessor("uidIndexer", [eventFactory](const Akonadi2::PipelineState &state, const Akonadi2::Entity &entity) { + static Index uidIndex(QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + "/akonadi2/storage", "org.kde.dummy.index.uid", Akonadi2::Storage::ReadWrite); + + auto adaptor = eventFactory->createAdaptor(entity); + const auto uid = adaptor->getProperty("uid"); + if (uid.isValid()) { + uidIndex.add(uid.toByteArray(), state.key()); + } + + //TODO would this be worthwhile for performance reasons? + // flatbuffers::Verifier verifyer(entity.local()->Data(), entity.local()->size()); + // if (!Akonadi2::Domain::Buffer::VerifyEventBuffer(verifyer)) { + // qWarning() << "invalid local buffer"; + // return; + // } + // auto localEvent = Akonadi2::Domain::Buffer::GetEvent(entity.local()->Data()); + // if (localEvent && localEvent->uid()) { + // qDebug() << "got uid: " << QByteArray::fromRawData(reinterpret_cast(localEvent->uid()->Data()), localEvent->uid()->size()); + // uidIndex.add(QByteArray::fromRawData(reinterpret_cast(localEvent->uid()->Data()), localEvent->uid()->size()), state.key()); + // } + }); + //event is the entitytype and not the domain type - pipeline->setPreprocessors("event", Akonadi2::Pipeline::NewPipeline, QVector() << eventIndexer); + pipeline->setPreprocessors("event", Akonadi2::Pipeline::NewPipeline, QVector() << eventIndexer << uidIndexer); mProcessor = new Processor(pipeline, QList() << &mUserQueue << &mSynchronizerQueue); QObject::connect(mProcessor, &Processor::error, [this](int errorCode, const QString &msg) { onProcessorError(errorCode, msg); }); } -- cgit v1.2.3