From d6a01b3f82d626856001356c0875aa738a0346ac Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 4 Oct 2016 08:25:18 +0200 Subject: Support for subqueries. This allows us to match properties from a subquery. Unfortunately this also means that DataStoreQuery needs access to all type implementations to issue the subquery (for potentially another type). --- common/datastorequery.cpp | 45 +++++++++++++++++++++++++++++++++-- common/datastorequery.h | 4 ++++ common/domain/applicationdomaintype.h | 1 + common/query.cpp | 5 ++++ common/query.h | 13 +++++++++- common/queryrunner.cpp | 2 +- common/queryrunner.h | 3 +++ common/typeindex.cpp | 34 +++++++++++++++++++------- common/typeindex.h | 2 ++ tests/querytest.cpp | 36 ++++++++++++++++++++++++++++ 10 files changed, 132 insertions(+), 13 deletions(-) diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp index 3c4ae00..c4fbe13 100644 --- a/common/datastorequery.cpp +++ b/common/datastorequery.cpp @@ -21,6 +21,11 @@ #include "log.h" #include "entitybuffer.h" #include "entity_generated.h" +#include "applicationdomaintype.h" + +#include "folder.h" +#include "mail.h" +#include "event.h" using namespace Sink; @@ -373,8 +378,44 @@ QVector DataStoreQuery::indexLookup(const QByteArray &property, cons /* } */ /* } */ +template +QSharedPointer prepareQuery(const QByteArray &type, Args && ... args) +{ + if (type == ApplicationDomain::getTypeName()) { + return ApplicationDomain::TypeImplementation::prepareQuery(std::forward(args)...); + } else if (type == ApplicationDomain::getTypeName()) { + return ApplicationDomain::TypeImplementation::prepareQuery(std::forward(args)...); + } else if (type == ApplicationDomain::getTypeName()) { + return ApplicationDomain::TypeImplementation::prepareQuery(std::forward(args)...); + } + Q_ASSERT(false); + return QSharedPointer(); +} + +QByteArrayList DataStoreQuery::executeSubquery(const Query &subquery) +{ + Q_ASSERT(!subquery.type.isEmpty()); + auto sub = prepareQuery(subquery.type, subquery, mTransaction); + auto result = sub->execute(); + QByteArrayList ids; + while (result.next([&ids](const QByteArray &uid, const Sink::EntityBuffer &, Sink::Operation) { + ids << uid; + })) + {} + return ids; +} + void DataStoreQuery::setupQuery() { + for (const auto &k : mQuery.propertyFilter.keys()) { + const auto comparator = mQuery.propertyFilter.value(k); + if (comparator.value.canConvert()) { + SinkTrace() << "Executing subquery for property: " << k; + const auto result = executeSubquery(comparator.value.value()); + mQuery.propertyFilter.insert(k, Query::Comparator(QVariant::fromValue(result), Query::Comparator::In)); + } + } + FilterBase::Ptr baseSet; QSet remainingFilters = mQuery.getBaseFilters().keys().toSet(); QByteArray appliedSorting; @@ -459,7 +500,7 @@ ResultSet DataStoreQuery::update(qint64 baseRevision) SinkTrace() << "Changed: " << incrementalResultSet; mSource->add(incrementalResultSet); ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool { - if (mCollector->next([callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) { + if (mCollector->next([this, callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) { SinkTrace() << "Got incremental result: " << uid << operation; callback(uid, buffer, operation); })) @@ -477,7 +518,7 @@ ResultSet DataStoreQuery::execute() SinkTrace() << "Executing query"; ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool { - if (mCollector->next([callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) { + if (mCollector->next([this, callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) { if (operation != Sink::Operation_Removal) { SinkTrace() << "Got initial result: " << uid << operation; callback(uid, buffer, Sink::Operation_Creation); diff --git a/common/datastorequery.h b/common/datastorequery.h index 6620bbe..03b4eac 100644 --- a/common/datastorequery.h +++ b/common/datastorequery.h @@ -24,6 +24,7 @@ #include "typeindex.h" #include "query.h" #include "entitybuffer.h" +#include "log.h" class Source; @@ -52,6 +53,7 @@ protected: QVector loadIncrementalResultSet(qint64 baseRevision); void setupQuery(); + QByteArrayList executeSubquery(const Sink::Query &subquery); Sink::Query mQuery; Sink::Storage::Transaction &mTransaction; @@ -62,6 +64,8 @@ protected: bool mInitialQuery; QSharedPointer mCollector; QSharedPointer mSource; + + SINK_DEBUG_COMPONENT(mType) }; diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h index 8b96758..2c93639 100644 --- a/common/domain/applicationdomaintype.h +++ b/common/domain/applicationdomaintype.h @@ -74,6 +74,7 @@ struct NAME { \ static constexpr const char *name = #LOWERCASENAME; \ typedef QByteArray Type; \ + typedef ApplicationDomain::TYPE ReferenceType; \ }; \ void set##NAME(const ApplicationDomain::TYPE &value) { setProperty(NAME::name, value); } \ void set##NAME(const QByteArray &value) { setProperty(NAME::name, QVariant::fromValue(value)); } \ diff --git a/common/query.cpp b/common/query.cpp index 3de80d8..a43ef6c 100644 --- a/common/query.cpp +++ b/common/query.cpp @@ -62,6 +62,11 @@ bool Query::Comparator::matches(const QVariant &v) const return false; } return v.value().contains(value.toByteArray()); + case In: + if (!v.isValid()) { + return false; + } + return value.value().contains(v.toByteArray()); case Invalid: default: break; diff --git a/common/query.h b/common/query.h index 3362ac7..403c5b5 100644 --- a/common/query.h +++ b/common/query.h @@ -46,7 +46,8 @@ public: enum Comparators { Invalid, Equals, - Contains + Contains, + In }; Comparator(); @@ -158,6 +159,7 @@ public: QByteArrayList requestedProperties; QByteArray parentProperty; QByteArray sortProperty; + QByteArray type; int limit; bool liveQuery; bool synchronousQuery; @@ -222,6 +224,14 @@ public: return filter(T::name, QVariant::fromValue(value.identifier())); } + template + Query &filter(const Query &query) + { + auto q = query; + q.type = ApplicationDomain::getTypeName(); + return filter(T::name, QVariant::fromValue(q)); + } + Query &filter(const ApplicationDomain::SinkResource &resource) { resources << resource.identifier(); @@ -353,3 +363,4 @@ public: QDebug operator<<(QDebug dbg, const Sink::Query::Comparator &c); Q_DECLARE_OPERATORS_FOR_FLAGS(Sink::Query::Flags) +Q_DECLARE_METATYPE(Sink::Query); diff --git a/common/queryrunner.cpp b/common/queryrunner.cpp index d3f8f66..1835e1f 100644 --- a/common/queryrunner.cpp +++ b/common/queryrunner.cpp @@ -66,7 +66,7 @@ private: template QueryRunner::QueryRunner(const Sink::Query &query, const Sink::ResourceAccessInterface::Ptr &resourceAccess, const QByteArray &instanceIdentifier, const DomainTypeAdaptorFactoryInterface::Ptr &factory, const QByteArray &bufferType) - : QueryRunnerBase(), mResourceAccess(resourceAccess), mResultProvider(new ResultProvider), mBatchSize(query.limit) + : QueryRunnerBase(), mResourceInstanceIdentifier(instanceIdentifier), mResourceAccess(resourceAccess), mResultProvider(new ResultProvider), mBatchSize(query.limit) { SinkTrace() << "Starting query"; if (query.limit && query.sortProperty.isEmpty()) { diff --git a/common/queryrunner.h b/common/queryrunner.h index 439a990..78aabf6 100644 --- a/common/queryrunner.h +++ b/common/queryrunner.h @@ -25,6 +25,7 @@ #include "domaintypeadaptorfactoryinterface.h" #include "storage.h" #include "query.h" +#include "log.h" /** * Base clase because you can't have the Q_OBJECT macro in template classes @@ -96,6 +97,8 @@ public: typename Sink::ResultEmitter::Ptr emitter(); private: + QByteArray mResourceInstanceIdentifier; + SINK_DEBUG_COMPONENT(mResourceInstanceIdentifier) QSharedPointer mResourceAccess; QSharedPointer> mResultProvider; ResultTransformation mResultTransformation; diff --git a/common/typeindex.cpp b/common/typeindex.cpp index 272237c..816e7ee 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -24,6 +24,8 @@ SINK_DEBUG_AREA("typeindex") +using namespace Sink; + static QByteArray getByteArray(const QVariant &value) { if (value.type() == QVariant::DateTime) { @@ -138,16 +140,32 @@ void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDoma } } +static QVector indexLookup(Index &index, Query::Comparator filter) +{ + QVector keys; + QByteArrayList lookupKeys; + if (filter.comparator == Query::Comparator::Equals) { + lookupKeys << getByteArray(filter.value); + } else if (filter.comparator == Query::Comparator::In) { + lookupKeys = filter.value.value(); + } else { + Q_ASSERT(false); + } + + for (const auto &lookupKey : lookupKeys) { + index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; }, + [lookupKey](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << lookupKey; }, true); + } + return keys; +} + QVector TypeIndex::query(const Sink::Query &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction) { QVector keys; for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { if (query.hasFilter(it.key()) && query.sortProperty == it.value()) { Index index(indexName(it.key(), it.value()), transaction); - const auto lookupKey = getByteArray(query.getFilter(it.key()).value); - SinkTrace() << "looking for " << lookupKey; - index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; }, - [it](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << it.key() << it.value(); }, true); + keys << indexLookup(index, query.getFilter(it.key())); appliedFilters << it.key(); appliedSorting = it.value(); SinkTrace() << "Index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys."; @@ -157,9 +175,7 @@ QVector TypeIndex::query(const Sink::Query &query, QSet for (const auto &property : mProperties) { if (query.hasFilter(property)) { Index index(indexName(property), transaction); - const auto lookupKey = getByteArray(query.getFilter(property).value); - index.lookup( - lookupKey, [&](const QByteArray &value) { keys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); + keys << indexLookup(index, query.getFilter(property)); appliedFilters << property; SinkTrace() << "Index lookup on " << property << " found " << keys.size() << " keys."; return keys; @@ -177,7 +193,7 @@ QVector TypeIndex::lookup(const QByteArray &property, const QVariant Index index(indexName(property), transaction); const auto lookupKey = getByteArray(value); index.lookup( - lookupKey, [&](const QByteArray &value) { keys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); + lookupKey, [&, this](const QByteArray &value) { keys << value; }, [property, this](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); SinkTrace() << "Index lookup on " << property << " found " << keys.size() << " keys."; return keys; } else if (mSecondaryProperties.contains(property)) { @@ -189,7 +205,7 @@ QVector TypeIndex::lookup(const QByteArray &property, const QVariant Index index(indexName(property + resultProperty), transaction); const auto lookupKey = getByteArray(value); index.lookup( - lookupKey, [&](const QByteArray &value) { secondaryKeys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); + lookupKey, [&, this](const QByteArray &value) { secondaryKeys << value; }, [property, this](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); SinkTrace() << "Looked up secondary keys: " << secondaryKeys; for (const auto &secondary : secondaryKeys) { keys += lookup(resultProperty, secondary, transaction); diff --git a/common/typeindex.h b/common/typeindex.h index 7266f02..4972e95 100644 --- a/common/typeindex.h +++ b/common/typeindex.h @@ -22,6 +22,7 @@ #include "bufferadaptor.h" #include "storage.h" #include "query.h" +#include "log.h" #include class TypeIndex @@ -79,6 +80,7 @@ public: private: QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const; QByteArray mType; + SINK_DEBUG_COMPONENT(mType) QByteArrayList mProperties; QMap mSortedProperties; // diff --git a/tests/querytest.cpp b/tests/querytest.cpp index afa8e33..6011a99 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -492,6 +492,42 @@ private slots: QCOMPARE(folders.size(), 1); } + void testSubquery() + { + // Setup + auto folder1 = Folder::createEntity("sink.dummy.instance1"); + folder1.setSpecialPurpose(QByteArrayList() << "purpose1"); + VERIFYEXEC(Sink::Store::create(folder1)); + + auto folder2 = Folder::createEntity("sink.dummy.instance1"); + folder2.setSpecialPurpose(QByteArrayList() << "purpose2"); + VERIFYEXEC(Sink::Store::create(folder2)); + + { + auto mail = Mail::createEntity("sink.dummy.instance1"); + mail.setUid("mail1"); + mail.setFolder(folder1); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + auto mail = Mail::createEntity("sink.dummy.instance1"); + mail.setUid("mail2"); + mail.setFolder(folder2); + VERIFYEXEC(Sink::Store::create(mail)); + } + + // Ensure all local data is processed + Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1").exec().waitForFinished(); + + //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. + Query query; + query.filter(Sink::Query().containsFilter("purpose1")); + query.request(); + + auto mails = Sink::Store::read(query); + QCOMPARE(mails.size(), 1); + QCOMPARE(mails.first().getUid().toLatin1(), QByteArray("mail1")); + } }; QTEST_MAIN(QueryTest) -- cgit v1.2.3