Support for subqueries.

This allows us to match properties from a subquery.
Unfortunately this also means that DataStoreQuery needs access to all
type implementations to issue the subquery (for potentially another type).

10 files changed, 132 insertions, 13 deletions

diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp
index 3c4ae00..c4fbe13 100644
--- a/common/datastorequery.cpp
+++ b/common/datastorequery.cpp
@@ -21,6 +21,11 @@
 #include "log.h"
 #include "entitybuffer.h"
 #include "entity_generated.h"
+#include "applicationdomaintype.h"
+
+#include "folder.h"
+#include "mail.h"
+#include "event.h"
 
 using namespace Sink;
 
@@ -373,8 +378,44 @@ QVector<QByteArray> DataStoreQuery::indexLookup(const QByteArray &property, cons
 /*     } */
 /* } */
 
+template <typename ... Args>
+QSharedPointer<DataStoreQuery> prepareQuery(const QByteArray &type, Args && ... args)
+{
+    if (type == ApplicationDomain::getTypeName<ApplicationDomain::Folder>()) {
+        return ApplicationDomain::TypeImplementation<ApplicationDomain::Folder>::prepareQuery(std::forward<Args>(args)...);
+    } else if (type == ApplicationDomain::getTypeName<ApplicationDomain::Mail>()) {
+        return ApplicationDomain::TypeImplementation<ApplicationDomain::Mail>::prepareQuery(std::forward<Args>(args)...);
+    } else if (type == ApplicationDomain::getTypeName<ApplicationDomain::Event>()) {
+        return ApplicationDomain::TypeImplementation<ApplicationDomain::Event>::prepareQuery(std::forward<Args>(args)...);
+    }
+    Q_ASSERT(false);
+    return QSharedPointer<DataStoreQuery>();
+}
+
+QByteArrayList DataStoreQuery::executeSubquery(const Query &subquery)
+{
+    Q_ASSERT(!subquery.type.isEmpty());
+    auto sub = prepareQuery(subquery.type, subquery, mTransaction);
+    auto result = sub->execute();
+    QByteArrayList ids;
+    while (result.next([&ids](const QByteArray &uid, const Sink::EntityBuffer &, Sink::Operation) {
+            ids << uid;
+        }))
+    {}
+    return ids;
+}
+
 void DataStoreQuery::setupQuery()
 {
+    for (const auto &k : mQuery.propertyFilter.keys()) {
+        const auto comparator = mQuery.propertyFilter.value(k);
+        if (comparator.value.canConvert<Query>()) {
+            SinkTrace() << "Executing subquery for property: " << k;
+            const auto result = executeSubquery(comparator.value.value<Query>());
+            mQuery.propertyFilter.insert(k, Query::Comparator(QVariant::fromValue(result), Query::Comparator::In));
+        }
+    }
+
     FilterBase::Ptr baseSet;
     QSet<QByteArray> remainingFilters = mQuery.getBaseFilters().keys().toSet();
     QByteArray appliedSorting;
@@ -459,7 +500,7 @@ ResultSet DataStoreQuery::update(qint64 baseRevision)
     SinkTrace() << "Changed: " << incrementalResultSet;
     mSource->add(incrementalResultSet);
     ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool {
-        if (mCollector->next([callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) {
+        if (mCollector->next([this, callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) {
                 SinkTrace() << "Got incremental result: " << uid << operation;
                 callback(uid, buffer, operation);
             }))
@@ -477,7 +518,7 @@ ResultSet DataStoreQuery::execute()
     SinkTrace() << "Executing query";
 
     ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool {
-        if (mCollector->next([callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) {
+        if (mCollector->next([this, callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) {
                 if (operation != Sink::Operation_Removal) {
                     SinkTrace() << "Got initial result: " << uid << operation;
                     callback(uid, buffer, Sink::Operation_Creation);
diff --git a/common/datastorequery.h b/common/datastorequery.h
index 6620bbe..03b4eac 100644
--- a/common/datastorequery.h
+++ b/common/datastorequery.h
@@ -24,6 +24,7 @@
 #include "typeindex.h"
 #include "query.h"
 #include "entitybuffer.h"
+#include "log.h"
 
 
 class Source;
@@ -52,6 +53,7 @@ protected:
     QVector<QByteArray> loadIncrementalResultSet(qint64 baseRevision);
 
     void setupQuery();
+    QByteArrayList executeSubquery(const Sink::Query &subquery);
 
     Sink::Query mQuery;
     Sink::Storage::Transaction &mTransaction;
@@ -62,6 +64,8 @@ protected:
     bool mInitialQuery;
     QSharedPointer<FilterBase> mCollector;
     QSharedPointer<Source> mSource;
+
+    SINK_DEBUG_COMPONENT(mType)
 };
 
 
diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h
index 8b96758..2c93639 100644
--- a/common/domain/applicationdomaintype.h
+++ b/common/domain/applicationdomaintype.h
@@ -74,6 +74,7 @@
     struct NAME { \
         static constexpr const char *name = #LOWERCASENAME; \
         typedef QByteArray Type; \
+        typedef ApplicationDomain::TYPE ReferenceType; \
     }; \
     void set##NAME(const ApplicationDomain::TYPE &value) { setProperty(NAME::name, value); } \
     void set##NAME(const QByteArray &value) { setProperty(NAME::name, QVariant::fromValue(value)); } \
diff --git a/common/query.cpp b/common/query.cpp
index 3de80d8..a43ef6c 100644
--- a/common/query.cpp
+++ b/common/query.cpp
@@ -62,6 +62,11 @@ bool Query::Comparator::matches(const QVariant &v) const
                 return false;
             }
             return v.value<QByteArrayList>().contains(value.toByteArray());
+        case In:
+            if (!v.isValid()) {
+                return false;
+            }
+            return value.value<QByteArrayList>().contains(v.toByteArray());
         case Invalid:
         default:
             break;
diff --git a/common/query.h b/common/query.h
index 3362ac7..403c5b5 100644
--- a/common/query.h
+++ b/common/query.h
@@ -46,7 +46,8 @@ public:
         enum Comparators {
             Invalid,
             Equals,
-            Contains
+            Contains,
+            In
         };
 
         Comparator();
@@ -158,6 +159,7 @@ public:
     QByteArrayList requestedProperties;
     QByteArray parentProperty;
     QByteArray sortProperty;
+    QByteArray type;
     int limit;
     bool liveQuery;
     bool synchronousQuery;
@@ -222,6 +224,14 @@ public:
         return filter(T::name, QVariant::fromValue(value.identifier()));
     }
 
+    template <typename T>
+    Query &filter(const Query &query)
+    {
+        auto q = query;
+        q.type = ApplicationDomain::getTypeName<typename T::ReferenceType>();
+        return filter(T::name, QVariant::fromValue(q));
+    }
+
     Query &filter(const ApplicationDomain::SinkResource &resource)
     {
         resources << resource.identifier();
@@ -353,3 +363,4 @@ public:
 QDebug operator<<(QDebug dbg, const Sink::Query::Comparator &c);
 
 Q_DECLARE_OPERATORS_FOR_FLAGS(Sink::Query::Flags)
+Q_DECLARE_METATYPE(Sink::Query);
diff --git a/common/queryrunner.cpp b/common/queryrunner.cpp
index d3f8f66..1835e1f 100644
--- a/common/queryrunner.cpp
+++ b/common/queryrunner.cpp
@@ -66,7 +66,7 @@ private:
 template <class DomainType>
 QueryRunner<DomainType>::QueryRunner(const Sink::Query &query, const Sink::ResourceAccessInterface::Ptr &resourceAccess, const QByteArray &instanceIdentifier,
     const DomainTypeAdaptorFactoryInterface::Ptr &factory, const QByteArray &bufferType)
-    : QueryRunnerBase(), mResourceAccess(resourceAccess), mResultProvider(new ResultProvider<typename DomainType::Ptr>), mBatchSize(query.limit)
+    : QueryRunnerBase(), mResourceInstanceIdentifier(instanceIdentifier), mResourceAccess(resourceAccess), mResultProvider(new ResultProvider<typename DomainType::Ptr>), mBatchSize(query.limit)
 {
     SinkTrace() << "Starting query";
     if (query.limit && query.sortProperty.isEmpty()) {
diff --git a/common/queryrunner.h b/common/queryrunner.h
index 439a990..78aabf6 100644
--- a/common/queryrunner.h
+++ b/common/queryrunner.h
@@ -25,6 +25,7 @@
 #include "domaintypeadaptorfactoryinterface.h"
 #include "storage.h"
 #include "query.h"
+#include "log.h"
 
 /**
  * Base clase because you can't have the Q_OBJECT macro in template classes
@@ -96,6 +97,8 @@ public:
     typename Sink::ResultEmitter<typename DomainType::Ptr>::Ptr emitter();
 
 private:
+    QByteArray mResourceInstanceIdentifier;
+    SINK_DEBUG_COMPONENT(mResourceInstanceIdentifier)
     QSharedPointer<Sink::ResourceAccessInterface> mResourceAccess;
     QSharedPointer<Sink::ResultProvider<typename DomainType::Ptr>> mResultProvider;
     ResultTransformation mResultTransformation;
diff --git a/common/typeindex.cpp b/common/typeindex.cpp
index 272237c..816e7ee 100644
--- a/common/typeindex.cpp
+++ b/common/typeindex.cpp
@@ -24,6 +24,8 @@
 
 SINK_DEBUG_AREA("typeindex")
 
+using namespace Sink;
+
 static QByteArray getByteArray(const QVariant &value)
 {
     if (value.type() == QVariant::DateTime) {
@@ -138,16 +140,32 @@ void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDoma
     }
 }
 
+static QVector<QByteArray> indexLookup(Index &index, Query::Comparator filter)
+{
+    QVector<QByteArray> keys;
+    QByteArrayList lookupKeys;
+    if (filter.comparator == Query::Comparator::Equals) {
+        lookupKeys << getByteArray(filter.value);
+    } else if (filter.comparator == Query::Comparator::In) {
+        lookupKeys = filter.value.value<QByteArrayList>();
+    } else {
+        Q_ASSERT(false);
+    }
+
+    for (const auto &lookupKey : lookupKeys) {
+        index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; },
+            [lookupKey](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << lookupKey; }, true);
+    }
+    return keys;
+}
+
 QVector<QByteArray> TypeIndex::query(const Sink::Query &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction)
 {
     QVector<QByteArray> keys;
     for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) {
         if (query.hasFilter(it.key()) && query.sortProperty == it.value()) {
             Index index(indexName(it.key(), it.value()), transaction);
-            const auto lookupKey = getByteArray(query.getFilter(it.key()).value);
-            SinkTrace() << "looking for " << lookupKey;
-            index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; },
-                [it](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << it.key() << it.value(); }, true);
+            keys << indexLookup(index, query.getFilter(it.key()));
             appliedFilters << it.key();
             appliedSorting = it.value();
             SinkTrace() << "Index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys.";
@@ -157,9 +175,7 @@ QVector<QByteArray> TypeIndex::query(const Sink::Query &query, QSet<QByteArray>
     for (const auto &property : mProperties) {
         if (query.hasFilter(property)) {
             Index index(indexName(property), transaction);
-            const auto lookupKey = getByteArray(query.getFilter(property).value);
-            index.lookup(
-                lookupKey, [&](const QByteArray &value) { keys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; });
+            keys << indexLookup(index, query.getFilter(property));
             appliedFilters << property;
             SinkTrace() << "Index lookup on " << property << " found " << keys.size() << " keys.";
             return keys;
@@ -177,7 +193,7 @@ QVector<QByteArray> TypeIndex::lookup(const QByteArray &property, const QVariant
         Index index(indexName(property), transaction);
         const auto lookupKey = getByteArray(value);
         index.lookup(
-            lookupKey, [&](const QByteArray &value) { keys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; });
+            lookupKey, [&, this](const QByteArray &value) { keys << value; }, [property, this](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; });
         SinkTrace() << "Index lookup on " << property << " found " << keys.size() << " keys.";
         return keys;
     } else if (mSecondaryProperties.contains(property)) {
@@ -189,7 +205,7 @@ QVector<QByteArray> TypeIndex::lookup(const QByteArray &property, const QVariant
         Index index(indexName(property + resultProperty), transaction);
         const auto lookupKey = getByteArray(value);
         index.lookup(
-            lookupKey, [&](const QByteArray &value) { secondaryKeys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; });
+            lookupKey, [&, this](const QByteArray &value) { secondaryKeys << value; }, [property, this](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; });
         SinkTrace() << "Looked up secondary keys: " << secondaryKeys;
         for (const auto &secondary : secondaryKeys) {
             keys += lookup(resultProperty, secondary, transaction);
diff --git a/common/typeindex.h b/common/typeindex.h
index 7266f02..4972e95 100644
--- a/common/typeindex.h
+++ b/common/typeindex.h
@@ -22,6 +22,7 @@
 #include "bufferadaptor.h"
 #include "storage.h"
 #include "query.h"
+#include "log.h"
 #include <QByteArray>
 
 class TypeIndex
@@ -79,6 +80,7 @@ public:
 private:
     QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const;
     QByteArray mType;
+    SINK_DEBUG_COMPONENT(mType)
     QByteArrayList mProperties;
     QMap<QByteArray, QByteArray> mSortedProperties;
     //<Property, ResultProperty>
diff --git a/tests/querytest.cpp b/tests/querytest.cpp
index afa8e33..6011a99 100644
--- a/tests/querytest.cpp
+++ b/tests/querytest.cpp
@@ -492,6 +492,42 @@ private slots:
         QCOMPARE(folders.size(), 1);
     }
 
+    void testSubquery()
+    {
+        // Setup
+        auto folder1 = Folder::createEntity<Folder>("sink.dummy.instance1");
+        folder1.setSpecialPurpose(QByteArrayList() << "purpose1");
+        VERIFYEXEC(Sink::Store::create<Folder>(folder1));
+
+        auto folder2 = Folder::createEntity<Folder>("sink.dummy.instance1");
+        folder2.setSpecialPurpose(QByteArrayList() << "purpose2");
+        VERIFYEXEC(Sink::Store::create<Folder>(folder2));
+
+        {
+            auto mail = Mail::createEntity<Mail>("sink.dummy.instance1");
+            mail.setUid("mail1");
+            mail.setFolder(folder1);
+            VERIFYEXEC(Sink::Store::create(mail));
+        }
+        {
+            auto mail = Mail::createEntity<Mail>("sink.dummy.instance1");
+            mail.setUid("mail2");
+            mail.setFolder(folder2);
+            VERIFYEXEC(Sink::Store::create(mail));
+        }
+
+        // Ensure all local data is processed
+        Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1").exec().waitForFinished();
+
+        //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter.
+        Query query;
+        query.filter<Mail::Folder>(Sink::Query().containsFilter<Folder::SpecialPurpose>("purpose1"));
+        query.request<Mail::Uid>();
+
+        auto mails = Sink::Store::read<Mail>(query);
+        QCOMPARE(mails.size(), 1);
+        QCOMPARE(mails.first().getUid().toLatin1(), QByteArray("mail1"));
+    }
 };
 
 QTEST_MAIN(QueryTest)