From 393846f660802d53d6ff6744cea0c1fa05019ba3 Mon Sep 17 00:00:00 2001
From: Christian Mollekopf <chrigi_1@fastmail.fm>
Date: Sun, 8 Oct 2017 22:10:04 +0200
Subject: Optimized the incremental update case.

This brings the incremental closer to a regular query (about 1.5 times
as bad instead of 3.5 times).
For a comparison look at MailQueryBenchmark::testIncremental()

The optimization is built on the assumption that we i.e. get an update
with 100 revisions, and thus the optimization applies to the case where
we have multiple revisions within that batch that are part of the same
reduction. In such a case we can avoid redoing the reduction lookup over
and over.
---
 common/datastorequery.cpp    | 14 +++++++++++++-
 common/datastorequery.h      |  4 +++-
 tests/mailquerybenchmark.cpp |  9 ++++++---
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp
index f3d9415..f5152b7 100644
--- a/common/datastorequery.cpp
+++ b/common/datastorequery.cpp
@@ -208,6 +208,7 @@ public:
     };
 
     QSet<QByteArray> mReducedValues;
+    QSet<QByteArray> mIncrementallyReducedValues;
     QHash<QByteArray, QByteArray> mSelectedValues;
     QByteArray mReductionProperty;
     QByteArray mSelectionProperty;
@@ -225,6 +226,11 @@ public:
 
     virtual ~Reduce(){}
 
+    void updateComplete() Q_DECL_OVERRIDE
+    {
+        mIncrementallyReducedValues.clear();
+    }
+
     static QByteArray getByteArray(const QVariant &value) {
         if (value.type() == QVariant::DateTime) {
             return value.toDateTime().toString().toLatin1();
@@ -304,7 +310,8 @@ public:
                     //During initial query, do nothing. The lookup above will take care of it.
                     //During updates adjust the reduction according to the modification/addition or removal
                     //We have to redo the reduction for every element, because of the aggregation values.
-                    if (mIncremental) {
+                    if (mIncremental && !mIncrementallyReducedValues.contains(reductionValueBa)) {
+                        mIncrementallyReducedValues.insert(reductionValueBa);
                         //Redo the reduction to find new aggregated values
                         QMap<QByteArray, QVariant> aggregateValues;
                         auto selectionResult = reduceOnValue(reductionValue, aggregateValues);
@@ -621,6 +628,11 @@ ResultSet DataStoreQuery::update(qint64 baseRevision)
 void DataStoreQuery::updateComplete()
 {
     mSource->mIncrementalIds.clear();
+    auto source = mCollector;
+    while (source) {
+        source->updateComplete();
+        source = source->mSource;
+    }
 }
 
 ResultSet DataStoreQuery::execute()
diff --git a/common/datastorequery.h b/common/datastorequery.h
index de4ae26..cc501e6 100644
--- a/common/datastorequery.h
+++ b/common/datastorequery.h
@@ -107,11 +107,13 @@ public:
         return mDatastore->indexLookup(property, value);
     }
 
-    virtual void skip() { mSource->skip(); };
+    virtual void skip() { mSource->skip(); }
 
     //Returns true for as long as a result is available
     virtual bool next(const std::function<void(const ResultSet::Result &)> &callback) = 0;
 
+    virtual void updateComplete() { }
+
     FilterBase::Ptr mSource;
     DataStoreQuery *mDatastore;
     bool mIncremental = false;
diff --git a/tests/mailquerybenchmark.cpp b/tests/mailquerybenchmark.cpp
index ca1e026..402f31f 100644
--- a/tests/mailquerybenchmark.cpp
+++ b/tests/mailquerybenchmark.cpp
@@ -219,11 +219,14 @@ private slots:
 
         populateDatabase(count, 10, false, count);
         time.restart();
-        context.mResourceAccess->revisionChanged(2000);
+        for (int i = 0; i <= 10; i++) {
+            //Simulate revision updates in steps of 100
+            context.mResourceAccess->revisionChanged(1000 + i * 100);
+        }
         //We should have 200 items in total in the end. 2000 mails / 10 folders => 200 reduced mails
         QTRY_COMPARE(added.count(), 200);
-        //For every email we have to redo the reduction and increase the count, which is a modification.
-        QTRY_COMPARE(modified.count(), 900);
+        //We get one modification per thread from the first 100 (1000 mails / 10 folders), everything else is optimized away because we ignore repeated updates to the same thread.
+        QTRY_COMPARE(modified.count(), 100);
         std::cout << "Incremental query took " << time.elapsed() << std::endl;
         std::cout << "added " << added.count() << std::endl;
         std::cout << "modified " << modified.count() << std::endl;
-- 
cgit v1.2.3