From c741d7fbb21ff959ca4b6d4f7681aead277044b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Nicole?= Date: Fri, 27 Jul 2018 13:40:03 +0200 Subject: Use Key API in ResultSet Summary: Depends on D14099 Notes: - Tests pass without many modifications outside of resultset.cpp/.h - `mGenerator` doesn't seem to be used? Benchmarks ========= Run benchmarks: | Develop | D14099 | This patch | | ---------------------------------- | ---------------------------------- | ---------------------------------- | | Current Rss usage [kb]: 40700 | Current Rss usage [kb]: 38564 | Current Rss usage [kb]: 39112 | | Peak Rss usage [kb]: 40700 | Peak Rss usage [kb]: 38564 | Peak Rss usage [kb]: 39112 | | Rss growth [kb]: 15920 | Rss growth [kb]: 13352 | Rss growth [kb]: 13432 | | Rss growth per entity [byte]: 3260 | Rss growth per entity [byte]: 2734 | Rss growth per entity [byte]: 2750 | | Rss without db [kb]: 29736 | Rss without db [kb]: 29248 | Rss without db [kb]: 30100 | | Percentage peak rss error: 0 | Percentage peak rss error: 0 | Percentage peak rss error: 0 | | On disk [kb]: 10788 | On disk [kb]: 9140 | On disk [kb]: 8836 | | Buffer size total [kb]: 898 | Buffer size total [kb]: 898 | Buffer size total [kb]: 898 | | Write amplification: 12.0075 | Write amplification: 10.1732 | Write amplification: 9.83485 | Test Disk Usage: | Develop | D14099 | This patch | | ----------------------------------- | ----------------------------------- | ----------------------------------- | | Free pages: 412 | Free pages: 309 | Free pages: 312 | | Total pages: 760 | Total pages: 599 | Total pages: 603 | | Used size: 1425408 | Used size: 1187840 | Used size: 1191936 | | Calculated key + value size: 856932 | Calculated key + value size: 702866 | Calculated key + value size: 702866 | | Calculated total db sizes: 970752 | Calculated total db sizes: 954368 | Calculated total db sizes: 933888 | | Main store on disk: 3112960 | Main store on disk: 2453504 | Main store on disk: 2469888 | | Total on disk: 3293184 | Total on disk: 2633728 | Total on disk: 2650112 | | Used size amplification: 1.66339 | Used size amplification: 1.68999 | Used size amplification: 1.69582 | | Write amplification: 3.63268 | Write amplification: 3.49071 | Write amplification: 3.51402 | Reviewers: cmollekopf Reviewed By: cmollekopf Tags: #sink Differential Revision: https://phabricator.kde.org/D14289 --- common/datastorequery.cpp | 6 +++--- common/queryrunner.cpp | 9 ++++++++- common/resultset.cpp | 8 +++++--- common/resultset.h | 21 +++++++++++++-------- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp index aa315c6..52243da 100644 --- a/common/datastorequery.cpp +++ b/common/datastorequery.cpp @@ -271,7 +271,7 @@ public: struct ReductionResult { Identifier selection; - QVector aggregateIds; + QVector aggregateIds; QMap aggregateValues; }; @@ -284,14 +284,14 @@ public: for (auto &aggregator : mAggregators) { aggregator.reset(); } - QVector reducedAndFilteredResults; + QVector reducedAndFilteredResults; for (const auto &r : results) { readEntity(r, [&, this](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { //We need to apply all property filters that we have until the reduction, because the index lookup was unfiltered. if (!matchesFilter(entity)) { return; } - reducedAndFilteredResults << r.toDisplayByteArray(); + reducedAndFilteredResults << r; Q_ASSERT(operation != Sink::Operation_Removal); for (auto &aggregator : mAggregators) { if (!aggregator.property.isEmpty()) { diff --git a/common/queryrunner.cpp b/common/queryrunner.cpp index 9ac3517..7d6d279 100644 --- a/common/queryrunner.cpp +++ b/common/queryrunner.cpp @@ -271,7 +271,14 @@ void QueryWorker::resultProviderCallback(const Sink::Query &query, S for (auto it = result.aggregateValues.constBegin(); it != result.aggregateValues.constEnd(); it++) { valueCopy->setProperty(it.key(), it.value()); } - valueCopy->aggregatedIds() = result.aggregateIds; + valueCopy->aggregatedIds() = [&] { + QVector aggregateIdsBA; + aggregateIdsBA.reserve(result.aggregateIds.size()); + for (const auto &id : result.aggregateIds) { + aggregateIdsBA << id.toDisplayByteArray(); + } + return aggregateIdsBA; + }(); if (mResultTransformation) { mResultTransformation(*valueCopy); } diff --git a/common/resultset.cpp b/common/resultset.cpp index 08954c9..88110e1 100644 --- a/common/resultset.cpp +++ b/common/resultset.cpp @@ -20,6 +20,8 @@ #include "log.h" +using Sink::Storage::Identifier; + ResultSet::ResultSet() : mIt(nullptr) { } @@ -32,7 +34,7 @@ ResultSet::ResultSet(const IdGenerator &generator) : mIt(nullptr), mGenerator(ge { } -ResultSet::ResultSet(const QVector &resultSet) +ResultSet::ResultSet(const QVector &resultSet) : mResultSet(resultSet), mIt(mResultSet.constBegin()), mSkip([this]() { @@ -114,11 +116,11 @@ ResultSet::ReplayResult ResultSet::replaySet(int offset, int batchSize, const Ca return {counter, false}; } -QByteArray ResultSet::id() +Identifier ResultSet::id() { if (mIt) { if (mIt == mResultSet.constEnd()) { - return QByteArray(); + return {}; } Q_ASSERT(mIt != mResultSet.constEnd()); return *mIt; diff --git a/common/resultset.h b/common/resultset.h index 5587c54..9d83c67 100644 --- a/common/resultset.h +++ b/common/resultset.h @@ -25,6 +25,7 @@ #include "metadata_generated.h" #include "entitybuffer.h" #include "applicationdomaintype.h" +#include "storage/key.h" /* * An iterator to a result set. @@ -35,21 +36,25 @@ class ResultSet { public: struct Result { - Result(const Sink::ApplicationDomain::ApplicationDomainType &e, Sink::Operation op, const QMap &v = {}, const QVector &a = {}) : entity(e), operation(op), aggregateValues(v), aggregateIds(a) {} + Result(const Sink::ApplicationDomain::ApplicationDomainType &e, Sink::Operation op, + const QMap &v = {}, const QVector &a = {}) + : entity(e), operation(op), aggregateValues(v), aggregateIds(a) + { + } Sink::ApplicationDomain::ApplicationDomainType entity; Sink::Operation operation; QMap aggregateValues; - QVector aggregateIds; + QVector aggregateIds; }; typedef std::function Callback; typedef std::function ValueGenerator; - typedef std::function IdGenerator; + typedef std::function IdGenerator; typedef std::function SkipValue; ResultSet(); ResultSet(const ValueGenerator &generator, const SkipValue &skip); ResultSet(const IdGenerator &generator); - ResultSet(const QVector &resultSet); + ResultSet(const QVector &resultSet); ResultSet(const ResultSet &other); bool next(); @@ -63,14 +68,14 @@ public: }; ReplayResult replaySet(int offset, int batchSize, const Callback &callback); - QByteArray id(); + Sink::Storage::Identifier id(); bool isEmpty(); private: - QVector mResultSet; - QVector::ConstIterator mIt; - QByteArray mCurrentValue; + QVector mResultSet; + QVector::ConstIterator mIt; + Sink::Storage::Identifier mCurrentValue; IdGenerator mGenerator; ValueGenerator mValueGenerator; SkipValue mSkip; -- cgit v1.2.3