From 529dd17eec62a9702b8837f8f1976dfbf28fdd82 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 16 Feb 2016 17:43:22 +0100 Subject: Prepared sort indexes --- common/queryrunner.cpp | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) (limited to 'common/queryrunner.cpp') diff --git a/common/queryrunner.cpp b/common/queryrunner.cpp index 2d188ae..299b987 100644 --- a/common/queryrunner.cpp +++ b/common/queryrunner.cpp @@ -54,12 +54,12 @@ private: void readEntity(const Sink::Storage::NamedDatabase &db, const QByteArray &key, const std::function &resultCallback); - ResultSet loadInitialResultSet(const Sink::Query &query, Sink::Storage::Transaction &transaction, QSet &remainingFilters); + ResultSet loadInitialResultSet(const Sink::Query &query, Sink::Storage::Transaction &transaction, QSet &remainingFilters, QByteArray &remainingSorting); ResultSet loadIncrementalResultSet(qint64 baseRevision, const Sink::Query &query, Sink::Storage::Transaction &transaction, QSet &remainingFilters); ResultSet filterAndSortSet(ResultSet &resultSet, const std::function &filter, const Sink::Storage::NamedDatabase &db, bool initialQuery, const QByteArray &sortProperty); std::function getFilter(const QSet remainingFilters, const Sink::Query &query); - qint64 load(const Sink::Query &query, const std::function &)> &baseSetRetriever, Sink::ResultProviderInterface &resultProvider, bool initialQuery, int offset, int batchSize); + qint64 load(const Sink::Query &query, const std::function &, QByteArray &)> &baseSetRetriever, Sink::ResultProviderInterface &resultProvider, bool initialQuery, int offset, int batchSize); private: QueryRunnerBase::ResultTransformation mResultTransformation; @@ -154,6 +154,7 @@ static inline ResultSet fullScan(const Sink::Storage::Transaction &transaction, Warning() << "Error during query: " << error.message; }); + Trace() << "Full scan retrieved " << keys.size() << " results."; return ResultSet(keys); } @@ -207,7 +208,7 @@ void QueryWorker::replaySet(ResultSet &resultSet, Sink::ResultProvid } return true; })){}; - Trace() << "Replayed " << counter << " results"; + Trace() << "Replayed " << counter << " results." << "Limit " << batchSize; } template @@ -234,14 +235,18 @@ void QueryWorker::readEntity(const Sink::Storage::NamedDatabase &db, } template -ResultSet QueryWorker::loadInitialResultSet(const Sink::Query &query, Sink::Storage::Transaction &transaction, QSet &remainingFilters) +ResultSet QueryWorker::loadInitialResultSet(const Sink::Query &query, Sink::Storage::Transaction &transaction, QSet &remainingFilters, QByteArray &remainingSorting) { if (!query.ids.isEmpty()) { return ResultSet(query.ids.toVector()); } QSet appliedFilters; - auto resultSet = Sink::ApplicationDomain::TypeImplementation::queryIndexes(query, mResourceInstanceIdentifier, appliedFilters, transaction); + QByteArray appliedSorting; + auto resultSet = Sink::ApplicationDomain::TypeImplementation::queryIndexes(query, mResourceInstanceIdentifier, appliedFilters, appliedSorting, transaction); remainingFilters = query.propertyFilter.keys().toSet() - appliedFilters; + if (appliedSorting.isEmpty()) { + remainingSorting = query.sortProperty; + } //We do a full scan if there were no indexes available to create the initial set. if (appliedFilters.isEmpty()) { @@ -284,6 +289,7 @@ ResultSet QueryWorker::filterAndSortSet(ResultSet &resultSet, const { const bool sortingRequired = !sortProperty.isEmpty(); if (initialQuery && sortingRequired) { + Trace() << "Sorting the resultset in memory according to property: " << sortProperty; //Sort the complete set by reading the sort property and filling into a sorted map auto sortedMap = QSharedPointer>::create(); while (resultSet.next()) { @@ -293,7 +299,7 @@ ResultSet QueryWorker::filterAndSortSet(ResultSet &resultSet, const if ((operation != Sink::Operation_Removal) && filter(domainObject)) { if (!sortProperty.isEmpty()) { const auto sortValue = domainObject->getProperty(sortProperty); - if (sortValue.canConvert()) { + if (sortValue.type() == QVariant::DateTime) { sortedMap->insert(QByteArray::number(std::numeric_limits::max() - sortValue.toDateTime().toTime_t()), domainObject->identifier()); } else { sortedMap->insert(sortValue.toString().toLatin1(), domainObject->identifier()); @@ -305,6 +311,7 @@ ResultSet QueryWorker::filterAndSortSet(ResultSet &resultSet, const }); } + Trace() << "Sorted " << sortedMap->size() << " values."; auto iterator = QSharedPointer >::create(*sortedMap); ResultSet::ValueGenerator generator = [this, iterator, sortedMap, &db, filter, initialQuery](std::function callback) -> bool { if (iterator->hasNext()) { @@ -377,7 +384,7 @@ std::function -qint64 QueryWorker::load(const Sink::Query &query, const std::function &)> &baseSetRetriever, Sink::ResultProviderInterface &resultProvider, bool initialQuery, int offset, int batchSize) +qint64 QueryWorker::load(const Sink::Query &query, const std::function &, QByteArray &)> &baseSetRetriever, Sink::ResultProviderInterface &resultProvider, bool initialQuery, int offset, int batchSize) { QTime time; time.start(); @@ -390,9 +397,10 @@ qint64 QueryWorker::load(const Sink::Query &query, const std::functi auto db = Storage::mainDatabase(transaction, mBufferType); QSet remainingFilters; - auto resultSet = baseSetRetriever(transaction, remainingFilters); + QByteArray remainingSorting; + auto resultSet = baseSetRetriever(transaction, remainingFilters, remainingSorting); Trace() << "Base set retrieved. " << time.elapsed(); - auto filteredSet = filterAndSortSet(resultSet, getFilter(remainingFilters, query), db, initialQuery, query.sortProperty); + auto filteredSet = filterAndSortSet(resultSet, getFilter(remainingFilters, query), db, initialQuery, remainingSorting); Trace() << "Filtered set retrieved. " << time.elapsed(); replaySet(filteredSet, resultProvider, query.requestedProperties, offset, batchSize); Trace() << "Filtered set replayed. " << time.elapsed(); @@ -408,7 +416,7 @@ qint64 QueryWorker::executeIncrementalQuery(const Sink::Query &query const qint64 baseRevision = resultProvider.revision() + 1; Trace() << "Running incremental query " << baseRevision; - auto revision = load(query, [&](Sink::Storage::Transaction &transaction, QSet &remainingFilters) -> ResultSet { + auto revision = load(query, [&](Sink::Storage::Transaction &transaction, QSet &remainingFilters, QByteArray &remainingSorting) -> ResultSet { return loadIncrementalResultSet(baseRevision, query, transaction, remainingFilters); }, resultProvider, false, 0, 0); Trace() << "Incremental query took: " << time.elapsed() << " ms"; @@ -431,8 +439,8 @@ qint64 QueryWorker::executeInitialQuery(const Sink::Query &query, co modifiedQuery.propertyFilter.insert(query.parentProperty, QVariant()); } } - auto revision = load(modifiedQuery, [&](Sink::Storage::Transaction &transaction, QSet &remainingFilters) -> ResultSet { - return loadInitialResultSet(modifiedQuery, transaction, remainingFilters); + auto revision = load(modifiedQuery, [&](Sink::Storage::Transaction &transaction, QSet &remainingFilters, QByteArray &remainingSorting) -> ResultSet { + return loadInitialResultSet(modifiedQuery, transaction, remainingFilters, remainingSorting); }, resultProvider, true, offset, batchsize); Trace() << "Initial query took: " << time.elapsed() << " ms"; resultProvider.initialResultSetComplete(parent); -- cgit v1.2.3