From 00717f6c8b8a9c6dbd56a80d685c5082fc03f6a5 Mon Sep 17 00:00:00 2001 From: Minijackson Date: Fri, 25 May 2018 11:28:22 +0200 Subject: Implement range queries --- common/domain/typeimplementations.cpp | 5 +- common/domain/typeimplementations_p.h | 18 ++++- common/index.cpp | 14 ++++ common/index.h | 4 ++ common/query.cpp | 12 +++- common/query.h | 1 + common/typeindex.cpp | 122 ++++++++++++++++++++++++++++++---- common/typeindex.h | 13 ++-- tests/querytest.cpp | 107 +++++++++++++++++++++++++++++ 9 files changed, 275 insertions(+), 21 deletions(-) diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp index e65c998..a8f4baf 100644 --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -38,7 +38,7 @@ using namespace Sink::ApplicationDomain; MAPPER.addMapping(&Sink::ApplicationDomain::Buffer::ENTITYTYPE::LOWERCASEPROPERTY, &Sink::ApplicationDomain::Buffer::ENTITYTYPE##Builder::add_##LOWERCASEPROPERTY); typedef IndexConfig, + SortedIndex, ValueIndex, ValueIndex, ValueIndex, @@ -64,7 +64,8 @@ typedef IndexConfig AddressbookIndexConfig; typedef IndexConfig + ValueIndex, + SortedIndex > EventIndexConfig; typedef IndexConfig +template class SortedIndex { public: @@ -78,6 +78,22 @@ public: } }; +template +class SortedIndex +{ +public: + static void configure(TypeIndex &index) + { + index.addSortedProperty(); + } + + template + static QMap databases() + { + return {{QByteArray{EntityType::name} +".index." + SortProperty::name + ".sorted", 1}}; + } +}; + template class SecondaryIndex { diff --git a/common/index.cpp b/common/index.cpp index ff87ae2..94b2eea 100644 --- a/common/index.cpp +++ b/common/index.cpp @@ -59,3 +59,17 @@ QByteArray Index::lookup(const QByteArray &key) lookup(key, [&](const QByteArray &value) { result = QByteArray(value.constData(), value.size()); }, [](const Index::Error &) { }); return result; } + +void Index::rangeLookup(const QByteArray &lowerBound, const QByteArray &upperBound, + const std::function &resultHandler, + const std::function &errorHandler) +{ + mDb.findAllInRange(lowerBound, upperBound, + [&](const QByteArray &key, const QByteArray &value) { + resultHandler(value); + }, + [&](const Sink::Storage::DataStore::Error &error) { + SinkWarningCtx(mLogCtx) << "Error while retrieving value:" << error << mName; + errorHandler(Error(error.store, error.code, error.message)); + }); +} diff --git a/common/index.h b/common/index.h index f16a426..043cc90 100644 --- a/common/index.h +++ b/common/index.h @@ -40,6 +40,10 @@ public: bool matchSubStringKeys = false); QByteArray lookup(const QByteArray &key); + void rangeLookup(const QByteArray &lowerBound, const QByteArray &upperBound, + const std::function &resultHandler, + const std::function &errorHandler); + private: Q_DISABLE_COPY(Index); Sink::Storage::DataStore::Transaction mTransaction; diff --git a/common/query.cpp b/common/query.cpp index 5f6d095..404a304 100644 --- a/common/query.cpp +++ b/common/query.cpp @@ -132,8 +132,8 @@ bool QueryBase::Filter::operator==(const QueryBase::Filter &other) const bool QueryBase::operator==(const QueryBase &other) const { - auto ret = mType == other.mType - && mSortProperty == other.mSortProperty + auto ret = mType == other.mType + && mSortProperty == other.mSortProperty && mBaseFilterStage == other.mBaseFilterStage; return ret; } @@ -171,6 +171,14 @@ bool QueryBase::Comparator::matches(const QVariant &v) const return false; } return value.value().contains(v.toByteArray()); + case Within: { + auto range = value.value>(); + if (range.size() < 2) { + return false; + } + + return range[0] <= v && v <= range[1]; + } case Fulltext: case Invalid: default: diff --git a/common/query.h b/common/query.h index f0c1e01..7130116 100644 --- a/common/query.h +++ b/common/query.h @@ -36,6 +36,7 @@ public: Equals, Contains, In, + Within, Fulltext }; diff --git a/common/typeindex.cpp b/common/typeindex.cpp index a897ad0..5564144 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -21,9 +21,12 @@ #include "log.h" #include "index.h" #include "fulltextindex.h" + #include #include +#include + using namespace Sink; static QByteArray getByteArray(const QVariant &value) @@ -50,15 +53,34 @@ static QByteArray getByteArray(const QVariant &value) return "toplevel"; } -static QByteArray toSortableByteArray(const QDateTime &date) + +static QByteArray toSortableByteArrayImpl(const QDateTime &date) { // Sort invalid last if (!date.isValid()) { return QByteArray::number(std::numeric_limits::max()); } - return QByteArray::number(std::numeric_limits::max() - date.toTime_t()); + static unsigned int uint_num_digits = std::log10(std::numeric_limits::max()) + 1; + return QByteArray::number(std::numeric_limits::max() - date.toTime_t()).rightJustified(uint_num_digits, '0'); } +static QByteArray toSortableByteArray(const QVariant &value) +{ + if (!value.isValid()) { + // FIXME: we don't know the type, so we don't know what to return + // This mean we're fixing every sorted index keys to use unsigned int + return QByteArray::number(std::numeric_limits::max()); + } + + switch (value.type()) { + case QMetaType::QDateTime: + return toSortableByteArrayImpl(value.toDateTime()); + default: + SinkWarning() << "Not knowing how to convert a" << value.typeName() + << "to a sortable key, falling back to default conversion"; + return getByteArray(value); + } +} TypeIndex::TypeIndex(const QByteArray &type, const Sink::Log::Context &ctx) : mLogCtx(ctx), mType(type) { @@ -72,6 +94,11 @@ QByteArray TypeIndex::indexName(const QByteArray &property, const QByteArray &so return mType + ".index." + property + ".sort." + sortProperty; } +QByteArray TypeIndex::sortedIndexName(const QByteArray &property) const +{ + return mType + ".index." + property + ".sorted"; +} + template <> void TypeIndex::addProperty(const QByteArray &property) { @@ -137,6 +164,22 @@ void TypeIndex::addProperty(const QByteArray &prop addProperty(property); } +template <> +void TypeIndex::addSortedProperty(const QByteArray &property) +{ + auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, + Sink::Storage::DataStore::Transaction &transaction) { + const auto sortableDate = toSortableByteArray(value); + if (add) { + Index(sortedIndexName(property), transaction).add(sortableDate, identifier); + } else { + Index(sortedIndexName(property), transaction).remove(sortableDate, identifier); + } + }; + mSortIndexer.insert(property, indexer); + mSortedProperties << property; +} + template <> void TypeIndex::addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty) { @@ -149,8 +192,8 @@ void TypeIndex::addPropertyWithSorting(const QByteArray & Index(indexName(property, sortProperty), transaction).remove(propertyValue + toSortableByteArray(date), identifier); } }; - mSortIndexer.insert(property + sortProperty, indexer); - mSortedProperties.insert(property, sortProperty); + mGroupedSortIndexer.insert(property + sortProperty, indexer); + mGroupedSortedProperties.insert(property, sortProperty); } template <> @@ -166,10 +209,15 @@ void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink:: auto indexer = mIndexer.value(property); indexer(add, identifier, value, transaction); } - for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { + for (const auto &property : mSortedProperties) { + const auto value = entity.getProperty(property); + auto indexer = mSortIndexer.value(property); + indexer(add, identifier, value, transaction); + } + for (auto it = mGroupedSortedProperties.constBegin(); it != mGroupedSortedProperties.constEnd(); it++) { const auto value = entity.getProperty(it.key()); const auto sortValue = entity.getProperty(it.value()); - auto indexer = mSortIndexer.value(it.key() + it.value()); + auto indexer = mGroupedSortIndexer.value(it.key() + it.value()); indexer(add, identifier, value, sortValue, transaction); } for (const auto &indexer : mCustomIndexer) { @@ -207,22 +255,61 @@ void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDoma updateIndex(false, identifier, entity, transaction, resourceInstanceId); } -static QVector indexLookup(Index &index, QueryBase::Comparator filter) +static QVector indexLookup(Index &index, QueryBase::Comparator filter, + std::function valueToKey = getByteArray) { QVector keys; QByteArrayList lookupKeys; if (filter.comparator == Query::Comparator::Equals) { - lookupKeys << getByteArray(filter.value); + lookupKeys << valueToKey(filter.value); } else if (filter.comparator == Query::Comparator::In) { - lookupKeys = filter.value.value(); + for(const QVariant &value : filter.value.value()) { + lookupKeys << valueToKey(value); + } + //lookupKeys = filter.value.value(); } else { Q_ASSERT(false); } for (const auto &lookupKey : lookupKeys) { index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; }, - [lookupKey](const Index::Error &error) { SinkWarning() << "Lookup error in index: " << error.message << lookupKey; }, true); + [lookupKey](const Index::Error &error) { + SinkWarning() << "Lookup error in index: " << error.message << lookupKey; + }, + true); + } + return keys; +} + +static QVector sortedIndexLookup(Index &index, QueryBase::Comparator filter) +{ + if (filter.comparator == Query::Comparator::In || filter.comparator == Query::Comparator::Contains) { + SinkWarning() << "In and Contains comparison not supported on sorted indexes"; + } + + if (filter.comparator != Query::Comparator::Within) { + return indexLookup(index, filter, toSortableByteArray); + } + + QVector keys; + + QByteArray lowerBound, upperBound; + auto bounds = filter.value.value(); + if (bounds[0].canConvert()) { + // Inverse the bounds because dates are stored newest first + upperBound = toSortableByteArray(bounds[0].toDateTime()); + lowerBound = toSortableByteArray(bounds[1].toDateTime()); + } else { + lowerBound = bounds[0].toByteArray(); + upperBound = bounds[1].toByteArray(); } + + index.rangeLookup(lowerBound, upperBound, [&](const QByteArray &value) { keys << value; }, + [bounds](const Index::Error &error) { + SinkWarning() << "Lookup error in index:" << error.message + << "with bounds:" << bounds[0] << bounds[1]; + }); + return keys; } @@ -239,16 +326,27 @@ QVector TypeIndex::query(const Sink::QueryBase &query, QSet void addProperty(const QByteArray &property); + template + void addSortedProperty(const QByteArray &property); template void addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty); @@ -54,9 +56,9 @@ public: } template - void addPropertyWithSorting() + void addSortedProperty() { - addPropertyWithSorting(T::name); + addSortedProperty(T::name); } template @@ -112,14 +114,17 @@ private: friend class Sink::Storage::EntityStore; void updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const; + QByteArray sortedIndexName(const QByteArray &property) const; Sink::Log::Context mLogCtx; QByteArray mType; QByteArrayList mProperties; - QMap mSortedProperties; + QByteArrayList mSortedProperties; + QMap mGroupedSortedProperties; // QMap mSecondaryProperties; QList mCustomIndexer; Sink::Storage::DataStore::Transaction *mTransaction; QHash> mIndexer; - QHash> mSortIndexer; + QHash> mSortIndexer; + QHash> mGroupedSortIndexer; }; diff --git a/tests/querytest.cpp b/tests/querytest.cpp index fa016a2..36b6e90 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -1510,6 +1510,113 @@ private slots: } } + void mailsWithDates() + { + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedDate(QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)); + mail.setExtractedMessageId("message1"); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedDate(QDateTime::fromString("2018-05-23T13:50:00Z", Qt::ISODate)); + mail.setExtractedMessageId("message2"); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedDate(QDateTime::fromString("2018-05-27T13:50:00Z", Qt::ISODate)); + mail.setExtractedMessageId("message3"); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedMessageId("message4"); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedDate(QDateTime::fromString("2078-05-23T13:49:41Z", Qt::ISODate)); + mail.setExtractedMessageId("message5"); + VERIFYEXEC(Sink::Store::create(mail)); + } + VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); + } + + void testMailDate() + { + mailsWithDates(); + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 1); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QDateTime::fromString("2018-05-27T13:49:41Z", Qt::ISODate)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 0); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QDateTime::fromString("2018-05-27T13:50:00Z", Qt::ISODate)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 1); + } + + } + + void testMailRange() + { + mailsWithDates(); + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 1); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-25T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 2); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-30T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 3); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2118-05-30T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 4); + } + } }; QTEST_MAIN(QueryTest) -- cgit v1.2.3