summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--common/datastorequery.cpp520
-rw-r--r--common/datastorequery.h74
-rw-r--r--common/domain/event.cpp1
-rw-r--r--common/domain/event.h5
-rw-r--r--common/domain/folder.cpp1
-rw-r--r--common/domain/folder.h4
-rw-r--r--common/domain/mail.cpp53
-rw-r--r--common/domain/mail.h3
-rw-r--r--common/entityreader.cpp1
-rw-r--r--common/mailpreprocessor.cpp8
-rw-r--r--common/modelresult.cpp12
-rw-r--r--common/query.cpp3
-rw-r--r--common/typeindex.cpp10
-rw-r--r--common/typeindex.h2
-rw-r--r--examples/maildirresource/tests/maildirthreadtest.cpp2
-rw-r--r--tests/querytest.cpp104
16 files changed, 540 insertions, 263 deletions
diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp
index cc070be..95df1a0 100644
--- a/common/datastorequery.cpp
+++ b/common/datastorequery.cpp
@@ -27,13 +27,173 @@ using namespace Sink;
27 27
28SINK_DEBUG_AREA("datastorequery") 28SINK_DEBUG_AREA("datastorequery")
29 29
30class Source : public FilterBase {
31 public:
32 typedef QSharedPointer<Source> Ptr;
33
34 QVector<QByteArray> mIds;
35 QVector<QByteArray>::ConstIterator mIt;
36
37 Source (const QVector<QByteArray> &ids, DataStoreQuery *store)
38 : FilterBase(store),
39 mIds(ids),
40 mIt(mIds.constBegin())
41 {
42
43 }
44
45 virtual ~Source(){}
46
47 virtual void skip() Q_DECL_OVERRIDE
48 {
49 if (mIt != mIds.constEnd()) {
50 mIt++;
51 }
52 };
53
54 void add(const QVector<QByteArray> &ids)
55 {
56 mIds = ids;
57 mIt = mIds.constBegin();
58 }
59
60 bool next(const std::function<void(Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &entityBuffer)> &callback) Q_DECL_OVERRIDE
61 {
62 if (mIt == mIds.constEnd()) {
63 return false;
64 }
65 readEntity(*mIt, [callback](const QByteArray &uid, const Sink::EntityBuffer &entityBuffer) {
66 callback(entityBuffer.operation(), uid, entityBuffer);
67 });
68 mIt++;
69 return mIt != mIds.constEnd();
70 }
71};
72
73class Collector : public FilterBase {
74public:
75 typedef QSharedPointer<Collector> Ptr;
76
77 Collector(FilterBase::Ptr source, DataStoreQuery *store)
78 : FilterBase(source, store)
79 {
80
81 }
82 virtual ~Collector(){}
83
84 bool next(const std::function<void(Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &entityBuffer)> &callback) Q_DECL_OVERRIDE
85 {
86 return mSource->next(callback);
87 }
88};
89
90class Filter : public FilterBase {
91public:
92 typedef QSharedPointer<Filter> Ptr;
93
94 QHash<QByteArray, Sink::Query::Comparator> propertyFilter;
95
96 Filter(FilterBase::Ptr source, DataStoreQuery *store)
97 : FilterBase(source, store)
98 {
99
100 }
101
102 virtual ~Filter(){}
103
104 bool next(const std::function<void(Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &entityBuffer)> &callback) Q_DECL_OVERRIDE {
105 bool foundValue = false;
106 while(!foundValue && mSource->next([this, callback, &foundValue](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &entityBuffer) {
107 SinkTrace() << "Filter: " << uid << operation;
108
109 //Always accept removals. They can't match the filter since the data is gone.
110 if (operation == Sink::Operation_Removal) {
111 SinkTrace() << "Removal: " << uid << operation;
112 callback(operation, uid, entityBuffer);
113 foundValue = true;
114 } else if (matchesFilter(uid, entityBuffer)) {
115 SinkTrace() << "Accepted: " << uid << operation;
116 callback(operation, uid, entityBuffer);
117 foundValue = true;
118 //TODO if something did not match the filter so far but does now, turn into an add operation.
119 } else {
120 SinkTrace() << "Rejected: " << uid << operation;
121 //TODO emit a removal if we had the uid in the result set and this is a modification.
122 //We don't know if this results in a removal from the dataset, so we emit a removal notification anyways
123 callback(Sink::Operation_Removal, uid, entityBuffer);
124 }
125 return false;
126 }))
127 {}
128 return foundValue;
129 }
130
131 bool matchesFilter(const QByteArray &uid, const Sink::EntityBuffer &entityBuffer) {
132 for (const auto &filterProperty : propertyFilter.keys()) {
133 const auto property = getProperty(entityBuffer.entity(), filterProperty);
134 const auto comparator = propertyFilter.value(filterProperty);
135 if (!comparator.matches(property)) {
136 SinkTrace() << "Filtering entity due to property mismatch on filter: " << filterProperty << property << ":" << comparator.value;
137 return false;
138 }
139 }
140 return true;
141 }
142};
143
144/* class Reduction : public FilterBase { */
145/* public: */
146/* typedef QSharedPointer<Reduction> Ptr; */
147
148/* QHash<QByteArray, QDateTime> aggregateValues; */
149
150/* Reduction(FilterBase::Ptr source, DataStoreQuery *store) */
151/* : FilterBase(source, store) */
152/* { */
153
154/* } */
155
156/* virtual ~Reduction(){} */
157
158/* bool next(const std::function<void(const QByteArray &uid, const Sink::EntityBuffer &entityBuffer)> &callback) Q_DECL_OVERRIDE { */
159/* bool foundValue = false; */
160/* while(!foundValue && mSource->next([this, callback, &foundValue](const QByteArray &uid, const Sink::EntityBuffer &entityBuffer) { */
161/* const auto operation = entityBuffer.operation(); */
162/* SinkTrace() << "Filter: " << uid << operation; */
163/* //Always accept removals. They can't match the filter since the data is gone. */
164/* if (operation == Sink::Operation_Removal) { */
165/* callback(uid, entityBuffer); */
166/* foundValue = true; */
167/* } else if (matchesFilter(uid, entityBuffer)) { */
168/* callback(uid, entityBuffer); */
169/* foundValue = true; */
170/* } */
171/* return false; */
172/* })) */
173/* {} */
174/* return foundValue; */
175/* } */
176
177/* bool matchesFilter(const QByteArray &uid, const Sink::EntityBuffer &entityBuffer) { */
178/* for (const auto &filterProperty : propertyFilter.keys()) { */
179/* const auto property = getProperty(entityBuffer.entity(), filterProperty); */
180/* const auto comparator = propertyFilter.value(filterProperty); */
181/* if (!comparator.matches(property)) { */
182/* SinkTrace() << "Filtering entity due to property mismatch on filter: " << filterProperty << property << ":" << comparator.value; */
183/* return false; */
184/* } */
185/* } */
186/* return true; */
187/* } */
188/* }; */
189
30DataStoreQuery::DataStoreQuery(const Sink::Query &query, const QByteArray &type, Sink::Storage::Transaction &transaction, TypeIndex &typeIndex, std::function<QVariant(const Sink::Entity &entity, const QByteArray &property)> getProperty) 190DataStoreQuery::DataStoreQuery(const Sink::Query &query, const QByteArray &type, Sink::Storage::Transaction &transaction, TypeIndex &typeIndex, std::function<QVariant(const Sink::Entity &entity, const QByteArray &property)> getProperty)
31 : mQuery(query), mTransaction(transaction), mType(type), mTypeIndex(typeIndex), mDb(Storage::mainDatabase(mTransaction, mType)), mGetProperty(getProperty) 191 : mQuery(query), mTransaction(transaction), mType(type), mTypeIndex(typeIndex), mDb(Storage::mainDatabase(mTransaction, mType)), mGetProperty(getProperty)
32{ 192{
33 193 setupQuery();
34} 194}
35 195
36static inline ResultSet fullScan(const Sink::Storage::Transaction &transaction, const QByteArray &bufferType) 196static inline QVector<QByteArray> fullScan(const Sink::Storage::Transaction &transaction, const QByteArray &bufferType)
37{ 197{
38 // TODO use a result set with an iterator, to read values on demand 198 // TODO use a result set with an iterator, to read values on demand
39 SinkTrace() << "Looking for : " << bufferType; 199 SinkTrace() << "Looking for : " << bufferType;
@@ -52,59 +212,7 @@ static inline ResultSet fullScan(const Sink::Storage::Transaction &transaction,
52 [](const Sink::Storage::Error &error) { SinkWarning() << "Error during query: " << error.message; }); 212 [](const Sink::Storage::Error &error) { SinkWarning() << "Error during query: " << error.message; });
53 213
54 SinkTrace() << "Full scan retrieved " << keys.size() << " results."; 214 SinkTrace() << "Full scan retrieved " << keys.size() << " results.";
55 return ResultSet(keys.toList().toVector()); 215 return keys.toList().toVector();
56}
57
58ResultSet DataStoreQuery::loadInitialResultSet(QSet<QByteArray> &remainingFilters, QByteArray &remainingSorting)
59{
60 if (!mQuery.ids.isEmpty()) {
61 return ResultSet(mQuery.ids.toVector());
62 }
63 QSet<QByteArray> appliedFilters;
64 QByteArray appliedSorting;
65
66 auto resultSet = mTypeIndex.query(mQuery, appliedFilters, appliedSorting, mTransaction);
67
68 remainingFilters = mQuery.propertyFilter.keys().toSet() - appliedFilters;
69 if (appliedSorting.isEmpty()) {
70 remainingSorting = mQuery.sortProperty;
71 }
72
73 // We do a full scan if there were no indexes available to create the initial set.
74 if (appliedFilters.isEmpty()) {
75 // TODO this should be replaced by an index lookup as well
76 resultSet = fullScan(mTransaction, mType);
77 }
78 return resultSet;
79}
80
81ResultSet DataStoreQuery::loadIncrementalResultSet(qint64 baseRevision, QSet<QByteArray> &remainingFilters)
82{
83 const auto bufferType = mType;
84 auto revisionCounter = QSharedPointer<qint64>::create(baseRevision);
85 remainingFilters = mQuery.propertyFilter.keys().toSet();
86 return ResultSet([this, bufferType, revisionCounter]() -> QByteArray {
87 const qint64 topRevision = Sink::Storage::maxRevision(mTransaction);
88 // Spit out the revision keys one by one.
89 while (*revisionCounter <= topRevision) {
90 const auto uid = Sink::Storage::getUidFromRevision(mTransaction, *revisionCounter);
91 const auto type = Sink::Storage::getTypeFromRevision(mTransaction, *revisionCounter);
92 // SinkTrace() << "Revision" << *revisionCounter << type << uid;
93 Q_ASSERT(!uid.isEmpty());
94 Q_ASSERT(!type.isEmpty());
95 if (type != bufferType) {
96 // Skip revision
97 *revisionCounter += 1;
98 continue;
99 }
100 const auto key = Sink::Storage::assembleKey(uid, *revisionCounter);
101 *revisionCounter += 1;
102 return key;
103 }
104 SinkTrace() << "Finished reading incremental result set:" << *revisionCounter;
105 // We're done
106 return QByteArray();
107 });
108} 216}
109 217
110void DataStoreQuery::readEntity(const QByteArray &key, const BufferCallback &resultCallback) 218void DataStoreQuery::readEntity(const QByteArray &key, const BufferCallback &resultCallback)
@@ -122,156 +230,194 @@ QVariant DataStoreQuery::getProperty(const Sink::Entity &entity, const QByteArra
122 return mGetProperty(entity, property); 230 return mGetProperty(entity, property);
123} 231}
124 232
125ResultSet DataStoreQuery::filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, const QByteArray &sortProperty) 233/* ResultSet DataStoreQuery::filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, const QByteArray &sortProperty) */
126{ 234/* { */
127 const bool sortingRequired = !sortProperty.isEmpty(); 235/* const bool sortingRequired = !sortProperty.isEmpty(); */
128 if (mInitialQuery && sortingRequired) { 236/* if (mInitialQuery && sortingRequired) { */
129 SinkTrace() << "Sorting the resultset in memory according to property: " << sortProperty; 237/* SinkTrace() << "Sorting the resultset in memory according to property: " << sortProperty; */
130 // Sort the complete set by reading the sort property and filling into a sorted map 238/* // Sort the complete set by reading the sort property and filling into a sorted map */
131 auto sortedMap = QSharedPointer<QMap<QByteArray, QByteArray>>::create(); 239/* auto sortedMap = QSharedPointer<QMap<QByteArray, QByteArray>>::create(); */
132 while (resultSet.next()) { 240/* while (resultSet.next()) { */
133 // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) 241/* // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) */
134 readEntity(resultSet.id(), 242/* readEntity(resultSet.id(), */
135 [this, filter, sortedMap, sortProperty, &resultSet](const QByteArray &uid, const Sink::EntityBuffer &buffer) { 243/* [this, filter, sortedMap, sortProperty, &resultSet](const QByteArray &uid, const Sink::EntityBuffer &buffer) { */
136
137 const auto operation = buffer.operation();
138
139 // We're not interested in removals during the initial query
140 if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) {
141 if (!sortProperty.isEmpty()) {
142 const auto sortValue = getProperty(buffer.entity(), sortProperty);
143 if (sortValue.type() == QVariant::DateTime) {
144 sortedMap->insert(QByteArray::number(std::numeric_limits<unsigned int>::max() - sortValue.toDateTime().toTime_t()), uid);
145 } else {
146 sortedMap->insert(sortValue.toString().toLatin1(), uid);
147 }
148 } else {
149 sortedMap->insert(uid, uid);
150 }
151 }
152 });
153 }
154 244
155 SinkTrace() << "Sorted " << sortedMap->size() << " values."; 245/* const auto operation = buffer.operation(); */
156 auto iterator = QSharedPointer<QMapIterator<QByteArray, QByteArray>>::create(*sortedMap);
157 ResultSet::ValueGenerator generator = [this, iterator, sortedMap, filter](
158 std::function<void(const QByteArray &uid, const Sink::EntityBuffer &entity, Sink::Operation)> callback) -> bool {
159 if (iterator->hasNext()) {
160 readEntity(iterator->next().value(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) {
161 callback(uid, buffer, Sink::Operation_Creation);
162 });
163 return true;
164 }
165 return false;
166 };
167 246
168 auto skip = [iterator]() { 247/* // We're not interested in removals during the initial query */
169 if (iterator->hasNext()) { 248/* if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) { */
170 iterator->next(); 249/* if (!sortProperty.isEmpty()) { */
171 } 250/* const auto sortValue = getProperty(buffer.entity(), sortProperty); */
172 }; 251/* if (sortValue.type() == QVariant::DateTime) { */
173 return ResultSet(generator, skip); 252/* sortedMap->insert(QByteArray::number(std::numeric_limits<unsigned int>::max() - sortValue.toDateTime().toTime_t()), uid); */
174 } else { 253/* } else { */
175 auto resultSetPtr = QSharedPointer<ResultSet>::create(resultSet); 254/* sortedMap->insert(sortValue.toString().toLatin1(), uid); */
176 ResultSet::ValueGenerator generator = [this, resultSetPtr, filter](const ResultSet::Callback &callback) -> bool { 255/* } */
177 if (resultSetPtr->next()) { 256/* } else { */
178 SinkTrace() << "Reading the next value: " << resultSetPtr->id(); 257/* sortedMap->insert(uid, uid); */
179 // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) 258/* } */
180 readEntity(resultSetPtr->id(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) { 259/* } */
181 const auto operation = buffer.operation(); 260/* }); */
182 if (mInitialQuery) { 261/* } */
183 // We're not interested in removals during the initial query
184 if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) {
185 // In the initial set every entity is new
186 callback(uid, buffer, Sink::Operation_Creation);
187 }
188 } else {
189 // Always remove removals, they probably don't match due to non-available properties
190 if ((operation == Sink::Operation_Removal) || filter(uid, buffer)) {
191 // TODO only replay if this is in the currently visible set (or just always replay, worst case we have a couple to many results)
192 callback(uid, buffer, operation);
193 }
194 }
195 });
196 return true;
197 }
198 return false;
199 };
200 auto skip = [resultSetPtr]() { resultSetPtr->skip(1); };
201 return ResultSet(generator, skip);
202 }
203}
204 262
263/* SinkTrace() << "Sorted " << sortedMap->size() << " values."; */
264/* auto iterator = QSharedPointer<QMapIterator<QByteArray, QByteArray>>::create(*sortedMap); */
265/* ResultSet::ValueGenerator generator = [this, iterator, sortedMap, filter]( */
266/* std::function<void(const QByteArray &uid, const Sink::EntityBuffer &entity, Sink::Operation)> callback) -> bool { */
267/* if (iterator->hasNext()) { */
268/* readEntity(iterator->next().value(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) { */
269/* callback(uid, buffer, Sink::Operation_Creation); */
270/* }); */
271/* return true; */
272/* } */
273/* return false; */
274/* }; */
205 275
206DataStoreQuery::FilterFunction DataStoreQuery::getFilter(const QSet<QByteArray> &remainingFilters) 276/* auto skip = [iterator]() { */
277/* if (iterator->hasNext()) { */
278/* iterator->next(); */
279/* } */
280/* }; */
281/* return ResultSet(generator, skip); */
282/* } else { */
283/* auto resultSetPtr = QSharedPointer<ResultSet>::create(resultSet); */
284/* ResultSet::ValueGenerator generator = [this, resultSetPtr, filter](const ResultSet::Callback &callback) -> bool { */
285/* if (resultSetPtr->next()) { */
286/* SinkTrace() << "Reading the next value: " << resultSetPtr->id(); */
287/* // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) */
288/* readEntity(resultSetPtr->id(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) { */
289/* const auto operation = buffer.operation(); */
290/* if (mInitialQuery) { */
291/* // We're not interested in removals during the initial query */
292/* if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) { */
293/* // In the initial set every entity is new */
294/* callback(uid, buffer, Sink::Operation_Creation); */
295/* } */
296/* } else { */
297/* // Always remove removals, they probably don't match due to non-available properties */
298/* if ((operation == Sink::Operation_Removal) || filter(uid, buffer)) { */
299/* // TODO only replay if this is in the currently visible set (or just always replay, worst case we have a couple to many results) */
300/* callback(uid, buffer, operation); */
301/* } */
302/* } */
303/* }); */
304/* return true; */
305/* } */
306/* return false; */
307/* }; */
308/* auto skip = [resultSetPtr]() { resultSetPtr->skip(1); }; */
309/* return ResultSet(generator, skip); */
310/* } */
311/* } */
312
313void DataStoreQuery::setupQuery()
207{ 314{
208 auto query = mQuery; 315 FilterBase::Ptr baseSet;
209 return [this, remainingFilters, query](const QByteArray &uid, const Sink::EntityBuffer &entity) -> bool { 316 QSet<QByteArray> remainingFilters;
210 if (!query.ids.isEmpty()) { 317 QByteArray appliedSorting;
211 if (!query.ids.contains(uid)) { 318 if (!mQuery.ids.isEmpty()) {
212 SinkTrace() << "Filter by uid: " << uid; 319 mSource = Source::Ptr::create(mQuery.ids.toVector(), this);
213 return false; 320 baseSet = mSource;
214 } 321 remainingFilters = mQuery.propertyFilter.keys().toSet();
215 } 322 } else {
216 for (const auto &filterProperty : remainingFilters) { 323 QSet<QByteArray> appliedFilters;
217 const auto property = getProperty(entity.entity(), filterProperty); 324
218 const auto comparator = query.propertyFilter.value(filterProperty); 325 auto resultSet = mTypeIndex.query(mQuery, appliedFilters, appliedSorting, mTransaction);
219 if (!comparator.matches(property)) { 326 remainingFilters = mQuery.propertyFilter.keys().toSet() - appliedFilters;
220 SinkTrace() << "Filtering entity due to property mismatch on filter: " << filterProperty << property << ":" << comparator.value; 327
221 return false; 328 // We do a full scan if there were no indexes available to create the initial set.
222 } 329 if (appliedFilters.isEmpty()) {
330 // TODO this should be replaced by an index lookup on the uid index
331 mSource = Source::Ptr::create(fullScan(mTransaction, mType), this);
332 } else {
333 mSource = Source::Ptr::create(resultSet, this);
223 } 334 }
224 return true; 335 baseSet = mSource;
225 }; 336 }
226} 337 if (!mQuery.propertyFilter.isEmpty()) {
338 auto filter = Filter::Ptr::create(baseSet, this);
339 filter->propertyFilter = mQuery.propertyFilter;
340 /* for (const auto &f : remainingFilters) { */
341 /* filter->propertyFilter.insert(f, mQuery.propertyFilter.value(f)); */
342 /* } */
343 baseSet = filter;
344 }
345 /* if (appliedSorting.isEmpty() && !mQuery.sortProperty.isEmpty()) { */
346 /* //Apply manual sorting */
347 /* baseSet = Sort::Ptr::create(baseSet, mQuery.sortProperty); */
348 /* } */
227 349
228ResultSet DataStoreQuery::createFilteredSet(ResultSet &resultSet, const std::function<bool(const QByteArray &, const Sink::EntityBuffer &buffer)> &filter) 350 /* if (mQuery.threadLeaderOnly) { */
229{ 351 /* auto reduce = Reduce::Ptr::create(baseSet, this); */
230 auto resultSetPtr = QSharedPointer<ResultSet>::create(resultSet); 352
231 ResultSet::ValueGenerator generator = [this, resultSetPtr, filter](const ResultSet::Callback &callback) -> bool { 353 /* baseSet = reduce; */
232 return resultSetPtr->next([=](const QByteArray &uid, const Sink::EntityBuffer &buffer, Sink::Operation operation) { 354 /* } */
233 if (mInitialQuery) { 355
234 // We're not interested in removals during the initial query 356 mCollector = Collector::Ptr::create(baseSet, this);
235 if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) {
236 // In the initial set every entity is new
237 callback(uid, buffer, Sink::Operation_Creation);
238 }
239 } else {
240 // Always remove removals, they probably don't match due to non-available properties
241 if ((operation == Sink::Operation_Removal) || filter(uid, buffer)) {
242 // TODO only replay if this is in the currently visible set (or just always replay, worst case we have a couple to many results)
243 callback(uid, buffer, operation);
244 }
245 }
246 });
247 };
248 auto skip = [resultSetPtr]() { resultSetPtr->skip(1); };
249 return ResultSet(generator, skip);
250} 357}
251 358
252ResultSet DataStoreQuery::postSortFilter(ResultSet &resultSet) 359QVector<QByteArray> DataStoreQuery::loadIncrementalResultSet(qint64 baseRevision)
253{ 360{
254 return resultSet; 361 const auto bufferType = mType;
362 auto revisionCounter = QSharedPointer<qint64>::create(baseRevision);
363 QVector<QByteArray> changedKeys;
364 const qint64 topRevision = Sink::Storage::maxRevision(mTransaction);
365 // Spit out the revision keys one by one.
366 while (*revisionCounter <= topRevision) {
367 const auto uid = Sink::Storage::getUidFromRevision(mTransaction, *revisionCounter);
368 const auto type = Sink::Storage::getTypeFromRevision(mTransaction, *revisionCounter);
369 // SinkTrace() << "Revision" << *revisionCounter << type << uid;
370 Q_ASSERT(!uid.isEmpty());
371 Q_ASSERT(!type.isEmpty());
372 if (type != bufferType) {
373 // Skip revision
374 *revisionCounter += 1;
375 continue;
376 }
377 const auto key = Sink::Storage::assembleKey(uid, *revisionCounter);
378 *revisionCounter += 1;
379 changedKeys << key;
380 }
381 SinkTrace() << "Finished reading incremental result set:" << *revisionCounter;
382 return changedKeys;
255} 383}
256 384
385
257ResultSet DataStoreQuery::update(qint64 baseRevision) 386ResultSet DataStoreQuery::update(qint64 baseRevision)
258{ 387{
259 SinkTrace() << "Executing query update"; 388 SinkTrace() << "Executing query update";
260 mInitialQuery = false; 389 auto incrementalResultSet = loadIncrementalResultSet(baseRevision);
261 QSet<QByteArray> remainingFilters; 390 SinkTrace() << "Changed: " << incrementalResultSet;
262 QByteArray remainingSorting; 391 mSource->add(incrementalResultSet);
263 auto resultSet = loadIncrementalResultSet(baseRevision, remainingFilters); 392 ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool {
264 auto filteredSet = filterAndSortSet(resultSet, getFilter(remainingFilters), remainingSorting); 393 if (mCollector->next([callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) {
265 return postSortFilter(filteredSet); 394 SinkTrace() << "Got incremental result: " << uid << operation;
395 callback(uid, buffer, operation);
396 }))
397 {
398 return true;
399 }
400 return false;
401 };
402 return ResultSet(generator, [this]() { mCollector->skip(); });
266} 403}
267 404
405
268ResultSet DataStoreQuery::execute() 406ResultSet DataStoreQuery::execute()
269{ 407{
270 SinkTrace() << "Executing query"; 408 SinkTrace() << "Executing query";
271 mInitialQuery = true; 409
272 QSet<QByteArray> remainingFilters; 410 ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool {
273 QByteArray remainingSorting; 411 if (mCollector->next([callback](Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &buffer) {
274 auto resultSet = loadInitialResultSet(remainingFilters, remainingSorting); 412 if (operation != Sink::Operation_Removal) {
275 auto filteredSet = filterAndSortSet(resultSet, getFilter(remainingFilters), remainingSorting); 413 SinkTrace() << "Got initial result: " << uid << operation;
276 return postSortFilter(filteredSet); 414 callback(uid, buffer, Sink::Operation_Creation);
415 }
416 }))
417 {
418 return true;
419 }
420 return false;
421 };
422 return ResultSet(generator, [this]() { mCollector->skip(); });
277} 423}
diff --git a/common/datastorequery.h b/common/datastorequery.h
index 7712ac7..c9f6a3a 100644
--- a/common/datastorequery.h
+++ b/common/datastorequery.h
@@ -25,7 +25,12 @@
25#include "query.h" 25#include "query.h"
26#include "entitybuffer.h" 26#include "entitybuffer.h"
27 27
28
29class Source;
30class FilterBase;
31
28class DataStoreQuery { 32class DataStoreQuery {
33 friend class FilterBase;
29public: 34public:
30 typedef QSharedPointer<DataStoreQuery> Ptr; 35 typedef QSharedPointer<DataStoreQuery> Ptr;
31 36
@@ -42,14 +47,17 @@ protected:
42 47
43 virtual void readEntity(const QByteArray &key, const BufferCallback &resultCallback); 48 virtual void readEntity(const QByteArray &key, const BufferCallback &resultCallback);
44 49
45 virtual ResultSet loadInitialResultSet(QSet<QByteArray> &remainingFilters, QByteArray &remainingSorting); 50 /* virtual ResultSet loadInitialResultSet(QSet<QByteArray> &remainingFilters, QByteArray &remainingSorting); */
46 virtual ResultSet loadIncrementalResultSet(qint64 baseRevision, QSet<QByteArray> &remainingFilters); 51 /* virtual ResultSet loadIncrementalResultSet(qint64 baseRevision, QSet<QByteArray> &remainingFilters); */
47 52
48 virtual ResultSet filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, const QByteArray &sortProperty); 53 /* virtual ResultSet filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, const QByteArray &sortProperty); */
49 virtual ResultSet postSortFilter(ResultSet &resultSet); 54 /* virtual ResultSet postSortFilter(ResultSet &resultSet); */
50 virtual FilterFunction getFilter(const QSet<QByteArray> &remainingFilters); 55 /* virtual FilterFunction getFilter(const QSet<QByteArray> &remainingFilters); */
51 56
52 ResultSet createFilteredSet(ResultSet &resultSet, const std::function<bool(const QByteArray &, const Sink::EntityBuffer &buffer)> &); 57 ResultSet createFilteredSet(ResultSet &resultSet, const std::function<bool(const QByteArray &, const Sink::EntityBuffer &buffer)> &);
58 QVector<QByteArray> loadIncrementalResultSet(qint64 baseRevision);
59
60 void setupQuery();
53 61
54 Sink::Query mQuery; 62 Sink::Query mQuery;
55 Sink::Storage::Transaction &mTransaction; 63 Sink::Storage::Transaction &mTransaction;
@@ -58,8 +66,64 @@ protected:
58 Sink::Storage::NamedDatabase mDb; 66 Sink::Storage::NamedDatabase mDb;
59 std::function<QVariant(const Sink::Entity &entity, const QByteArray &property)> mGetProperty; 67 std::function<QVariant(const Sink::Entity &entity, const QByteArray &property)> mGetProperty;
60 bool mInitialQuery; 68 bool mInitialQuery;
69 QSharedPointer<FilterBase> mCollector;
70 QSharedPointer<Source> mSource;
71};
72
73
74class FilterBase {
75public:
76 typedef QSharedPointer<FilterBase> Ptr;
77 FilterBase(DataStoreQuery *store)
78 : mDatastore(store)
79 {
80
81 }
82
83 FilterBase(FilterBase::Ptr source, DataStoreQuery *store)
84 : mSource(source),
85 mDatastore(store)
86 {
87 }
88
89 virtual ~FilterBase(){}
90
91 void readEntity(const QByteArray &key, const std::function<void(const QByteArray &, const Sink::EntityBuffer &buffer)> &callback)
92 {
93 Q_ASSERT(mDatastore);
94 mDatastore->readEntity(key, callback);
95 }
96
97 QVariant getProperty(const Sink::Entity &entity, const QByteArray &property)
98 {
99 Q_ASSERT(mDatastore);
100 return mDatastore->getProperty(entity, property);
101 }
102
103 virtual void skip() { mSource->skip(); };
104
105 //Returns true for as long as a result is available
106 virtual bool next(const std::function<void(Sink::Operation operation, const QByteArray &uid, const Sink::EntityBuffer &entityBuffer)> &callback) = 0;
107
108 QSharedPointer<FilterBase> mSource;
109 DataStoreQuery *mDatastore;
61}; 110};
62 111
112/* class Reduce { */
113/* QByteArray property; */
114
115/* //Property - value, reduction result */
116/* QHash<QVariant, QVariant> mReducedValue; */
117/* }; */
118
119/* class Bloom { */
120/* QByteArray property; */
121
122/* //Property - value, reduction result */
123/* QSet<QVariant> mPropertyValues; */
124
125/* }; */
126
63 127
64 128
65 129
diff --git a/common/domain/event.cpp b/common/domain/event.cpp
index 118ffa3..f3abd62 100644
--- a/common/domain/event.cpp
+++ b/common/domain/event.cpp
@@ -33,6 +33,7 @@
33#include "../definitions.h" 33#include "../definitions.h"
34#include "../typeindex.h" 34#include "../typeindex.h"
35#include "entitybuffer.h" 35#include "entitybuffer.h"
36#include "datastorequery.h"
36#include "entity_generated.h" 37#include "entity_generated.h"
37 38
38#include "event_generated.h" 39#include "event_generated.h"
diff --git a/common/domain/event.h b/common/domain/event.h
index e1ca061..684b58e 100644
--- a/common/domain/event.h
+++ b/common/domain/event.h
@@ -21,7 +21,6 @@
21#include "applicationdomaintype.h" 21#include "applicationdomaintype.h"
22 22
23#include "storage.h" 23#include "storage.h"
24#include "datastorequery.h"
25 24
26class ResultSet; 25class ResultSet;
27class QByteArray; 26class QByteArray;
@@ -31,6 +30,8 @@ class ReadPropertyMapper;
31template<typename T> 30template<typename T>
32class WritePropertyMapper; 31class WritePropertyMapper;
33 32
33class DataStoreQuery;
34
34namespace Sink { 35namespace Sink {
35 class Query; 36 class Query;
36 37
@@ -51,7 +52,7 @@ public:
51 typedef Sink::ApplicationDomain::Buffer::Event Buffer; 52 typedef Sink::ApplicationDomain::Buffer::Event Buffer;
52 typedef Sink::ApplicationDomain::Buffer::EventBuilder BufferBuilder; 53 typedef Sink::ApplicationDomain::Buffer::EventBuilder BufferBuilder;
53 static QSet<QByteArray> indexedProperties(); 54 static QSet<QByteArray> indexedProperties();
54 static DataStoreQuery::Ptr prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction); 55 static QSharedPointer<DataStoreQuery> prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction);
55 static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); 56 static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
56 static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); 57 static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
57 static QSharedPointer<ReadPropertyMapper<Buffer> > initializeReadPropertyMapper(); 58 static QSharedPointer<ReadPropertyMapper<Buffer> > initializeReadPropertyMapper();
diff --git a/common/domain/folder.cpp b/common/domain/folder.cpp
index 17d9f13..824fa0b 100644
--- a/common/domain/folder.cpp
+++ b/common/domain/folder.cpp
@@ -33,6 +33,7 @@
33#include "../definitions.h" 33#include "../definitions.h"
34#include "../typeindex.h" 34#include "../typeindex.h"
35#include "entitybuffer.h" 35#include "entitybuffer.h"
36#include "datastorequery.h"
36#include "entity_generated.h" 37#include "entity_generated.h"
37 38
38#include "folder_generated.h" 39#include "folder_generated.h"
diff --git a/common/domain/folder.h b/common/domain/folder.h
index ff87006..e4631de 100644
--- a/common/domain/folder.h
+++ b/common/domain/folder.h
@@ -21,10 +21,10 @@
21#include "applicationdomaintype.h" 21#include "applicationdomaintype.h"
22 22
23#include "storage.h" 23#include "storage.h"
24#include "datastorequery.h"
25 24
26class ResultSet; 25class ResultSet;
27class QByteArray; 26class QByteArray;
27class DataStoreQuery;
28 28
29template<typename T> 29template<typename T>
30class ReadPropertyMapper; 30class ReadPropertyMapper;
@@ -45,7 +45,7 @@ class TypeImplementation<Sink::ApplicationDomain::Folder> {
45public: 45public:
46 typedef Sink::ApplicationDomain::Buffer::Folder Buffer; 46 typedef Sink::ApplicationDomain::Buffer::Folder Buffer;
47 typedef Sink::ApplicationDomain::Buffer::FolderBuilder BufferBuilder; 47 typedef Sink::ApplicationDomain::Buffer::FolderBuilder BufferBuilder;
48 static DataStoreQuery::Ptr prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction); 48 static QSharedPointer<DataStoreQuery> prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction);
49 static QSet<QByteArray> indexedProperties(); 49 static QSet<QByteArray> indexedProperties();
50 static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); 50 static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
51 static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); 51 static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
diff --git a/common/domain/mail.cpp b/common/domain/mail.cpp
index 0c737fa..483a2f2 100644
--- a/common/domain/mail.cpp
+++ b/common/domain/mail.cpp
@@ -33,6 +33,7 @@
33#include "../definitions.h" 33#include "../definitions.h"
34#include "../typeindex.h" 34#include "../typeindex.h"
35#include "entitybuffer.h" 35#include "entitybuffer.h"
36#include "datastorequery.h"
36#include "entity_generated.h" 37#include "entity_generated.h"
37 38
38#include "mail_generated.h" 39#include "mail_generated.h"
@@ -210,68 +211,16 @@ QSharedPointer<WritePropertyMapper<TypeImplementation<Mail>::BufferBuilder> > Ty
210 return propertyMapper; 211 return propertyMapper;
211} 212}
212 213
213class ThreadedDataStoreQuery : public DataStoreQuery
214{
215public:
216 typedef QSharedPointer<ThreadedDataStoreQuery> Ptr;
217 using DataStoreQuery::DataStoreQuery;
218
219protected:
220 ResultSet postSortFilter(ResultSet &resultSet) Q_DECL_OVERRIDE
221 {
222 auto query = mQuery;
223 if (query.threadLeaderOnly) {
224 auto rootCollection = QSharedPointer<QMap<QByteArray, QDateTime>>::create();
225 auto filter = [this, query, rootCollection](const QByteArray &uid, const Sink::EntityBuffer &entity) -> bool {
226 //TODO lookup thread
227 //if we got thread already in the result set compare dates and if newer replace
228 //else insert
229
230 const auto messageId = getProperty(entity.entity(), ApplicationDomain::Mail::MessageId::name).toByteArray();
231
232 Index msgIdIndex("msgId", mTransaction);
233 Index msgIdThreadIdIndex("msgIdThreadId", mTransaction);
234 auto thread = msgIdThreadIdIndex.lookup(messageId);
235 SinkTrace() << "MsgId: " << messageId << " Thread: " << thread << getProperty(entity.entity(), ApplicationDomain::Mail::Date::name).toDateTime();
236
237 if (rootCollection->contains(thread)) {
238 auto date = rootCollection->value(thread);
239 //The mail we have in our result already is newer, so we can ignore this one
240 //This is always true during the initial query if the set has been sorted by date.
241 if (date > getProperty(entity.entity(), ApplicationDomain::Mail::Date::name).toDateTime()) {
242 return false;
243 }
244 qWarning() << "############################################################################";
245 qWarning() << "Found a newer mail, remove the old one";
246 qWarning() << "############################################################################";
247 }
248 rootCollection->insert(thread, getProperty(entity.entity(), ApplicationDomain::Mail::Date::name).toDateTime());
249 return true;
250 };
251 return createFilteredSet(resultSet, filter);
252 } else {
253 return resultSet;
254 }
255 }
256};
257 214
258DataStoreQuery::Ptr TypeImplementation<Mail>::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction) 215DataStoreQuery::Ptr TypeImplementation<Mail>::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction)
259{ 216{
260 if (query.threadLeaderOnly) {
261 auto mapper = initializeReadPropertyMapper();
262 return ThreadedDataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName<Mail>(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) {
263 217
264 const auto localBuffer = Sink::EntityBuffer::readBuffer<Buffer>(entity.local());
265 return mapper->getProperty(property, localBuffer);
266 });
267 218
268 } else {
269 auto mapper = initializeReadPropertyMapper(); 219 auto mapper = initializeReadPropertyMapper();
270 return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName<Mail>(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) { 220 return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName<Mail>(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) {
271 221
272 const auto localBuffer = Sink::EntityBuffer::readBuffer<Buffer>(entity.local()); 222 const auto localBuffer = Sink::EntityBuffer::readBuffer<Buffer>(entity.local());
273 return mapper->getProperty(property, localBuffer); 223 return mapper->getProperty(property, localBuffer);
274 }); 224 });
275 }
276} 225}
277 226
diff --git a/common/domain/mail.h b/common/domain/mail.h
index 3b0e9da..ea3ef9e 100644
--- a/common/domain/mail.h
+++ b/common/domain/mail.h
@@ -25,6 +25,7 @@
25 25
26class ResultSet; 26class ResultSet;
27class QByteArray; 27class QByteArray;
28class DataStoreQuery;
28 29
29template<typename T> 30template<typename T>
30class ReadPropertyMapper; 31class ReadPropertyMapper;
@@ -45,7 +46,7 @@ class TypeImplementation<Sink::ApplicationDomain::Mail> {
45public: 46public:
46 typedef Sink::ApplicationDomain::Buffer::Mail Buffer; 47 typedef Sink::ApplicationDomain::Buffer::Mail Buffer;
47 typedef Sink::ApplicationDomain::Buffer::MailBuilder BufferBuilder; 48 typedef Sink::ApplicationDomain::Buffer::MailBuilder BufferBuilder;
48 static DataStoreQuery::Ptr prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction); 49 static QSharedPointer<DataStoreQuery> prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction);
49 static QSet<QByteArray> indexedProperties(); 50 static QSet<QByteArray> indexedProperties();
50 static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); 51 static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
51 static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); 52 static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
diff --git a/common/entityreader.cpp b/common/entityreader.cpp
index d86f4a9..bd973d0 100644
--- a/common/entityreader.cpp
+++ b/common/entityreader.cpp
@@ -22,6 +22,7 @@
22#include "resultset.h" 22#include "resultset.h"
23#include "storage.h" 23#include "storage.h"
24#include "query.h" 24#include "query.h"
25#include "datastorequery.h"
25 26
26SINK_DEBUG_AREA("entityreader") 27SINK_DEBUG_AREA("entityreader")
27 28
diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp
index 0534338..ec5748f 100644
--- a/common/mailpreprocessor.cpp
+++ b/common/mailpreprocessor.cpp
@@ -120,14 +120,18 @@ void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail, Sink:
120{ 120{
121 MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(mail.getMimeMessagePath())); 121 MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(mail.getMimeMessagePath()));
122 auto msg = mimeMessageReader.mimeMessage(); 122 auto msg = mimeMessageReader.mimeMessage();
123 updatedIndexedProperties(mail, msg); 123 if (msg) {
124 updatedIndexedProperties(mail, msg);
125 }
124} 126}
125 127
126void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail,Sink::Storage::Transaction &transaction) 128void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail,Sink::Storage::Transaction &transaction)
127{ 129{
128 MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(newMail.getMimeMessagePath())); 130 MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(newMail.getMimeMessagePath()));
129 auto msg = mimeMessageReader.mimeMessage(); 131 auto msg = mimeMessageReader.mimeMessage();
130 updatedIndexedProperties(newMail, msg); 132 if (msg) {
133 updatedIndexedProperties(newMail, msg);
134 }
131} 135}
132 136
133 137
diff --git a/common/modelresult.cpp b/common/modelresult.cpp
index 56a39ee..d13bba9 100644
--- a/common/modelresult.cpp
+++ b/common/modelresult.cpp
@@ -188,7 +188,7 @@ void ModelResult<T, Ptr>::add(const Ptr &value)
188 return; 188 return;
189 } 189 }
190 auto parent = createIndexFromId(id); 190 auto parent = createIndexFromId(id);
191 // qDebug() << "Added entity " << childId << value->identifier() << id; 191 SinkTrace() << "Added entity " << childId << value->identifier() << id;
192 const auto keys = mTree[id]; 192 const auto keys = mTree[id];
193 int index = 0; 193 int index = 0;
194 for (; index < keys.size(); index++) { 194 for (; index < keys.size(); index++) {
@@ -200,13 +200,13 @@ void ModelResult<T, Ptr>::add(const Ptr &value)
200 SinkWarning() << "Entity already in model " << value->identifier(); 200 SinkWarning() << "Entity already in model " << value->identifier();
201 return; 201 return;
202 } 202 }
203 // qDebug() << "Inserting rows " << index << parent; 203 // SinkTrace() << "Inserting rows " << index << parent;
204 beginInsertRows(parent, index, index); 204 beginInsertRows(parent, index, index);
205 mEntities.insert(childId, value); 205 mEntities.insert(childId, value);
206 mTree[id].insert(index, childId); 206 mTree[id].insert(index, childId);
207 mParents.insert(childId, id); 207 mParents.insert(childId, id);
208 endInsertRows(); 208 endInsertRows();
209 // qDebug() << "Inserted rows " << mTree[id].size(); 209 // SinkTrace() << "Inserted rows " << mTree[id].size();
210} 210}
211 211
212 212
@@ -216,7 +216,7 @@ void ModelResult<T, Ptr>::remove(const Ptr &value)
216 auto childId = qHash(*value); 216 auto childId = qHash(*value);
217 auto id = parentId(value); 217 auto id = parentId(value);
218 auto parent = createIndexFromId(id); 218 auto parent = createIndexFromId(id);
219 // qDebug() << "Removed entity" << childId; 219 SinkTrace() << "Removed entity" << childId;
220 auto index = mTree[id].indexOf(childId); 220 auto index = mTree[id].indexOf(childId);
221 beginRemoveRows(parent, index, index); 221 beginRemoveRows(parent, index, index);
222 mEntities.remove(childId); 222 mEntities.remove(childId);
@@ -259,6 +259,7 @@ void ModelResult<T, Ptr>::setEmitter(const typename Sink::ResultEmitter<Ptr>::Pt
259 }); 259 });
260 }); 260 });
261 emitter->onModified([this](const Ptr &value) { 261 emitter->onModified([this](const Ptr &value) {
262 SinkTrace() << "Received modification: " << value->identifier();
262 threadBoundary.callInMainThread([this, value]() { 263 threadBoundary.callInMainThread([this, value]() {
263 modify(value); 264 modify(value);
264 }); 265 });
@@ -294,8 +295,9 @@ void ModelResult<T, Ptr>::modify(const Ptr &value)
294 return; 295 return;
295 } 296 }
296 auto parent = createIndexFromId(id); 297 auto parent = createIndexFromId(id);
297 // qDebug() << "Modified entity" << childId; 298 SinkTrace() << "Modified entity" << childId;
298 auto i = mTree[id].indexOf(childId); 299 auto i = mTree[id].indexOf(childId);
300 Q_ASSERT(i >= 0);
299 mEntities.remove(childId); 301 mEntities.remove(childId);
300 mEntities.insert(childId, value); 302 mEntities.insert(childId, value);
301 // TODO check for change of parents 303 // TODO check for change of parents
diff --git a/common/query.cpp b/common/query.cpp
index fd99367..3de80d8 100644
--- a/common/query.cpp
+++ b/common/query.cpp
@@ -51,6 +51,9 @@ bool Query::Comparator::matches(const QVariant &v) const
51 switch(comparator) { 51 switch(comparator) {
52 case Equals: 52 case Equals:
53 if (!v.isValid()) { 53 if (!v.isValid()) {
54 if (!value.isValid()) {
55 return true;
56 }
54 return false; 57 return false;
55 } 58 }
56 return v == value; 59 return v == value;
diff --git a/common/typeindex.cpp b/common/typeindex.cpp
index b96c5b5..1b04966 100644
--- a/common/typeindex.cpp
+++ b/common/typeindex.cpp
@@ -87,7 +87,7 @@ template <>
87void TypeIndex::addProperty<QDateTime>(const QByteArray &property) 87void TypeIndex::addProperty<QDateTime>(const QByteArray &property)
88{ 88{
89 auto indexer = [this, property](const QByteArray &identifier, const QVariant &value, Sink::Storage::Transaction &transaction) { 89 auto indexer = [this, property](const QByteArray &identifier, const QVariant &value, Sink::Storage::Transaction &transaction) {
90 // SinkTrace() << "Indexing " << mType + ".index." + property << date.toString(); 90 //SinkTrace() << "Indexing " << mType + ".index." + property << getByteArray(value);
91 Index(indexName(property), transaction).add(getByteArray(value), identifier); 91 Index(indexName(property), transaction).add(getByteArray(value), identifier);
92 }; 92 };
93 mIndexer.insert(property, indexer); 93 mIndexer.insert(property, indexer);
@@ -138,7 +138,7 @@ void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDoma
138 } 138 }
139} 139}
140 140
141ResultSet TypeIndex::query(const Sink::Query &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction) 141QVector<QByteArray> TypeIndex::query(const Sink::Query &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction)
142{ 142{
143 QVector<QByteArray> keys; 143 QVector<QByteArray> keys;
144 for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { 144 for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) {
@@ -151,7 +151,7 @@ ResultSet TypeIndex::query(const Sink::Query &query, QSet<QByteArray> &appliedFi
151 appliedFilters << it.key(); 151 appliedFilters << it.key();
152 appliedSorting = it.value(); 152 appliedSorting = it.value();
153 SinkTrace() << "Index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys."; 153 SinkTrace() << "Index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys.";
154 return ResultSet(keys); 154 return keys;
155 } 155 }
156 } 156 }
157 for (const auto &property : mProperties) { 157 for (const auto &property : mProperties) {
@@ -162,9 +162,9 @@ ResultSet TypeIndex::query(const Sink::Query &query, QSet<QByteArray> &appliedFi
162 lookupKey, [&](const QByteArray &value) { keys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); 162 lookupKey, [&](const QByteArray &value) { keys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; });
163 appliedFilters << property; 163 appliedFilters << property;
164 SinkTrace() << "Index lookup on " << property << " found " << keys.size() << " keys."; 164 SinkTrace() << "Index lookup on " << property << " found " << keys.size() << " keys.";
165 return ResultSet(keys); 165 return keys;
166 } 166 }
167 } 167 }
168 SinkTrace() << "No matching index"; 168 SinkTrace() << "No matching index";
169 return ResultSet(keys); 169 return keys;
170} 170}
diff --git a/common/typeindex.h b/common/typeindex.h
index a16179c..f5a32b9 100644
--- a/common/typeindex.h
+++ b/common/typeindex.h
@@ -37,7 +37,7 @@ public:
37 void add(const QByteArray &identifier, const Sink::ApplicationDomain::BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); 37 void add(const QByteArray &identifier, const Sink::ApplicationDomain::BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
38 void remove(const QByteArray &identifier, const Sink::ApplicationDomain::BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); 38 void remove(const QByteArray &identifier, const Sink::ApplicationDomain::BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
39 39
40 ResultSet query(const Sink::Query &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction); 40 QVector<QByteArray> query(const Sink::Query &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction);
41 41
42private: 42private:
43 QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const; 43 QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const;
diff --git a/examples/maildirresource/tests/maildirthreadtest.cpp b/examples/maildirresource/tests/maildirthreadtest.cpp
index a74869c..69d51a6 100644
--- a/examples/maildirresource/tests/maildirthreadtest.cpp
+++ b/examples/maildirresource/tests/maildirthreadtest.cpp
@@ -25,7 +25,7 @@
25#include "common/test.h" 25#include "common/test.h"
26#include "common/domain/applicationdomaintype.h" 26#include "common/domain/applicationdomaintype.h"
27 27
28#include "utils.h"; 28#include "utils.h"
29 29
30using namespace Sink; 30using namespace Sink;
31using namespace Sink::ApplicationDomain; 31using namespace Sink::ApplicationDomain;
diff --git a/tests/querytest.cpp b/tests/querytest.cpp
index d72dc7d..ab2a7e5 100644
--- a/tests/querytest.cpp
+++ b/tests/querytest.cpp
@@ -12,6 +12,8 @@
12#include "test.h" 12#include "test.h"
13#include "testutils.h" 13#include "testutils.h"
14 14
15using namespace Sink::ApplicationDomain;
16
15/** 17/**
16 * Test of the query system using the dummy resource. 18 * Test of the query system using the dummy resource.
17 * 19 *
@@ -97,6 +99,46 @@ private slots:
97 QCOMPARE(model->rowCount(), 1); 99 QCOMPARE(model->rowCount(), 1);
98 } 100 }
99 101
102 void testFilter()
103 {
104 // Setup
105 {
106 Mail mail("sink.dummy.instance1");
107 mail.setUid("test1");
108 mail.setFolder("folder1");
109 Sink::Store::create<Mail>(mail).exec().waitForFinished();
110 }
111 {
112 Mail mail("sink.dummy.instance1");
113 mail.setUid("test2");
114 mail.setFolder("folder2");
115 Sink::Store::create<Mail>(mail).exec().waitForFinished();
116 }
117
118 // Test
119 Sink::Query query;
120 query.resources << "sink.dummy.instance1";
121 query.liveQuery = true;
122 query.filter<Mail::Folder>("folder1");
123
124 // We fetch before the data is available and rely on the live query mechanism to deliver the actual data
125 auto model = Sink::Store::loadModel<Mail>(query);
126 QTRY_COMPARE(model->rowCount(), 1);
127
128 auto mail = model->index(0, 0, QModelIndex()).data(Sink::Store::DomainObjectRole).value<Sink::ApplicationDomain::Mail::Ptr>();
129 {
130 mail->setFolder("folder2");
131 Sink::Store::modify<Mail>(*mail).exec().waitForFinished();
132 }
133 QTRY_COMPARE(model->rowCount(), 0);
134
135 {
136 mail->setFolder("folder1");
137 Sink::Store::modify<Mail>(*mail).exec().waitForFinished();
138 }
139 QTRY_COMPARE(model->rowCount(), 1);
140 }
141
100 void testById() 142 void testById()
101 { 143 {
102 QByteArray id; 144 QByteArray id;
@@ -200,6 +242,13 @@ private slots:
200 Sink::Store::create<Sink::ApplicationDomain::Mail>(mail).exec().waitForFinished(); 242 Sink::Store::create<Sink::ApplicationDomain::Mail>(mail).exec().waitForFinished();
201 } 243 }
202 244
245 {
246 Sink::ApplicationDomain::Mail mail("sink.dummy.instance1");
247 mail.setProperty("uid", "test2");
248 mail.setProperty("sender", "doe@example.org");
249 Sink::Store::create<Sink::ApplicationDomain::Mail>(mail).exec().waitForFinished();
250 }
251
203 // Test 252 // Test
204 Sink::Query query; 253 Sink::Query query;
205 query.resources << "sink.dummy.instance1"; 254 query.resources << "sink.dummy.instance1";
@@ -256,6 +305,61 @@ private slots:
256 QCOMPARE(model->rowCount(), 1); 305 QCOMPARE(model->rowCount(), 1);
257 } 306 }
258 307
308 /*
309 * Filter by two properties to make sure that we also use a non-index based filter.
310 */
311 void testMailByUidAndFolder()
312 {
313 // Setup
314 Folder::Ptr folderEntity;
315 {
316 Folder folder("sink.dummy.instance1");
317 Sink::Store::create<Folder>(folder).exec().waitForFinished();
318
319 Sink::Query query;
320 query.resources << "sink.dummy.instance1";
321
322 // Ensure all local data is processed
323 Sink::ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished();
324
325 auto model = Sink::Store::loadModel<Folder>(query);
326 QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool());
327 QCOMPARE(model->rowCount(), 1);
328
329 folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value<Folder::Ptr>();
330 QVERIFY(!folderEntity->identifier().isEmpty());
331
332 Mail mail("sink.dummy.instance1");
333 mail.setProperty("uid", "test1");
334 mail.setProperty("folder", folderEntity->identifier());
335 Sink::Store::create<Mail>(mail).exec().waitForFinished();
336
337 Mail mail1("sink.dummy.instance1");
338 mail1.setProperty("uid", "test1");
339 mail1.setProperty("folder", "foobar");
340 Sink::Store::create<Mail>(mail1).exec().waitForFinished();
341
342 Mail mail2("sink.dummy.instance1");
343 mail2.setProperty("uid", "test2");
344 mail2.setProperty("folder", folderEntity->identifier());
345 Sink::Store::create<Mail>(mail2).exec().waitForFinished();
346 }
347
348 // Test
349 Sink::Query query;
350 query.resources << "sink.dummy.instance1";
351 query.filter<Mail::Folder>(*folderEntity);
352 query.filter<Mail::Uid>("test1");
353
354 // Ensure all local data is processed
355 Sink::ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished();
356
357 // We fetch before the data is available and rely on the live query mechanism to deliver the actual data
358 auto model = Sink::Store::loadModel<Mail>(query);
359 QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool());
360 QCOMPARE(model->rowCount(), 1);
361 }
362
259 void testMailByFolderSortedByDate() 363 void testMailByFolderSortedByDate()
260 { 364 {
261 // Setup 365 // Setup
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-29 07:02:04 +0000