From ebc5c48c03b6145e604da7c313b35321d0a71142 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 20 Sep 2016 17:18:21 +0200 Subject: A first draft of the threading algorithm. --- common/datastorequery.cpp | 55 +++++++--- common/datastorequery.h | 20 ++-- common/domain/applicationdomaintype.h | 2 + common/domain/event.cpp | 10 +- common/domain/event.h | 8 +- common/domain/folder.cpp | 9 +- common/domain/folder.h | 3 +- common/domain/mail.cpp | 117 ++++++++++++++++++--- common/domain/mail.fbs | 2 + common/domain/mail.h | 8 +- common/entityreader.cpp | 25 +---- common/mailpreprocessor.cpp | 102 +++++++++++++----- common/mailpreprocessor.h | 3 - common/query.h | 5 +- examples/maildirresource/tests/CMakeLists.txt | 2 + .../cur/100000000.R28.localhost.localdomain:2,S | 70 ++++++++++++ .../cur/100000001.R28.localhost.localdomain:2,S | 72 +++++++++++++ .../maildirresource/tests/maildirmailsynctest.cpp | 29 +---- .../maildirresource/tests/maildirthreadtest.cpp | 90 ++++++++++++++++ examples/maildirresource/tests/utils.h | 50 +++++++++ tests/CMakeLists.txt | 2 +- tests/mailthreadtest.cpp | 94 +++++++++++++++++ tests/mailthreadtest.h | 57 ++++++++++ 23 files changed, 688 insertions(+), 147 deletions(-) create mode 100644 examples/maildirresource/tests/data/maildir2/INBOX/cur/100000000.R28.localhost.localdomain:2,S create mode 100644 examples/maildirresource/tests/data/maildir2/INBOX/cur/100000001.R28.localhost.localdomain:2,S create mode 100644 examples/maildirresource/tests/maildirthreadtest.cpp create mode 100644 examples/maildirresource/tests/utils.h create mode 100644 tests/mailthreadtest.cpp create mode 100644 tests/mailthreadtest.h diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp index 3237c53..cc070be 100644 --- a/common/datastorequery.cpp +++ b/common/datastorequery.cpp @@ -122,17 +122,17 @@ QVariant DataStoreQuery::getProperty(const Sink::Entity &entity, const QByteArra return mGetProperty(entity, property); } -ResultSet DataStoreQuery::filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, bool initialQuery, const QByteArray &sortProperty) +ResultSet DataStoreQuery::filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, const QByteArray &sortProperty) { const bool sortingRequired = !sortProperty.isEmpty(); - if (initialQuery && sortingRequired) { + if (mInitialQuery && sortingRequired) { SinkTrace() << "Sorting the resultset in memory according to property: " << sortProperty; // Sort the complete set by reading the sort property and filling into a sorted map auto sortedMap = QSharedPointer>::create(); while (resultSet.next()) { // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) readEntity(resultSet.id(), - [this, filter, initialQuery, sortedMap, sortProperty, &resultSet](const QByteArray &uid, const Sink::EntityBuffer &buffer) { + [this, filter, sortedMap, sortProperty, &resultSet](const QByteArray &uid, const Sink::EntityBuffer &buffer) { const auto operation = buffer.operation(); @@ -154,10 +154,10 @@ ResultSet DataStoreQuery::filterAndSortSet(ResultSet &resultSet, const FilterFun SinkTrace() << "Sorted " << sortedMap->size() << " values."; auto iterator = QSharedPointer>::create(*sortedMap); - ResultSet::ValueGenerator generator = [this, iterator, sortedMap, filter, initialQuery]( + ResultSet::ValueGenerator generator = [this, iterator, sortedMap, filter]( std::function callback) -> bool { if (iterator->hasNext()) { - readEntity(iterator->next().value(), [this, filter, callback, initialQuery](const QByteArray &uid, const Sink::EntityBuffer &buffer) { + readEntity(iterator->next().value(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) { callback(uid, buffer, Sink::Operation_Creation); }); return true; @@ -173,13 +173,13 @@ ResultSet DataStoreQuery::filterAndSortSet(ResultSet &resultSet, const FilterFun return ResultSet(generator, skip); } else { auto resultSetPtr = QSharedPointer::create(resultSet); - ResultSet::ValueGenerator generator = [this, resultSetPtr, filter, initialQuery](const ResultSet::Callback &callback) -> bool { + ResultSet::ValueGenerator generator = [this, resultSetPtr, filter](const ResultSet::Callback &callback) -> bool { if (resultSetPtr->next()) { SinkTrace() << "Reading the next value: " << resultSetPtr->id(); // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) - readEntity(resultSetPtr->id(), [this, filter, callback, initialQuery](const QByteArray &uid, const Sink::EntityBuffer &buffer) { + readEntity(resultSetPtr->id(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) { const auto operation = buffer.operation(); - if (initialQuery) { + if (mInitialQuery) { // We're not interested in removals during the initial query if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) { // In the initial set every entity is new @@ -225,22 +225,53 @@ DataStoreQuery::FilterFunction DataStoreQuery::getFilter(const QSet }; } +ResultSet DataStoreQuery::createFilteredSet(ResultSet &resultSet, const std::function &filter) +{ + auto resultSetPtr = QSharedPointer::create(resultSet); + ResultSet::ValueGenerator generator = [this, resultSetPtr, filter](const ResultSet::Callback &callback) -> bool { + return resultSetPtr->next([=](const QByteArray &uid, const Sink::EntityBuffer &buffer, Sink::Operation operation) { + if (mInitialQuery) { + // We're not interested in removals during the initial query + if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) { + // In the initial set every entity is new + callback(uid, buffer, Sink::Operation_Creation); + } + } else { + // Always remove removals, they probably don't match due to non-available properties + if ((operation == Sink::Operation_Removal) || filter(uid, buffer)) { + // TODO only replay if this is in the currently visible set (or just always replay, worst case we have a couple to many results) + callback(uid, buffer, operation); + } + } + }); + }; + auto skip = [resultSetPtr]() { resultSetPtr->skip(1); }; + return ResultSet(generator, skip); +} + +ResultSet DataStoreQuery::postSortFilter(ResultSet &resultSet) +{ + return resultSet; +} + ResultSet DataStoreQuery::update(qint64 baseRevision) { SinkTrace() << "Executing query update"; + mInitialQuery = false; QSet remainingFilters; QByteArray remainingSorting; auto resultSet = loadIncrementalResultSet(baseRevision, remainingFilters); - auto filteredSet = filterAndSortSet(resultSet, getFilter(remainingFilters), false, remainingSorting); - return filteredSet; + auto filteredSet = filterAndSortSet(resultSet, getFilter(remainingFilters), remainingSorting); + return postSortFilter(filteredSet); } ResultSet DataStoreQuery::execute() { SinkTrace() << "Executing query"; + mInitialQuery = true; QSet remainingFilters; QByteArray remainingSorting; auto resultSet = loadInitialResultSet(remainingFilters, remainingSorting); - auto filteredSet = filterAndSortSet(resultSet, getFilter(remainingFilters), true, remainingSorting); - return filteredSet; + auto filteredSet = filterAndSortSet(resultSet, getFilter(remainingFilters), remainingSorting); + return postSortFilter(filteredSet); } diff --git a/common/datastorequery.h b/common/datastorequery.h index cf9d9e2..7712ac7 100644 --- a/common/datastorequery.h +++ b/common/datastorequery.h @@ -27,24 +27,29 @@ class DataStoreQuery { public: + typedef QSharedPointer Ptr; + DataStoreQuery(const Sink::Query &query, const QByteArray &type, Sink::Storage::Transaction &transaction, TypeIndex &typeIndex, std::function getProperty); ResultSet execute(); ResultSet update(qint64 baseRevision); -private: +protected: typedef std::function FilterFunction; typedef std::function BufferCallback; - QVariant getProperty(const Sink::Entity &entity, const QByteArray &property); + virtual QVariant getProperty(const Sink::Entity &entity, const QByteArray &property); + + virtual void readEntity(const QByteArray &key, const BufferCallback &resultCallback); - void readEntity(const QByteArray &key, const BufferCallback &resultCallback); + virtual ResultSet loadInitialResultSet(QSet &remainingFilters, QByteArray &remainingSorting); + virtual ResultSet loadIncrementalResultSet(qint64 baseRevision, QSet &remainingFilters); - ResultSet loadInitialResultSet(QSet &remainingFilters, QByteArray &remainingSorting); - ResultSet loadIncrementalResultSet(qint64 baseRevision, QSet &remainingFilters); + virtual ResultSet filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, const QByteArray &sortProperty); + virtual ResultSet postSortFilter(ResultSet &resultSet); + virtual FilterFunction getFilter(const QSet &remainingFilters); - ResultSet filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, bool initialQuery, const QByteArray &sortProperty); - FilterFunction getFilter(const QSet &remainingFilters); + ResultSet createFilteredSet(ResultSet &resultSet, const std::function &); Sink::Query mQuery; Sink::Storage::Transaction &mTransaction; @@ -52,6 +57,7 @@ private: TypeIndex &mTypeIndex; Sink::Storage::NamedDatabase mDb; std::function mGetProperty; + bool mInitialQuery; }; diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h index 67f33c8..c853397 100644 --- a/common/domain/applicationdomaintype.h +++ b/common/domain/applicationdomaintype.h @@ -238,6 +238,8 @@ struct SINK_EXPORT Mail : public Entity { SINK_PROPERTY(bool, Draft, draft); SINK_PROPERTY(bool, Trash, trash); SINK_PROPERTY(bool, Sent, sent); + SINK_EXTRACTED_PROPERTY(QByteArray, MessageId, messageId); + SINK_EXTRACTED_PROPERTY(QByteArray, ParentMessageId, parentMessageId); }; /** diff --git a/common/domain/event.cpp b/common/domain/event.cpp index dfbcb61..118ffa3 100644 --- a/common/domain/event.cpp +++ b/common/domain/event.cpp @@ -52,11 +52,6 @@ static TypeIndex &getIndex() return *index; } -ResultSet TypeImplementation::queryIndexes(const Sink::Query &query, const QByteArray &resourceInstanceIdentifier, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction) -{ - return getIndex().query(query, appliedFilters, appliedSorting, transaction); -} - void TypeImplementation::index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction) { return getIndex().add(identifier, bufferAdaptor, transaction); @@ -87,11 +82,10 @@ QSharedPointer::BufferBuilder> > T return propertyMapper; } -DataStoreQuery TypeImplementation::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction) +DataStoreQuery::Ptr TypeImplementation::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction) { - auto mapper = initializeReadPropertyMapper(); - return DataStoreQuery(query, ApplicationDomain::getTypeName(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) { + return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) { const auto localBuffer = Sink::EntityBuffer::readBuffer(entity.local()); return mapper->getProperty(property, localBuffer); diff --git a/common/domain/event.h b/common/domain/event.h index 4ac572c..e1ca061 100644 --- a/common/domain/event.h +++ b/common/domain/event.h @@ -51,13 +51,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Event Buffer; typedef Sink::ApplicationDomain::Buffer::EventBuilder BufferBuilder; static QSet indexedProperties(); - static DataStoreQuery prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction); - /** - * Returns the potential result set based on the indexes. - * - * An empty result set indicates that a full scan is required. - */ - static ResultSet queryIndexes(const Sink::Query &query, const QByteArray &resourceInstanceIdentifier, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction); + static DataStoreQuery::Ptr prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction); static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); static QSharedPointer > initializeReadPropertyMapper(); diff --git a/common/domain/folder.cpp b/common/domain/folder.cpp index 6d487b1..17d9f13 100644 --- a/common/domain/folder.cpp +++ b/common/domain/folder.cpp @@ -55,11 +55,6 @@ static TypeIndex &getIndex() return *index; } -ResultSet TypeImplementation::queryIndexes(const Sink::Query &query, const QByteArray &resourceInstanceIdentifier, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction) -{ - return getIndex().query(query, appliedFilters, appliedSorting, transaction); -} - void TypeImplementation::index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction) { SinkTrace() << "Indexing " << identifier; @@ -91,10 +86,10 @@ QSharedPointer::BufferBuilder> > return propertyMapper; } -DataStoreQuery TypeImplementation::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction) +DataStoreQuery::Ptr TypeImplementation::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction) { auto mapper = initializeReadPropertyMapper(); - return DataStoreQuery(query, ApplicationDomain::getTypeName(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) { + return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) { const auto localBuffer = Sink::EntityBuffer::readBuffer(entity.local()); return mapper->getProperty(property, localBuffer); }); diff --git a/common/domain/folder.h b/common/domain/folder.h index 77edc8a..ff87006 100644 --- a/common/domain/folder.h +++ b/common/domain/folder.h @@ -45,9 +45,8 @@ class TypeImplementation { public: typedef Sink::ApplicationDomain::Buffer::Folder Buffer; typedef Sink::ApplicationDomain::Buffer::FolderBuilder BufferBuilder; - static DataStoreQuery prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction); + static DataStoreQuery::Ptr prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction); static QSet indexedProperties(); - static ResultSet queryIndexes(const Sink::Query &query, const QByteArray &resourceInstanceIdentifier, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction); static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); static QSharedPointer > initializeReadPropertyMapper(); diff --git a/common/domain/mail.cpp b/common/domain/mail.cpp index bb5ad58..859ebef 100644 --- a/common/domain/mail.cpp +++ b/common/domain/mail.cpp @@ -41,6 +41,7 @@ SINK_DEBUG_AREA("mail"); static QMutex sMutex; +using namespace Sink; using namespace Sink::ApplicationDomain; static TypeIndex &getIndex() @@ -49,26 +50,54 @@ static TypeIndex &getIndex() static TypeIndex *index = 0; if (!index) { index = new TypeIndex("mail"); - index->addProperty("uid"); - index->addProperty("sender"); - index->addProperty("senderName"); - index->addProperty("subject"); - index->addProperty("date"); - index->addProperty("folder"); - index->addPropertyWithSorting("folder", "date"); + index->addProperty(Mail::Uid::name); + index->addProperty(Mail::Sender::name); + index->addProperty(Mail::SenderName::name); + index->addProperty(Mail::Subject::name); + index->addProperty(Mail::Date::name); + index->addProperty(Mail::Folder::name); + index->addPropertyWithSorting(Mail::Folder::name, Mail::Date::name); + index->addProperty(Mail::MessageId::name); + index->addProperty(Mail::ParentMessageId::name); } return *index; } -ResultSet TypeImplementation::queryIndexes(const Sink::Query &query, const QByteArray &resourceInstanceIdentifier, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction) +static void updateThreadingIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction) { - return getIndex().query(query, appliedFilters, appliedSorting, transaction); + auto messageId = bufferAdaptor.getProperty(Mail::MessageId::name).toByteArray(); + auto parentMessageId = bufferAdaptor.getProperty(Mail::ParentMessageId::name).toByteArray(); + + Index msgIdIndex("msgId", transaction); + Index msgIdThreadIdIndex("msgIdThreadId", transaction); + + //Add the message to the index + Q_ASSERT(msgIdIndex.lookup(messageId).isEmpty()); + msgIdIndex.add(messageId, identifier); + + //If parent is already available, add to thread of parent + QByteArray thread; + if (!parentMessageId.isEmpty() && !msgIdIndex.lookup(parentMessageId).isEmpty()) { + thread = msgIdThreadIdIndex.lookup(parentMessageId); + msgIdThreadIdIndex.add(messageId, thread); + } else { + thread = QUuid::createUuid().toByteArray(); + if (!parentMessageId.isEmpty()) { + //Register parent with thread for when it becomes available + msgIdThreadIdIndex.add(parentMessageId, thread); + } + } + Q_ASSERT(!thread.isEmpty()); + msgIdThreadIdIndex.add(messageId, thread); + + //Look for parentMessageId and resolve to local id if available } void TypeImplementation::index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction) { SinkTrace() << "Indexing " << identifier; getIndex().add(identifier, bufferAdaptor, transaction); + updateThreadingIndex(identifier, bufferAdaptor, transaction); } void TypeImplementation::removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction) @@ -91,6 +120,8 @@ QSharedPointer::Buffer> > TypeImplem propertyMapper->addMapping(&Buffer::draft); propertyMapper->addMapping(&Buffer::trash); propertyMapper->addMapping(&Buffer::sent); + propertyMapper->addMapping(&Buffer::messageId); + propertyMapper->addMapping(&Buffer::parentMessageId); return propertyMapper; } @@ -110,16 +141,72 @@ QSharedPointer::BufferBuilder> > Ty propertyMapper->addMapping(&BufferBuilder::add_draft); propertyMapper->addMapping(&BufferBuilder::add_trash); propertyMapper->addMapping(&BufferBuilder::add_sent); + propertyMapper->addMapping(&BufferBuilder::add_messageId); + propertyMapper->addMapping(&BufferBuilder::add_parentMessageId); return propertyMapper; } -DataStoreQuery TypeImplementation::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction) +class ThreadedDataStoreQuery : public DataStoreQuery +{ +public: + typedef QSharedPointer Ptr; + using DataStoreQuery::DataStoreQuery; + +protected: + ResultSet postSortFilter(ResultSet &resultSet) Q_DECL_OVERRIDE + { + auto query = mQuery; + if (query.threadLeaderOnly) { + auto rootCollection = QSharedPointer>::create(); + auto filter = [this, query, rootCollection](const QByteArray &uid, const Sink::EntityBuffer &entity) -> bool { + //TODO lookup thread + //if we got thread already in the result set compare dates and if newer replace + //else insert + + const auto messageId = getProperty(entity.entity(), ApplicationDomain::Mail::MessageId::name).toByteArray(); + + Index msgIdIndex("msgId", mTransaction); + Index msgIdThreadIdIndex("msgIdThreadId", mTransaction); + auto thread = msgIdThreadIdIndex.lookup(messageId); + SinkTrace() << "MsgId: " << messageId << " Thread: " << thread << getProperty(entity.entity(), ApplicationDomain::Mail::Date::name).toDateTime(); + + if (rootCollection->contains(thread)) { + auto date = rootCollection->value(thread); + //The mail we have in our result already is newer, so we can ignore this one + if (date > getProperty(entity.entity(), ApplicationDomain::Mail::Date::name).toDateTime()) { + return false; + } + qWarning() << "############################################################################"; + qWarning() << "Found a newer mail, remove the old one"; + qWarning() << "############################################################################"; + } + rootCollection->insert(thread, getProperty(entity.entity(), ApplicationDomain::Mail::Date::name).toDateTime()); + return true; + }; + return createFilteredSet(resultSet, filter); + } else { + return resultSet; + } + } +}; + +DataStoreQuery::Ptr TypeImplementation::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction) { - auto mapper = initializeReadPropertyMapper(); - return DataStoreQuery(query, ApplicationDomain::getTypeName(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) { + if (query.threadLeaderOnly) { + auto mapper = initializeReadPropertyMapper(); + return ThreadedDataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) { + + const auto localBuffer = Sink::EntityBuffer::readBuffer(entity.local()); + return mapper->getProperty(property, localBuffer); + }); - const auto localBuffer = Sink::EntityBuffer::readBuffer(entity.local()); - return mapper->getProperty(property, localBuffer); - }); + } else { + auto mapper = initializeReadPropertyMapper(); + return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) { + + const auto localBuffer = Sink::EntityBuffer::readBuffer(entity.local()); + return mapper->getProperty(property, localBuffer); + }); + } } diff --git a/common/domain/mail.fbs b/common/domain/mail.fbs index a0c0d82..f14e9f1 100644 --- a/common/domain/mail.fbs +++ b/common/domain/mail.fbs @@ -13,6 +13,8 @@ table Mail { draft:bool = false; trash:bool = false; sent:bool = false; + messageId:string; + parentMessageId:string; } root_type Mail; diff --git a/common/domain/mail.h b/common/domain/mail.h index d6af9c5..3b0e9da 100644 --- a/common/domain/mail.h +++ b/common/domain/mail.h @@ -45,14 +45,8 @@ class TypeImplementation { public: typedef Sink::ApplicationDomain::Buffer::Mail Buffer; typedef Sink::ApplicationDomain::Buffer::MailBuilder BufferBuilder; - static DataStoreQuery prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction); + static DataStoreQuery::Ptr prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction); static QSet indexedProperties(); - /** - * Returns the potential result set based on the indexes. - * - * An empty result set indicates that a full scan is required. - */ - static ResultSet queryIndexes(const Sink::Query &query, const QByteArray &resourceInstanceIdentifier, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::Transaction &transaction); static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction); static QSharedPointer > initializeReadPropertyMapper(); diff --git a/common/entityreader.cpp b/common/entityreader.cpp index faa154b..d86f4a9 100644 --- a/common/entityreader.cpp +++ b/common/entityreader.cpp @@ -150,27 +150,6 @@ void EntityReader::query(const Sink::Query &query, const std::functi }); } -/* template */ -/* void EntityReader::readEntity(const Sink::Storage::NamedDatabase &db, const QByteArray &key, */ -/* const std::function &resultCallback) */ -/* { */ -/* db.findLatest(key, */ -/* [=](const QByteArray &key, const QByteArray &value) -> bool { */ -/* Sink::EntityBuffer buffer(value.data(), value.size()); */ -/* const Sink::Entity &entity = buffer.entity(); */ -/* const auto metadataBuffer = Sink::EntityBuffer::readBuffer(entity.metadata()); */ -/* const qint64 revision = metadataBuffer ? metadataBuffer->revision() : -1; */ -/* const auto operation = metadataBuffer ? metadataBuffer->operation() : Sink::Operation_Creation; */ -/* auto adaptor = mDomainTypeAdaptorFactory.createAdaptor(entity); */ -/* Q_ASSERT(adaptor); */ -/* resultCallback(DomainType::Ptr::create(mResourceInstanceIdentifier, Sink::Storage::uidFromKey(key), revision, adaptor), operation); */ -/* return false; */ -/* }, */ -/* [&](const Sink::Storage::Error &error) { SinkWarning() << "Error during query: " << error.message << key; }); */ -/* } */ - - - template QPair EntityReader::executeInitialQuery(const Sink::Query &query, int offset, int batchsize, const std::function &callback) { @@ -178,7 +157,7 @@ QPair EntityReader::executeInitialQuery(const Sink:: time.start(); auto preparedQuery = ApplicationDomain::TypeImplementation::prepareQuery(query, mTransaction); - auto resultSet = preparedQuery.execute(); + auto resultSet = preparedQuery->execute(); SinkTrace() << "Filtered set retrieved. " << Log::TraceTime(time.elapsed()); auto replayedEntities = replaySet(resultSet, offset, batchsize, callback); @@ -195,7 +174,7 @@ QPair EntityReader::executeIncrementalQuery(const Si const qint64 baseRevision = lastRevision + 1; auto preparedQuery = ApplicationDomain::TypeImplementation::prepareQuery(query, mTransaction); - auto resultSet = preparedQuery.update(baseRevision); + auto resultSet = preparedQuery->update(baseRevision); SinkTrace() << "Filtered set retrieved. " << Log::TraceTime(time.elapsed()); auto replayedEntities = replaySet(resultSet, 0, 0, callback); diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp index 2863ad4..0534338 100644 --- a/common/mailpreprocessor.cpp +++ b/common/mailpreprocessor.cpp @@ -36,48 +36,98 @@ QString MailPropertyExtractor::getFilePathFromMimeMessagePath(const QString &s) return s; } -void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail) -{ - const auto mimeMessagePath = getFilePathFromMimeMessagePath(mail.getMimeMessagePath()); - if (mimeMessagePath.isNull()) { - SinkTrace() << "No mime message"; - return; - } - SinkTrace() << "Updating indexed properties " << mimeMessagePath; - QFile f(mimeMessagePath); - if (!f.open(QIODevice::ReadOnly)) { - SinkWarning() << "Failed to open the file: " << mimeMessagePath; - return; - } - if (!f.size()) { - SinkWarning() << "The file is empty."; - return; +struct MimeMessageReader { + MimeMessageReader(const QString &mimeMessagePath) + : f(mimeMessagePath), + mapped(0), + mappedSize(0) + { + if (mimeMessagePath.isNull()) { + SinkTrace() << "No mime message"; + return; + } + SinkTrace() << "Updating indexed properties " << mimeMessagePath; + if (!f.open(QIODevice::ReadOnly)) { + SinkWarning() << "Failed to open the file: " << mimeMessagePath; + return; + } + if (!f.size()) { + SinkWarning() << "The file is empty."; + return; + } + mappedSize = qMin((qint64)8000, f.size()); + mapped = f.map(0, mappedSize); + if (!mapped) { + SinkWarning() << "Failed to map the file: " << f.errorString(); + return; + } } - const auto mappedSize = qMin((qint64)8000, f.size()); - auto mapped = f.map(0, mappedSize); - if (!mapped) { - SinkWarning() << "Failed to map the file: " << f.errorString(); - return; + + KMime::Message::Ptr mimeMessage() + { + if (!mapped) { + return KMime::Message::Ptr(); + } + Q_ASSERT(mapped); + Q_ASSERT(mappedSize); + auto msg = KMime::Message::Ptr(new KMime::Message); + msg->setHead(KMime::CRLFtoLF(QByteArray::fromRawData(reinterpret_cast(mapped), mappedSize))); + msg->parse(); + return msg; } - KMime::Message *msg = new KMime::Message; - msg->setHead(KMime::CRLFtoLF(QByteArray::fromRawData(reinterpret_cast(mapped), mappedSize))); - msg->parse(); + QFile f; + uchar *mapped; + qint64 mappedSize; +}; +static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, KMime::Message::Ptr msg) +{ mail.setExtractedSubject(msg->subject(true)->asUnicodeString()); mail.setExtractedSender(msg->from(true)->asUnicodeString()); mail.setExtractedSenderName(msg->from(true)->asUnicodeString()); mail.setExtractedDate(msg->date(true)->dateTime()); + + //The rest should never change, unless we didn't have the headers available initially. + auto messageId = msg->messageID(true)->identifier(); + + //Ensure the mssageId is unique. + //If there already is one with the same id we'd have to assign a new message id, which probably doesn't make any sense. + + //The last is the parent + auto references = msg->references(true)->identifiers(); + + //The first is the parent + auto inReplyTo = msg->inReplyTo(true)->identifiers(); + QByteArray parentMessageId; + if (!references.isEmpty()) { + parentMessageId = references.last(); + //TODO we could use the rest of the references header to complete the ancestry in case we have missing parents. + } else { + if (!inReplyTo.isEmpty()) { + //According to RFC5256 we should ignore all but the first + parentMessageId = inReplyTo.first(); + } + } + + mail.setExtractedMessageId(messageId); + if (!parentMessageId.isEmpty()) { + mail.setExtractedParentMessageId(parentMessageId); + } } void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail, Sink::Storage::Transaction &transaction) { - updatedIndexedProperties(mail); + MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(mail.getMimeMessagePath())); + auto msg = mimeMessageReader.mimeMessage(); + updatedIndexedProperties(mail, msg); } void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail,Sink::Storage::Transaction &transaction) { - updatedIndexedProperties(newMail); + MimeMessageReader mimeMessageReader(getFilePathFromMimeMessagePath(newMail.getMimeMessagePath())); + auto msg = mimeMessageReader.mimeMessage(); + updatedIndexedProperties(newMail, msg); } diff --git a/common/mailpreprocessor.h b/common/mailpreprocessor.h index 473931c..b7cd0e7 100644 --- a/common/mailpreprocessor.h +++ b/common/mailpreprocessor.h @@ -28,9 +28,6 @@ public: virtual void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail,Sink::Storage::Transaction &transaction) Q_DECL_OVERRIDE; protected: virtual QString getFilePathFromMimeMessagePath(const QString &) const; - -private: - void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail); }; class SINK_EXPORT MimeMessageMover : public Sink::EntityPreprocessor diff --git a/common/query.h b/common/query.h index 68463cd..0808432 100644 --- a/common/query.h +++ b/common/query.h @@ -195,13 +195,13 @@ public: return *this; } - Query(const ApplicationDomain::Entity &value) : limit(0), liveQuery(false), synchronousQuery(false) + Query(const ApplicationDomain::Entity &value) : limit(0), liveQuery(false), synchronousQuery(false), threadLeaderOnly(false) { ids << value.identifier(); resources << value.resourceInstanceIdentifier(); } - Query(Flags flags = Flags()) : limit(0), liveQuery(false), synchronousQuery(false) + Query(Flags flags = Flags()) : limit(0), liveQuery(false), synchronousQuery(false), threadLeaderOnly(false) { } @@ -236,6 +236,7 @@ public: int limit; bool liveQuery; bool synchronousQuery; + bool threadLeaderOnly; }; } diff --git a/examples/maildirresource/tests/CMakeLists.txt b/examples/maildirresource/tests/CMakeLists.txt index d6f0fe7..b3e4050 100644 --- a/examples/maildirresource/tests/CMakeLists.txt +++ b/examples/maildirresource/tests/CMakeLists.txt @@ -8,6 +8,8 @@ include(SinkTest) auto_tests ( maildirmailtest maildirmailsynctest + maildirthreadtest ) target_link_libraries(maildirmailtest sink_resource_maildir) target_link_libraries(maildirmailsynctest sink_resource_maildir) +target_link_libraries(maildirthreadtest sink_resource_maildir) diff --git a/examples/maildirresource/tests/data/maildir2/INBOX/cur/100000000.R28.localhost.localdomain:2,S b/examples/maildirresource/tests/data/maildir2/INBOX/cur/100000000.R28.localhost.localdomain:2,S new file mode 100644 index 0000000..e899eb7 --- /dev/null +++ b/examples/maildirresource/tests/data/maildir2/INBOX/cur/100000000.R28.localhost.localdomain:2,S @@ -0,0 +1,70 @@ +Return-Path: +Received: from compute4.internal (compute4.nyi.mail.srv.osa [10.202.2.44]) + by slots3a1p1 (Cyrus git2.5+0-git-fastmail-8998) with LMTPA; + Mon, 11 Mar 2013 14:28:42 -0400 +X-Sieve: CMU Sieve 2.4 +X-Spam-score: 0.0 +X-Spam-hits: BAYES_00 -1.9, RCVD_IN_DNSWL_MED -2.3, RP_MATCHES_RCVD -0.704, + LANGUAGES unknown, BAYES_USED global, SA_VERSION 3.3.1 +X-Spam-source: IP='46.4.96.248', Host='postbox.kde.org', Country='unk', FromHeader='org', + MailFrom='org' +X-Spam-charsets: plain='us-ascii' +X-Resolved-to: chrigi_1@fastmail.fm +X-Delivered-to: chrigi_1@fastmail.fm +X-Mail-from: nepomuk-bounces@kde.org +Received: from mx4.nyi.mail.srv.osa ([10.202.2.203]) + by compute4.internal (LMTPProxy); Mon, 11 Mar 2013 14:28:42 -0400 +Received: from postbox.kde.org (postbox.kde.org [46.4.96.248]) + by mx4.messagingengine.com (Postfix) with ESMTP id 1C9D2440F88 + for ; Mon, 11 Mar 2013 14:28:42 -0400 (EDT) +Received: from postbox.kde.org (localhost [IPv6:::1]) + by postbox.kde.org (Postfix) with ESMTP id 00FFEB3732B; + Mon, 11 Mar 2013 18:28:40 +0000 (UTC) +DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=kde.org; s=default; + t=1363026520; bh=cOdvyBAJJ8ho64q0H7rxkl+cB2y6TiyVOX0fO3yZ64U=; + h=Date:From:To:Message-ID:In-Reply-To:References:MIME-Version: + Subject:List-Id:List-Unsubscribe:List-Archive:List-Post:List-Help: + List-Subscribe:Content-Type:Content-Transfer-Encoding:Sender; b=dv + dJAFu+6JCuNun5WIuP4ysfKpLh0DeuhEEfy2cQavUGMICJ27k7tI73x6gN37V5Q/evJ + NDFna3/IhNBsAQeLiXs28HKxzcVhbnq5jdFR6fbyo6k1fOKt5vTT1GTDZ+3zIGPD1CU + ioDBGxPb/Ds6gee90tjadOj6o+Oc+2ZSq94= +X-Original-To: nepomuk@kde.org +X-Remote-Delivered-To: nepomuk@localhost.kde.org +Received: from build.kde.org (build.kde.org [IPv6:2a01:4f8:160:9363::5]) + by postbox.kde.org (Postfix) with ESMTP id 4491CB3732B + for ; Mon, 11 Mar 2013 18:28:27 +0000 (UTC) +Received: from localhost ([127.0.0.1]) by build.kde.org with esmtp (Exim 4.72) + (envelope-from ) id 1UF7SV-0000gs-11 + for nepomuk@kde.org; Mon, 11 Mar 2013 18:28:27 +0000 +Date: Mon, 11 Mar 2013 18:28:27 +0000 (UTC) +From: KDE CI System +To: nepomuk@kde.org +Message-ID: <1977027405.27.1363026507008.JavaMail.jenkins@build> +MIME-Version: 1.0 +X-Jenkins-Job: nepomuk-core_stable +X-Jenkins-Result: UNSTABLE +X-Scanned-By: MIMEDefang 2.71 on 46.4.96.248 +Subject: First message +X-BeenThere: nepomuk@kde.org +X-Mailman-Version: 2.1.14 +Precedence: list +List-Id: The Semantic KDE +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Errors-To: nepomuk-bounces@kde.org +Sender: nepomuk-bounces@kde.org +X-Truedomain: NotChecked + +See + +_______________________________________________ +Nepomuk mailing list +Nepomuk@kde.org +https://mail.kde.org/mailman/listinfo/nepomuk diff --git a/examples/maildirresource/tests/data/maildir2/INBOX/cur/100000001.R28.localhost.localdomain:2,S b/examples/maildirresource/tests/data/maildir2/INBOX/cur/100000001.R28.localhost.localdomain:2,S new file mode 100644 index 0000000..cfdd438 --- /dev/null +++ b/examples/maildirresource/tests/data/maildir2/INBOX/cur/100000001.R28.localhost.localdomain:2,S @@ -0,0 +1,72 @@ +Return-Path: +Received: from compute4.internal (compute4.nyi.mail.srv.osa [10.202.2.44]) + by slots3a1p1 (Cyrus git2.5+0-git-fastmail-8998) with LMTPA; + Mon, 11 Mar 2013 14:28:42 -0400 +X-Sieve: CMU Sieve 2.4 +X-Spam-score: 0.0 +X-Spam-hits: BAYES_00 -1.9, RCVD_IN_DNSWL_MED -2.3, RP_MATCHES_RCVD -0.704, + LANGUAGES unknown, BAYES_USED global, SA_VERSION 3.3.1 +X-Spam-source: IP='46.4.96.248', Host='postbox.kde.org', Country='unk', FromHeader='org', + MailFrom='org' +X-Spam-charsets: plain='us-ascii' +X-Resolved-to: chrigi_1@fastmail.fm +X-Delivered-to: chrigi_1@fastmail.fm +X-Mail-from: nepomuk-bounces@kde.org +Received: from mx4.nyi.mail.srv.osa ([10.202.2.203]) + by compute4.internal (LMTPProxy); Mon, 11 Mar 2013 14:28:42 -0400 +Received: from postbox.kde.org (postbox.kde.org [46.4.96.248]) + by mx4.messagingengine.com (Postfix) with ESMTP id 1C9D2440F88 + for ; Mon, 11 Mar 2013 14:28:42 -0400 (EDT) +Received: from postbox.kde.org (localhost [IPv6:::1]) + by postbox.kde.org (Postfix) with ESMTP id 00FFEB3732B; + Mon, 11 Mar 2013 18:28:40 +0000 (UTC) +DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=kde.org; s=default; + t=1363026520; bh=cOdvyBAJJ8ho64q0H7rxkl+cB2y6TiyVOX0fO3yZ64U=; + h=Date:From:To:Message-ID:In-Reply-To:References:MIME-Version: + Subject:List-Id:List-Unsubscribe:List-Archive:List-Post:List-Help: + List-Subscribe:Content-Type:Content-Transfer-Encoding:Sender; b=dv + dJAFu+6JCuNun5WIuP4ysfKpLh0DeuhEEfy2cQavUGMICJ27k7tI73x6gN37V5Q/evJ + NDFna3/IhNBsAQeLiXs28HKxzcVhbnq5jdFR6fbyo6k1fOKt5vTT1GTDZ+3zIGPD1CU + ioDBGxPb/Ds6gee90tjadOj6o+Oc+2ZSq94= +X-Original-To: nepomuk@kde.org +X-Remote-Delivered-To: nepomuk@localhost.kde.org +Received: from build.kde.org (build.kde.org [IPv6:2a01:4f8:160:9363::5]) + by postbox.kde.org (Postfix) with ESMTP id 4491CB3732B + for ; Mon, 11 Mar 2013 18:28:27 +0000 (UTC) +Received: from localhost ([127.0.0.1]) by build.kde.org with esmtp (Exim 4.72) + (envelope-from ) id 1UF7SV-0000gs-11 + for nepomuk@kde.org; Mon, 11 Mar 2013 18:28:27 +0000 +Date: Mon, 11 Mar 2013 19:28:27 +0000 (UTC) +From: KDE CI System +To: nepomuk@kde.org +Message-ID: <1000000001.27.1363026507008.JavaMail.jenkins@build> +In-Reply-To: <1977027405.27.1363026507008.JavaMail.jenkins@build> +References: <1977027405.27.1363026507008.JavaMail.jenkins@build> +MIME-Version: 1.0 +X-Jenkins-Job: nepomuk-core_stable +X-Jenkins-Result: UNSTABLE +X-Scanned-By: MIMEDefang 2.71 on 46.4.96.248 +Subject: ThreadLeader +X-BeenThere: nepomuk@kde.org +X-Mailman-Version: 2.1.14 +Precedence: list +List-Id: The Semantic KDE +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Errors-To: nepomuk-bounces@kde.org +Sender: nepomuk-bounces@kde.org +X-Truedomain: NotChecked + +See + +_______________________________________________ +Nepomuk mailing list +Nepomuk@kde.org +https://mail.kde.org/mailman/listinfo/nepomuk diff --git a/examples/maildirresource/tests/maildirmailsynctest.cpp b/examples/maildirresource/tests/maildirmailsynctest.cpp index f2cf3e0..bafd5a8 100644 --- a/examples/maildirresource/tests/maildirmailsynctest.cpp +++ b/examples/maildirresource/tests/maildirmailsynctest.cpp @@ -25,36 +25,11 @@ #include "common/test.h" #include "common/domain/applicationdomaintype.h" +#include "utils.h" + using namespace Sink; using namespace Sink::ApplicationDomain; -static bool copyRecursively(const QString &srcFilePath, const QString &tgtFilePath) -{ - QFileInfo srcFileInfo(srcFilePath); - if (srcFileInfo.isDir()) { - QDir targetDir(tgtFilePath); - targetDir.cdUp(); - if (!targetDir.mkdir(QFileInfo(srcFilePath).fileName())) { - qWarning() << "Failed to create directory " << tgtFilePath; - return false; - } - QDir sourceDir(srcFilePath); - QStringList fileNames = sourceDir.entryList(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot | QDir::Hidden | QDir::System); - foreach (const QString &fileName, fileNames) { - const QString newSrcFilePath = srcFilePath + QLatin1Char('/') + fileName; - const QString newTgtFilePath = tgtFilePath + QLatin1Char('/') + fileName; - if (!copyRecursively(newSrcFilePath, newTgtFilePath)) - return false; - } - } else { - if (!QFile::copy(srcFilePath, tgtFilePath)) { - qWarning() << "Failed to copy file " << tgtFilePath; - return false; - } - } - return true; -} - /** * Test of complete system using the maildir resource. * diff --git a/examples/maildirresource/tests/maildirthreadtest.cpp b/examples/maildirresource/tests/maildirthreadtest.cpp new file mode 100644 index 0000000..a74869c --- /dev/null +++ b/examples/maildirresource/tests/maildirthreadtest.cpp @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2016 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include + +#include +#include "../maildirresource.h" +#include "../libmaildir/maildir.h" + +#include "common/test.h" +#include "common/domain/applicationdomaintype.h" + +#include "utils.h"; + +using namespace Sink; +using namespace Sink::ApplicationDomain; + +/** + * Test of complete system using the maildir resource. + * + * This test requires the maildir resource installed. + */ +class MaildirThreadTest : public Sink::MailThreadTest +{ + Q_OBJECT + + QTemporaryDir tempDir; + QString targetPath; + +protected: + void resetTestEnvironment() Q_DECL_OVERRIDE + { + targetPath = tempDir.path() + "/maildir2/"; + QDir dir(targetPath); + dir.removeRecursively(); + copyRecursively(TESTDATAPATH "/maildir2", targetPath); + } + + Sink::ApplicationDomain::SinkResource createResource() Q_DECL_OVERRIDE + { + auto resource = ApplicationDomain::MaildirResource::create("account1"); + resource.setProperty("path", targetPath); + return resource; + } + + Sink::ApplicationDomain::SinkResource createFaultyResource() Q_DECL_OVERRIDE + { + auto resource = ApplicationDomain::MaildirResource::create("account1"); + resource.setProperty("path", ""); + return resource; + } + + void removeResourceFromDisk(const QByteArray &identifier) Q_DECL_OVERRIDE + { + ::MaildirResource::removeFromDisk(identifier); + } + + QByteArray createMessage(const QStringList &folderPath, const QByteArray &message) Q_DECL_OVERRIDE + { + auto rootPath = tempDir.path() + "/maildir2/"; + KPIM::Maildir maildir(rootPath + folderPath.join('/')); + return maildir.addEntry(message).toUtf8(); + } + + void removeMessage(const QStringList &folderPath, const QByteArray &messageIdentifier) Q_DECL_OVERRIDE + { + auto rootPath = tempDir.path() + "/maildir2/"; + KPIM::Maildir maildir(rootPath + folderPath.join('/')); + maildir.removeEntry(messageIdentifier); + } +}; + +QTEST_MAIN(MaildirThreadTest) + +#include "maildirthreadtest.moc" diff --git a/examples/maildirresource/tests/utils.h b/examples/maildirresource/tests/utils.h new file mode 100644 index 0000000..6e25353 --- /dev/null +++ b/examples/maildirresource/tests/utils.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2016 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include +#include +#include + +static bool copyRecursively(const QString &srcFilePath, const QString &tgtFilePath) +{ + QFileInfo srcFileInfo(srcFilePath); + if (srcFileInfo.isDir()) { + QDir targetDir(tgtFilePath); + targetDir.cdUp(); + if (!targetDir.mkdir(QFileInfo(srcFilePath).fileName())) { + qWarning() << "Failed to create directory " << tgtFilePath; + return false; + } + QDir sourceDir(srcFilePath); + QStringList fileNames = sourceDir.entryList(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot | QDir::Hidden | QDir::System); + foreach (const QString &fileName, fileNames) { + const QString newSrcFilePath = srcFilePath + QLatin1Char('/') + fileName; + const QString newTgtFilePath = tgtFilePath + QLatin1Char('/') + fileName; + if (!copyRecursively(newSrcFilePath, newTgtFilePath)) + return false; + } + } else { + if (!QFile::copy(srcFilePath, tgtFilePath)) { + qWarning() << "Failed to copy file " << tgtFilePath; + return false; + } + } + return true; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 294cce2..f8fe3b3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -12,7 +12,7 @@ add_definitions(-DTESTDATAPATH="${CMAKE_CURRENT_SOURCE_DIR}/data") find_package(KF5 COMPONENTS REQUIRED Mime) -add_library(sink_test SHARED testimplementations.cpp getrssusage.cpp mailtest.cpp mailsynctest.cpp) +add_library(sink_test SHARED testimplementations.cpp getrssusage.cpp mailtest.cpp mailsynctest.cpp mailthreadtest.cpp) qt5_use_modules(sink_test Core Test Concurrent) target_link_libraries(sink_test sink libhawd KF5::Mime) diff --git a/tests/mailthreadtest.cpp b/tests/mailthreadtest.cpp new file mode 100644 index 0000000..1bbe713 --- /dev/null +++ b/tests/mailthreadtest.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2016 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "mailthreadtest.h" + +#include + +#include +#include + +#include "store.h" +#include "resourcecontrol.h" +#include "log.h" +#include "test.h" + +using namespace Sink; +using namespace Sink::ApplicationDomain; + +SINK_DEBUG_AREA("mailthreadtest") + +//TODO extract resource test +// +void MailThreadTest::initTestCase() +{ + Test::initTest(); + QVERIFY(isBackendAvailable()); + resetTestEnvironment(); + auto resource = createResource(); + QVERIFY(!resource.identifier().isEmpty()); + + VERIFYEXEC(Store::create(resource)); + + mResourceInstanceIdentifier = resource.identifier(); + mCapabilities = resource.getProperty("capabilities").value(); +} + +void MailThreadTest::cleanup() +{ + //TODO the shutdown job fails if the resource is already shut down + // VERIFYEXEC(ResourceControl::shutdown(mResourceInstanceIdentifier)); + ResourceControl::shutdown(mResourceInstanceIdentifier).exec().waitForFinished(); + removeResourceFromDisk(mResourceInstanceIdentifier); +} + +void MailThreadTest::init() +{ + VERIFYEXEC(ResourceControl::start(mResourceInstanceIdentifier)); +} + + +void MailThreadTest::testListThreadLeader() +{ + Sink::Query query; + query.resources << mResourceInstanceIdentifier; + query.request().request().request().request(); + query.threadLeaderOnly = true; + query.sort(); + + // Ensure all local data is processed + VERIFYEXEC(Store::synchronize(query)); + ResourceControl::flushMessageQueue(query.resources).exec().waitForFinished(); + + auto job = Store::fetchAll(query).syncThen>([](const QList &mails) { + QCOMPARE(mails.size(), 1); + QVERIFY(mails.first()->getSubject().startsWith(QString("ThreadLeader"))); + const auto data = mails.first()->getMimeMessage(); + QVERIFY(!data.isEmpty()); + + KMime::Message m; + m.setContent(data); + m.parse(); + QCOMPARE(mails.first()->getSubject(), m.subject(true)->asUnicodeString()); + QVERIFY(!mails.first()->getFolder().isEmpty()); + QVERIFY(mails.first()->getDate().isValid()); + }); + VERIFYEXEC(job); +} + +#include "mailthreadtest.moc" diff --git a/tests/mailthreadtest.h b/tests/mailthreadtest.h new file mode 100644 index 0000000..d6b9c24 --- /dev/null +++ b/tests/mailthreadtest.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2016 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include +#include + +#include +#include "testutils.h" + +namespace Sink { + +/** + * Tests if the resource can thread emails. + */ +class MailThreadTest : public QObject +{ + Q_OBJECT + +protected: + QByteArray mResourceInstanceIdentifier; + QByteArrayList mCapabilities; + + virtual bool isBackendAvailable() { return true; } + virtual void resetTestEnvironment() = 0; + virtual Sink::ApplicationDomain::SinkResource createResource() = 0; + virtual Sink::ApplicationDomain::SinkResource createFaultyResource() = 0; + virtual void removeResourceFromDisk(const QByteArray &mResourceInstanceIdentifier) = 0; + virtual QByteArray createMessage(const QStringList &folderPath, const QByteArray &message) = 0; + virtual void removeMessage(const QStringList &folderPath, const QByteArray &messageIdentifier) = 0; + +private slots: + void initTestCase(); + void init(); + void cleanup(); + + void testListThreadLeader(); +}; + +} + -- cgit v1.2.3