From 766f40bed69c053c22ae28cfb4115e717c44b616 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 20 Apr 2017 08:40:51 +0200 Subject: Going towards 0.3 --- CMakeLists.txt | 2 +- dist/sink.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 53ae5df..695676d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0) cmake_policy(SET CMP0048 NEW) cmake_policy(SET CMP0028 NEW) -project(sink VERSION 0.2.0) +project(sink VERSION 0.3) option(BUILD_MAILDIR "BUILD_MAILDIR" ON) option(BUILD_DAV "BUILD_DAV" OFF) diff --git a/dist/sink.spec b/dist/sink.spec index 60015f7..246439d 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,6 +1,6 @@ Name: sink -Version: 0.2.0 +Version: 0.3 Release: 0%{?dist} Summary: sink -- cgit v1.2.3 From ac73ca1d2a23d3b62cca20545e019355f9d00035 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 20 Apr 2017 14:59:59 +0200 Subject: Handle host not found --- examples/imapresource/imapserverproxy.cpp | 10 +++++++--- examples/imapresource/imapserverproxy.h | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp index 0cc43b8..3305f60 100644 --- a/examples/imapresource/imapserverproxy.cpp +++ b/examples/imapresource/imapserverproxy.cpp @@ -161,12 +161,16 @@ KAsync::Job ImapServerProxy::login(const QString &username, const QString // SinkTrace() << "Found user namespaces: " << mNamespaces.user; }).then([=] (const KAsync::Error &error) { if (error) { - if (error.errorCode == KIMAP2::LoginJob::ErrorCode::ERR_COULD_NOT_CONNECT) { + switch (error.errorCode) { + case KIMAP2::LoginJob::ErrorCode::ERR_HOST_NOT_FOUND: + return KAsync::error(HostNotFoundError, "Host not found: " + error.errorMessage); + case KIMAP2::LoginJob::ErrorCode::ERR_COULD_NOT_CONNECT: return KAsync::error(CouldNotConnectError, "Failed to connect: " + error.errorMessage); - } else if (error.errorCode == KIMAP2::LoginJob::ErrorCode::ERR_SSL_HANDSHAKE_FAILED) { + case KIMAP2::LoginJob::ErrorCode::ERR_SSL_HANDSHAKE_FAILED: return KAsync::error(SslHandshakeError, "Ssl handshake failed: " + error.errorMessage); + default: + return KAsync::error(error); } - return KAsync::error(error); } return KAsync::null(); }); diff --git a/examples/imapresource/imapserverproxy.h b/examples/imapresource/imapserverproxy.h index 872f032..58c49a2 100644 --- a/examples/imapresource/imapserverproxy.h +++ b/examples/imapresource/imapserverproxy.h @@ -31,6 +31,7 @@ namespace Imap { enum ErrorCode { NoError, + HostNotFoundError, CouldNotConnectError, SslHandshakeError }; -- cgit v1.2.3 From 8d20128618b427270c6c7db49a1716f65b8ff840 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 21 Apr 2017 12:06:49 +0200 Subject: Gmail support. For the time being we hardcode a list of folders that we synchronize that we know are not duplicating messages. --- examples/imapresource/imapresource.cpp | 36 ++++++++++++++++++++++++++----- examples/imapresource/imapserverproxy.cpp | 31 +++++++++++++++++++++++++- examples/imapresource/imapserverproxy.h | 13 ++++++----- 3 files changed, 69 insertions(+), 11 deletions(-) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 0579dae..0b3dcf5 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -87,6 +87,25 @@ static QByteArray parentRid(const Imap::Folder &folder) return folder.parentPath().toUtf8(); } +static QByteArray getSpecialPurposeType(const QByteArrayList &flags) +{ + if (Imap::flagsContain(Imap::FolderFlags::Trash, flags)) { + return ApplicationDomain::SpecialPurpose::Mail::trash; + } + if (Imap::flagsContain(Imap::FolderFlags::Drafts, flags)) { + return ApplicationDomain::SpecialPurpose::Mail::drafts; + } + if (Imap::flagsContain(Imap::FolderFlags::Sent, flags)) { + return ApplicationDomain::SpecialPurpose::Mail::sent; + } + return {}; +} + +static bool hasSpecialPurposeFlag(const QByteArrayList &flags) +{ + return !getSpecialPurposeType(flags).isEmpty(); +} + class ImapSynchronizer : public Sink::Synchronizer { Q_OBJECT @@ -100,7 +119,7 @@ public: QByteArray createFolder(const Imap::Folder &f) { const auto parentFolderRid = parentRid(f); - SinkTraceCtx(mLogCtx) << "Creating folder: " << f.name() << parentFolderRid; + SinkTraceCtx(mLogCtx) << "Creating folder: " << f.name() << parentFolderRid << f.flags; const auto remoteId = folderRid(f); Sink::ApplicationDomain::Folder folder; @@ -108,10 +127,17 @@ public: folder.setIcon("folder"); folder.setEnabled(f.subscribed); QHash mergeCriteria; - if (SpecialPurpose::isSpecialPurposeFolderName(f.name()) && parentFolderRid.isEmpty()) { - auto type = SpecialPurpose::getSpecialPurposeType(f.name()); - folder.setSpecialPurpose(QByteArrayList() << type); - mergeCriteria.insert(ApplicationDomain::Folder::SpecialPurpose::name, Query::Comparator(type, Query::Comparator::Contains)); + auto specialPurpose = [&] { + if (hasSpecialPurposeFlag(f.flags)) { + return getSpecialPurposeType(f.flags); + } else if (SpecialPurpose::isSpecialPurposeFolderName(f.name()) && parentFolderRid.isEmpty()) { + return SpecialPurpose::getSpecialPurposeType(f.name()); + } + return QByteArray{}; + }(); + if (!specialPurpose.isEmpty()) { + folder.setSpecialPurpose(QByteArrayList() << specialPurpose); + mergeCriteria.insert(ApplicationDomain::Folder::SpecialPurpose::name, Query::Comparator(specialPurpose, Query::Comparator::Contains)); } if (!parentFolderRid.isEmpty()) { diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp index 3305f60..36dbcf5 100644 --- a/examples/imapresource/imapserverproxy.cpp +++ b/examples/imapresource/imapserverproxy.cpp @@ -57,6 +57,7 @@ const char* Imap::FolderFlags::Trash = "\\Trash"; const char* Imap::FolderFlags::Archive = "\\Archive"; const char* Imap::FolderFlags::Junk = "\\Junk"; const char* Imap::FolderFlags::Flagged = "\\Flagged"; +const char* Imap::FolderFlags::Drafts = "\\Drafts"; const char* Imap::Capabilities::Namespace = "NAMESPACE"; const char* Imap::Capabilities::Uidplus = "UIDPLUS"; @@ -192,6 +193,12 @@ KAsync::Job ImapServerProxy::logout() } } +bool ImapServerProxy::isGmail() const +{ + //Magic capability that only gmail has + return mCapabilities.contains("X-GM-EXT-1"); +} + KAsync::Job ImapServerProxy::select(const QString &mailbox) { auto select = new KIMAP2::SelectJob(mSession); @@ -441,6 +448,15 @@ QString ImapServerProxy::getNamespace(const QString &name) return ns.name; } +static bool caseInsensitiveContains(const QByteArray &f, const QByteArrayList &list) { + return list.contains(f) || list.contains(f.toLower()); +} + +bool Imap::flagsContain(const QByteArray &f, const QByteArrayList &flags) +{ + return caseInsensitiveContains(f, flags); +} + KAsync::Job ImapServerProxy::fetchFolders(std::function callback) { SinkTrace() << "Fetching folders"; @@ -448,8 +464,21 @@ KAsync::Job ImapServerProxy::fetchFolders(std::function &){ *subscribedList << mailbox.name; }).then(list(KIMAP2::ListJob::IncludeUnsubscribed, [=](const KIMAP2::MailBoxDescriptor &mailbox, const QList &flags) { - bool noselect = flags.contains(QByteArray(FolderFlags::Noselect).toLower()) || flags.contains(QByteArray(FolderFlags::Noselect)); + bool noselect = caseInsensitiveContains(FolderFlags::Noselect, flags); bool subscribed = subscribedList->contains(mailbox.name); + if (isGmail()) { + bool inbox = mailbox.name.toLower() == "inbox"; + bool sent = caseInsensitiveContains(FolderFlags::Sent, flags); + bool drafts = caseInsensitiveContains(FolderFlags::Drafts, flags); + bool trash = caseInsensitiveContains(FolderFlags::Trash, flags); + bool isgmailParent = mailbox.name.toLower() == "[gmail]"; + /** + * Because gmail duplicates messages all over the place we only support a few selected folders for now that should be mostly exclusive. + */ + if (!(inbox || sent || drafts || trash || isgmailParent)) { + return; + } + } SinkLog() << "Found mailbox: " << mailbox.name << flags << FolderFlags::Noselect << noselect << " sub: " << subscribed; auto ns = getNamespace(mailbox.name); callback(Folder{mailbox.name, ns, mailbox.separator, noselect, subscribed, flags}); diff --git a/examples/imapresource/imapserverproxy.h b/examples/imapresource/imapserverproxy.h index 58c49a2..0d70ba5 100644 --- a/examples/imapresource/imapserverproxy.h +++ b/examples/imapresource/imapserverproxy.h @@ -61,6 +61,7 @@ namespace FolderFlags extern const char* Junk; extern const char* Flagged; extern const char* All; + extern const char* Drafts; } namespace Capabilities @@ -79,6 +80,8 @@ struct Message { bool fullPayload; }; +bool flagsContain(const QByteArray &f, const QByteArrayList &flags); + struct Folder { Folder() = default; Folder(const QString &path, const QString &ns, const QChar &separator, bool noselect_, bool subscribed_, const QByteArrayList &flags_) @@ -226,11 +229,6 @@ private: }; class ImapServerProxy { - KIMAP2::Session *mSession; - QStringList mCapabilities; - Namespaces mNamespaces; - - public: ImapServerProxy(const QString &serverUrl, int port, SessionCache *sessionCache = nullptr); @@ -280,9 +278,14 @@ public: KAsync::Job> fetchUids(const Folder &folder); private: + bool isGmail() const; + QString getNamespace(const QString &name); QObject mGuard; SessionCache *mSessionCache; + KIMAP2::Session *mSession; + QStringList mCapabilities; + Namespaces mNamespaces; }; } -- cgit v1.2.3 From 0d9d054be6a1716abff20a49e44d94b9b79ab107 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 21 Apr 2017 13:46:45 +0200 Subject: Apparently this folder can have various names. --- examples/imapresource/imapserverproxy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp index 36dbcf5..08a0081 100644 --- a/examples/imapresource/imapserverproxy.cpp +++ b/examples/imapresource/imapserverproxy.cpp @@ -471,7 +471,7 @@ KAsync::Job ImapServerProxy::fetchFolders(std::function Date: Fri, 21 Apr 2017 17:31:22 +0200 Subject: Make sure we always have a complete hierarchy from the IMAP server --- examples/imapresource/imapresource.cpp | 10 +++++----- examples/imapresource/imapserverproxy.cpp | 26 +++++++++++++++++++++++--- examples/imapresource/imapserverproxy.h | 9 +++++++++ 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 0b3dcf5..8577b8c 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -119,6 +119,8 @@ public: QByteArray createFolder(const Imap::Folder &f) { const auto parentFolderRid = parentRid(f); + bool isToplevel = parentFolderRid.isEmpty(); + SinkTraceCtx(mLogCtx) << "Creating folder: " << f.name() << parentFolderRid << f.flags; const auto remoteId = folderRid(f); @@ -126,22 +128,20 @@ public: folder.setName(f.name()); folder.setIcon("folder"); folder.setEnabled(f.subscribed); - QHash mergeCriteria; auto specialPurpose = [&] { if (hasSpecialPurposeFlag(f.flags)) { return getSpecialPurposeType(f.flags); - } else if (SpecialPurpose::isSpecialPurposeFolderName(f.name()) && parentFolderRid.isEmpty()) { + } else if (SpecialPurpose::isSpecialPurposeFolderName(f.name()) && isToplevel) { return SpecialPurpose::getSpecialPurposeType(f.name()); } return QByteArray{}; }(); if (!specialPurpose.isEmpty()) { folder.setSpecialPurpose(QByteArrayList() << specialPurpose); - mergeCriteria.insert(ApplicationDomain::Folder::SpecialPurpose::name, Query::Comparator(specialPurpose, Query::Comparator::Contains)); } - if (!parentFolderRid.isEmpty()) { - folder.setParent(syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, parentFolderRid)); + if (!isToplevel) { + folder.setParent(syncStore().resolveRemoteId(ApplicationDomain::Folder::name, parentFolderRid)); } createOrModify(ApplicationDomain::getTypeName(), remoteId, folder); return remoteId; diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp index 08a0081..d3ad2d4 100644 --- a/examples/imapresource/imapserverproxy.cpp +++ b/examples/imapresource/imapserverproxy.cpp @@ -457,10 +457,23 @@ bool Imap::flagsContain(const QByteArray &f, const QByteArrayList &flags) return caseInsensitiveContains(f, flags); } +static void reportFolder(const Folder &f, QSharedPointer> reportedList, std::function callback) { + if (!reportedList->contains(f.path())) { + reportedList->insert(f.path()); + auto c = f; + c.noselect = true; + callback(c); + if (!f.parentPath().isEmpty()){ + reportFolder(f.parentFolder(), reportedList, callback); + } + } +} + KAsync::Job ImapServerProxy::fetchFolders(std::function callback) { SinkTrace() << "Fetching folders"; auto subscribedList = QSharedPointer>::create() ; + auto reportedList = QSharedPointer>::create() ; return list(KIMAP2::ListJob::NoOption, [=](const KIMAP2::MailBoxDescriptor &mailbox, const QList &){ *subscribedList << mailbox.name; }).then(list(KIMAP2::ListJob::IncludeUnsubscribed, [=](const KIMAP2::MailBoxDescriptor &mailbox, const QList &flags) { @@ -471,17 +484,24 @@ KAsync::Job ImapServerProxy::fetchFolders(std::functioncontains(folder.parentPath())) { + reportFolder(folder.parentFolder(), reportedList, callback); + } + reportedList->insert(folder.path()); + callback(folder); })); } diff --git a/examples/imapresource/imapserverproxy.h b/examples/imapresource/imapserverproxy.h index 0d70ba5..2d90f39 100644 --- a/examples/imapresource/imapserverproxy.h +++ b/examples/imapresource/imapserverproxy.h @@ -118,6 +118,15 @@ struct Folder { return parentPath; } + Folder parentFolder() const + { + Folder parent; + parent.mPath = parentPath(); + parent.mNamespace = mNamespace; + parent.mSeparator = mSeparator; + return parent; + } + QString name() const { auto pathParts = mPath.split(mSeparator); -- cgit v1.2.3 From 9450bcb17d9633e56bf43242463583ae9c0c53e9 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 26 Apr 2017 13:19:49 +0200 Subject: Fixed build error --- common/notifier.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/notifier.cpp b/common/notifier.cpp index f52e28b..1af65e9 100644 --- a/common/notifier.cpp +++ b/common/notifier.cpp @@ -40,8 +40,8 @@ public: void listenForNotifications(const QSharedPointer &access) { QObject::connect(access.data(), &ResourceAccess::notification, &context, [this](const Notification ¬ification) { - for (const auto &handler : handler) { - handler(notification); + for (const auto &h : handler) { + h(notification); } }); resourceAccess << access; -- cgit v1.2.3 From 350824b0ecacabab7c7bb565df165d8c108ea43f Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 26 Apr 2017 16:50:42 +0200 Subject: Only create one session And not one for every imap proxy --- examples/imapresource/imapserverproxy.cpp | 22 +++++++++++++++------- examples/imapresource/imapserverproxy.h | 7 ++++++- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp index d3ad2d4..7fa0b5a 100644 --- a/examples/imapresource/imapserverproxy.cpp +++ b/examples/imapresource/imapserverproxy.cpp @@ -99,17 +99,25 @@ static KAsync::Job runJob(KJob *job) }); } -ImapServerProxy::ImapServerProxy(const QString &serverUrl, int port, SessionCache *sessionCache) : mSession(new KIMAP2::Session(serverUrl, qint16(port))), mSessionCache(sessionCache) +KIMAP2::Session *createNewSession(const QString &serverUrl, int port) { - QObject::connect(mSession, &KIMAP2::Session::sslErrors, [this](const QList &errors) { + auto newSession = new KIMAP2::Session(serverUrl, qint16(port)); + if (Sink::Test::testModeEnabled()) { + newSession->setTimeout(1); + } else { + newSession->setTimeout(40); + } + QObject::connect(newSession, &KIMAP2::Session::sslErrors, [=](const QList &errors) { SinkLog() << "Received ssl error: " << errors; - mSession->ignoreErrors(errors); + newSession->ignoreErrors(errors); }); + return newSession; +} - if (Sink::Test::testModeEnabled()) { - mSession->setTimeout(1); - } else { - mSession->setTimeout(40); +ImapServerProxy::ImapServerProxy(const QString &serverUrl, int port, SessionCache *sessionCache) : mSessionCache(sessionCache), mSession(nullptr) +{ + if (!mSessionCache || mSessionCache->isEmpty()) { + mSession = createNewSession(serverUrl, port); } } diff --git a/examples/imapresource/imapserverproxy.h b/examples/imapresource/imapserverproxy.h index 2d90f39..82f4f58 100644 --- a/examples/imapresource/imapserverproxy.h +++ b/examples/imapresource/imapserverproxy.h @@ -231,7 +231,12 @@ public: return session; } } - return CachedSession{}; + return {}; + } + + bool isEmpty() const + { + return mSessions.isEmpty(); } private: QList mSessions; -- cgit v1.2.3 From 60aff218bd211f6225423ead429fa83ef5d39494 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 26 Apr 2017 16:55:35 +0200 Subject: Bumped release version --- dist/sink.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/sink.spec b/dist/sink.spec index 246439d..dd43560 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 0%{?dist} +Release: 1%{?dist} Summary: sink Group: Applications/Desktop -- cgit v1.2.3 From 3017e10e7f06f41f1d64996a8191407c7a0a8f69 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 26 Apr 2017 17:54:21 +0200 Subject: We require at leaset 0.1.1 --- examples/imapresource/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/imapresource/CMakeLists.txt b/examples/imapresource/CMakeLists.txt index 46a8b08..15ff135 100644 --- a/examples/imapresource/CMakeLists.txt +++ b/examples/imapresource/CMakeLists.txt @@ -4,7 +4,7 @@ add_definitions(-DQT_PLUGIN) include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) find_package(KF5 COMPONENTS REQUIRED Mime) -find_package(KIMAP2 0.0.1 REQUIRED) +find_package(KIMAP2 0.1.1 REQUIRED) include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -- cgit v1.2.3 From b28d4cf578f4ceabb0d40f65e55ac533027e8733 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 27 Apr 2017 15:52:56 +0200 Subject: We get an offline status notification initially --- common/notification.cpp | 4 ++-- tests/notificationtest.cpp | 22 ++++++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/common/notification.cpp b/common/notification.cpp index e688b6d..da31e20 100644 --- a/common/notification.cpp +++ b/common/notification.cpp @@ -48,8 +48,8 @@ static QByteArray name(int type) QDebug operator<<(QDebug dbg, const Sink::Notification &n) { - dbg << "Notification(Type: " << name(n.type) << "Id, : " << n.id << ", Code: "; + dbg << "Notification(Type:" << name(n.type) << "Id, :" << n.id << ", Code:"; dbg << n.code; - dbg << ", Message: " << n.message << ", Entities: " << n.entities << ")"; + dbg << ", Message:" << n.message << ", Entities:" << n.entities << ")"; return dbg.space(); } diff --git a/tests/notificationtest.cpp b/tests/notificationtest.cpp index a34d325..e268a17 100644 --- a/tests/notificationtest.cpp +++ b/tests/notificationtest.cpp @@ -67,12 +67,22 @@ private slots: VERIFYEXEC(Sink::Store::synchronize(query)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); - QVERIFY(statusNotifications.size() <= 3); - QTRY_COMPARE(statusNotifications.size(), 3); - //Sync - QCOMPARE(statusNotifications.at(0).code, static_cast(ApplicationDomain::Status::ConnectedStatus)); - QCOMPARE(statusNotifications.at(1).code, static_cast(ApplicationDomain::Status::BusyStatus)); - QCOMPARE(statusNotifications.at(2).code, static_cast(ApplicationDomain::Status::ConnectedStatus)); + using namespace Sink::ApplicationDomain; + { + QList expected = { + Status::OfflineStatus, + Status::ConnectedStatus, + Status::BusyStatus, + Status::ConnectedStatus + }; + qInfo() << "Received notifications " << statusNotifications; + QVERIFY2(statusNotifications.size() <= expected.size(), "More notifications than expected."); + QTRY_COMPARE(statusNotifications.size(), expected.size()); + qInfo() << "All received notifications " << statusNotifications; + for (auto i = 0; i < statusNotifications.size(); i++) { + QCOMPARE(statusNotifications.at(i).code, static_cast(expected.at(i))); + } + } //Changereplay // It can happen that we get a changereplay notification pair first and then a second one at the end, // we therefore currently filter all changereplay notifications (see above). -- cgit v1.2.3 From 4db017949e6528867963c5495aa557dcfb6f4765 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 27 Apr 2017 17:06:03 +0200 Subject: Start building with kdav --- dist/sink.spec | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dist/sink.spec b/dist/sink.spec index dd43560..1f48e4f 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 1%{?dist} +Release: 2%{?dist} Summary: sink Group: Applications/Desktop @@ -23,6 +23,7 @@ BuildRequires: libgit2-devel BuildRequires: lmdb-devel BuildRequires: qt5-qtbase-devel BuildRequires: readline-devel +BuildRequires: kdav-devel %description sink @@ -47,7 +48,7 @@ mkdir -p build/ pushd build %{cmake} \ -DQT_PLUGIN_INSTALL_DIR:PATH=%{_libdir}/qt5/plugins/ \ - -DBUILD_DAV=OFF \ + -DBUILD_DAV=ON \ .. make %{?_smp_mflags} -- cgit v1.2.3 From 6b8432a5c0647d8fbe3cda549574ae13e07bb2f4 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 28 Apr 2017 11:48:06 +0200 Subject: Simplified propertymapper --- common/domain/typeimplementations.cpp | 10 ++-- common/domain/typeimplementations.h | 11 ++-- common/domainadaptor.h | 12 ++-- common/propertymapper.h | 109 ++++------------------------------ 4 files changed, 29 insertions(+), 113 deletions(-) diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp index eb3851e..5db35b7 100644 --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -61,7 +61,7 @@ void TypeImplementation::configure(IndexPropertyMapper &indexPropertyMappe }); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) +void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) { propertyMapper.addMapping(&Buffer::sender); propertyMapper.addMapping(&Buffer::to); @@ -108,7 +108,7 @@ void TypeImplementation::configure(TypeIndex &index) index.addProperty(Folder::Name::name); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) +void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) { propertyMapper.addMapping(&Buffer::parent); propertyMapper.addMapping(&Buffer::name); @@ -137,7 +137,7 @@ void TypeImplementation::configure(TypeIndex &index) index.addProperty(Contact::Uid::name); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) +void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) { propertyMapper.addMapping(&Buffer::uid); propertyMapper.addMapping(&Buffer::fn); @@ -171,7 +171,7 @@ void TypeImplementation::configure(TypeIndex &index) index.addProperty(Addressbook::Name::name); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) +void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) { propertyMapper.addMapping(&Buffer::parent); propertyMapper.addMapping(&Buffer::name); @@ -194,7 +194,7 @@ void TypeImplementation::configure(TypeIndex &index) index.addProperty(Event::Uid::name); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) +void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) { propertyMapper.addMapping(&Buffer::summary); propertyMapper.addMapping(&Buffer::description); diff --git a/common/domain/typeimplementations.h b/common/domain/typeimplementations.h index 37d6ca9..705b059 100644 --- a/common/domain/typeimplementations.h +++ b/common/domain/typeimplementations.h @@ -26,7 +26,6 @@ #include "contact_generated.h" #include "addressbook_generated.h" -template class ReadPropertyMapper; template class WritePropertyMapper; @@ -48,7 +47,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Mail Buffer; typedef Sink::ApplicationDomain::Buffer::MailBuilder BufferBuilder; static void configure(TypeIndex &index); - static void configure(ReadPropertyMapper &propertyMapper); + static void configure(ReadPropertyMapper &propertyMapper); static void configure(WritePropertyMapper &propertyMapper); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -59,7 +58,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Folder Buffer; typedef Sink::ApplicationDomain::Buffer::FolderBuilder BufferBuilder; static void configure(TypeIndex &); - static void configure(ReadPropertyMapper &); + static void configure(ReadPropertyMapper &); static void configure(WritePropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -70,7 +69,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Contact Buffer; typedef Sink::ApplicationDomain::Buffer::ContactBuilder BufferBuilder; static void configure(TypeIndex &); - static void configure(ReadPropertyMapper &); + static void configure(ReadPropertyMapper &); static void configure(WritePropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -81,7 +80,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Addressbook Buffer; typedef Sink::ApplicationDomain::Buffer::AddressbookBuilder BufferBuilder; static void configure(TypeIndex &); - static void configure(ReadPropertyMapper &); + static void configure(ReadPropertyMapper &); static void configure(WritePropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -92,7 +91,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Event Buffer; typedef Sink::ApplicationDomain::Buffer::EventBuilder BufferBuilder; static void configure(TypeIndex &); - static void configure(ReadPropertyMapper &); + static void configure(ReadPropertyMapper &); static void configure(WritePropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; diff --git a/common/domainadaptor.h b/common/domainadaptor.h index af5d5fc..9182001 100644 --- a/common/domainadaptor.h +++ b/common/domainadaptor.h @@ -157,8 +157,8 @@ public: LocalBuffer const *mLocalBuffer; ResourceBuffer const *mResourceBuffer; - QSharedPointer> mLocalMapper; - QSharedPointer> mResourceMapper; + QSharedPointer mLocalMapper; + QSharedPointer mResourceMapper; QSharedPointer mIndexMapper; TypeIndex *mIndex; }; @@ -176,8 +176,8 @@ class SINK_EXPORT DomainTypeAdaptorFactory : public DomainTypeAdaptorFactoryInte public: DomainTypeAdaptorFactory() - : mLocalMapper(QSharedPointer>::create()), - mResourceMapper(QSharedPointer>::create()), + : mLocalMapper(QSharedPointer::create()), + mResourceMapper(QSharedPointer::create()), mLocalWriteMapper(QSharedPointer>::create()), mResourceWriteMapper(QSharedPointer>::create()), mIndexMapper(QSharedPointer::create()) @@ -236,8 +236,8 @@ public: protected: - QSharedPointer> mLocalMapper; - QSharedPointer> mResourceMapper; + QSharedPointer mLocalMapper; + QSharedPointer mResourceMapper; QSharedPointer> mLocalWriteMapper; QSharedPointer> mResourceWriteMapper; QSharedPointer mIndexMapper; diff --git a/common/propertymapper.h b/common/propertymapper.h index 9ea0b73..bca45aa 100644 --- a/common/propertymapper.h +++ b/common/propertymapper.h @@ -65,13 +65,12 @@ QVariant SINK_EXPORT propertyToVariant(const flatbuffers::Vector class ReadPropertyMapper { public: virtual ~ReadPropertyMapper(){}; - virtual QVariant getProperty(const QByteArray &key, BufferType const *buffer) const + virtual QVariant getProperty(const QByteArray &key, void const *buffer) const { if (mReadAccessors.contains(key)) { auto accessor = mReadAccessors.value(key); @@ -79,69 +78,30 @@ public: } return QVariant(); } + bool hasMapping(const QByteArray &key) const { return mReadAccessors.contains(key); } + QList availableProperties() const { return mReadAccessors.keys(); } - void addMapping(const QByteArray &property, const std::function &mapping) - { - mReadAccessors.insert(property, mapping); - } - - template - void addMapping(const flatbuffers::String *(Buffer::*f)() const) - { - addMapping(T::name, [f](Buffer const *buffer) -> QVariant { return propertyToVariant((buffer->*f)()); }); - } - - template - void addMapping(uint8_t (Buffer::*f)() const) - { - addMapping(T::name, [f](Buffer const *buffer) -> QVariant { return propertyToVariant((buffer->*f)()); }); - } - - template - void addMapping(bool (Buffer::*f)() const) - { - addMapping(T::name, [f](Buffer const *buffer) -> QVariant { return propertyToVariant((buffer->*f)()); }); - } - - template - void addMapping(const flatbuffers::Vector *(Buffer::*f)() const) - { - addMapping(T::name, [f](Buffer const *buffer) -> QVariant { return propertyToVariant((buffer->*f)()); }); - } - template - void addMapping(const flatbuffers::Vector> *(Buffer::*f)() const) + void addMapping(const QByteArray &property, const std::function &mapping) { - addMapping(T::name, [f](Buffer const *buffer) -> QVariant { return propertyToVariant((buffer->*f)()); }); - } - - template - void addMapping(const Sink::ApplicationDomain::Buffer::MailContact *(Buffer::*f)() const) - { - addMapping(T::name, [f](Buffer const *buffer) -> QVariant { return propertyToVariant((buffer->*f)()); }); - } - - template - void addMapping(const flatbuffers::Vector> *(Buffer::*f)() const) - { - addMapping(T::name, [f](Buffer const *buffer) -> QVariant { return propertyToVariant((buffer->*f)()); }); + mReadAccessors.insert(property, mapping); } - template - void addMapping(const flatbuffers::Vector> *(Buffer::*f)() const) + template + void addMapping(FunctionReturnValue (Buffer::*f)() const) { - addMapping(T::name, [f](Buffer const *buffer) -> QVariant { return propertyToVariant((buffer->*f)()); }); + addMapping(T::name, [f](void const *buffer) -> QVariant { return propertyToVariant((static_cast(buffer)->*f)()); }); } private: - QHash> mReadAccessors; + QHash> mReadAccessors; }; template @@ -157,10 +117,12 @@ public: builderCalls << accessor(value, fbb); } } + bool hasMapping(const QByteArray &key) const { return mWriteAccessors.contains(key); } + void addMapping(const QByteArray &property, const std::function(const QVariant &, flatbuffers::FlatBufferBuilder &)> &mapping) { mWriteAccessors.insert(property, mapping); @@ -182,53 +144,8 @@ public: }); } - template - void addMapping(void (BufferBuilder::*f)(flatbuffers::Offset)) - { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { - auto offset = variantToProperty(value, fbb); - return [offset, f](BufferBuilder &builder) { (builder.*f)(offset); }; - }); - } - - template - void addMapping(void (BufferBuilder::*f)(flatbuffers::Offset>)) - { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { - auto offset = variantToProperty(value, fbb); - return [offset, f](BufferBuilder &builder) { (builder.*f)(offset); }; - }); - } - - template - void addMapping(void (BufferBuilder::*f)(flatbuffers::Offset>>)) - { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { - auto offset = variantToProperty(value, fbb); - return [offset, f](BufferBuilder &builder) { (builder.*f)(offset); }; - }); - } - - template - void addMapping(void (BufferBuilder::*f)(flatbuffers::Offset)) - { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { - auto offset = variantToProperty(value, fbb); - return [offset, f](BufferBuilder &builder) { (builder.*f)(offset); }; - }); - } - - template - void addMapping(void (BufferBuilder::*f)(flatbuffers::Offset>>)) - { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { - auto offset = variantToProperty(value, fbb); - return [offset, f](BufferBuilder &builder) { (builder.*f)(offset); }; - }); - } - - template - void addMapping(void (BufferBuilder::*f)(flatbuffers::Offset>>)) + template + void addMapping(void (BufferBuilder::*f)(flatbuffers::Offset)) { addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { auto offset = variantToProperty(value, fbb); -- cgit v1.2.3 From a4ce2b2eda8a2fea4263017e6868c327f22f5e47 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 28 Apr 2017 12:19:52 +0200 Subject: Removed the template argument from the write property mapper --- common/domain/typeimplementations.cpp | 80 +++++++++++++++++------------------ common/domain/typeimplementations.h | 11 +++-- common/domainadaptor.h | 16 +++---- common/propertymapper.h | 25 ++++++----- tests/domainadaptortest.cpp | 8 ++-- 5 files changed, 69 insertions(+), 71 deletions(-) diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp index 5db35b7..598e5a7 100644 --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -81,24 +81,24 @@ void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) propertyMapper.addMapping(&Buffer::parentMessageId); } -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) -{ - propertyMapper.addMapping(&BufferBuilder::add_sender); - propertyMapper.addMapping(&BufferBuilder::add_to); - propertyMapper.addMapping(&BufferBuilder::add_cc); - propertyMapper.addMapping(&BufferBuilder::add_bcc); - propertyMapper.addMapping(&BufferBuilder::add_subject); - propertyMapper.addMapping(&BufferBuilder::add_date); - propertyMapper.addMapping(&BufferBuilder::add_unread); - propertyMapper.addMapping(&BufferBuilder::add_important); - propertyMapper.addMapping(&BufferBuilder::add_folder); - propertyMapper.addMapping(&BufferBuilder::add_mimeMessage); - propertyMapper.addMapping(&BufferBuilder::add_fullPayloadAvailable); - propertyMapper.addMapping(&BufferBuilder::add_draft); - propertyMapper.addMapping(&BufferBuilder::add_trash); - propertyMapper.addMapping(&BufferBuilder::add_sent); - propertyMapper.addMapping(&BufferBuilder::add_messageId); - propertyMapper.addMapping(&BufferBuilder::add_parentMessageId); +void TypeImplementation::configure(WritePropertyMapper &propertyMapper) +{ + propertyMapper.addMapping(&BufferBuilder::add_sender); + propertyMapper.addMapping(&BufferBuilder::add_to); + propertyMapper.addMapping(&BufferBuilder::add_cc); + propertyMapper.addMapping(&BufferBuilder::add_bcc); + propertyMapper.addMapping(&BufferBuilder::add_subject); + propertyMapper.addMapping(&BufferBuilder::add_date); + propertyMapper.addMapping(&BufferBuilder::add_unread); + propertyMapper.addMapping(&BufferBuilder::add_important); + propertyMapper.addMapping(&BufferBuilder::add_folder); + propertyMapper.addMapping(&BufferBuilder::add_mimeMessage); + propertyMapper.addMapping(&BufferBuilder::add_fullPayloadAvailable); + propertyMapper.addMapping(&BufferBuilder::add_draft); + propertyMapper.addMapping(&BufferBuilder::add_trash); + propertyMapper.addMapping(&BufferBuilder::add_sent); + propertyMapper.addMapping(&BufferBuilder::add_messageId); + propertyMapper.addMapping(&BufferBuilder::add_parentMessageId); } @@ -117,13 +117,13 @@ void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) propertyMapper.addMapping(&Buffer::enabled); } -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) +void TypeImplementation::configure(WritePropertyMapper &propertyMapper) { - propertyMapper.addMapping(&BufferBuilder::add_parent); - propertyMapper.addMapping(&BufferBuilder::add_name); - propertyMapper.addMapping(&BufferBuilder::add_icon); - propertyMapper.addMapping(&BufferBuilder::add_specialpurpose); - propertyMapper.addMapping(&BufferBuilder::add_enabled); + propertyMapper.addMapping(&BufferBuilder::add_parent); + propertyMapper.addMapping(&BufferBuilder::add_name); + propertyMapper.addMapping(&BufferBuilder::add_icon); + propertyMapper.addMapping(&BufferBuilder::add_specialpurpose); + propertyMapper.addMapping(&BufferBuilder::add_enabled); } void TypeImplementation::configure(IndexPropertyMapper &) @@ -148,15 +148,15 @@ void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) propertyMapper.addMapping(&Buffer::lastname); } -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) +void TypeImplementation::configure(WritePropertyMapper &propertyMapper) { - propertyMapper.addMapping(&BufferBuilder::add_uid); - propertyMapper.addMapping(&BufferBuilder::add_fn); - propertyMapper.addMapping(&BufferBuilder::add_emails); - propertyMapper.addMapping(&BufferBuilder::add_vcard); - propertyMapper.addMapping(&BufferBuilder::add_addressbook); - propertyMapper.addMapping(&BufferBuilder::add_firstname); - propertyMapper.addMapping(&BufferBuilder::add_lastname); + propertyMapper.addMapping(&BufferBuilder::add_uid); + propertyMapper.addMapping(&BufferBuilder::add_fn); + propertyMapper.addMapping(&BufferBuilder::add_emails); + propertyMapper.addMapping(&BufferBuilder::add_vcard); + propertyMapper.addMapping(&BufferBuilder::add_addressbook); + propertyMapper.addMapping(&BufferBuilder::add_firstname); + propertyMapper.addMapping(&BufferBuilder::add_lastname); } void TypeImplementation::configure(IndexPropertyMapper &) @@ -177,10 +177,10 @@ void TypeImplementation::configure(ReadPropertyMapper &propertyMapp propertyMapper.addMapping(&Buffer::name); } -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) +void TypeImplementation::configure(WritePropertyMapper &propertyMapper) { - propertyMapper.addMapping(&BufferBuilder::add_parent); - propertyMapper.addMapping(&BufferBuilder::add_name); + propertyMapper.addMapping(&BufferBuilder::add_parent); + propertyMapper.addMapping(&BufferBuilder::add_name); } void TypeImplementation::configure(IndexPropertyMapper &) @@ -202,12 +202,12 @@ void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) propertyMapper.addMapping(&Buffer::attachment); } -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) +void TypeImplementation::configure(WritePropertyMapper &propertyMapper) { - propertyMapper.addMapping(&BufferBuilder::add_summary); - propertyMapper.addMapping(&BufferBuilder::add_description); - propertyMapper.addMapping(&BufferBuilder::add_uid); - propertyMapper.addMapping(&BufferBuilder::add_attachment); + propertyMapper.addMapping(&BufferBuilder::add_summary); + propertyMapper.addMapping(&BufferBuilder::add_description); + propertyMapper.addMapping(&BufferBuilder::add_uid); + propertyMapper.addMapping(&BufferBuilder::add_attachment); } void TypeImplementation::configure(IndexPropertyMapper &) diff --git a/common/domain/typeimplementations.h b/common/domain/typeimplementations.h index 705b059..8acea58 100644 --- a/common/domain/typeimplementations.h +++ b/common/domain/typeimplementations.h @@ -27,7 +27,6 @@ #include "addressbook_generated.h" class ReadPropertyMapper; -template class WritePropertyMapper; class IndexPropertyMapper; @@ -48,7 +47,7 @@ public: typedef Sink::ApplicationDomain::Buffer::MailBuilder BufferBuilder; static void configure(TypeIndex &index); static void configure(ReadPropertyMapper &propertyMapper); - static void configure(WritePropertyMapper &propertyMapper); + static void configure(WritePropertyMapper &propertyMapper); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -59,7 +58,7 @@ public: typedef Sink::ApplicationDomain::Buffer::FolderBuilder BufferBuilder; static void configure(TypeIndex &); static void configure(ReadPropertyMapper &); - static void configure(WritePropertyMapper &); + static void configure(WritePropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -70,7 +69,7 @@ public: typedef Sink::ApplicationDomain::Buffer::ContactBuilder BufferBuilder; static void configure(TypeIndex &); static void configure(ReadPropertyMapper &); - static void configure(WritePropertyMapper &); + static void configure(WritePropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -81,7 +80,7 @@ public: typedef Sink::ApplicationDomain::Buffer::AddressbookBuilder BufferBuilder; static void configure(TypeIndex &); static void configure(ReadPropertyMapper &); - static void configure(WritePropertyMapper &); + static void configure(WritePropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -92,7 +91,7 @@ public: typedef Sink::ApplicationDomain::Buffer::EventBuilder BufferBuilder; static void configure(TypeIndex &); static void configure(ReadPropertyMapper &); - static void configure(WritePropertyMapper &); + static void configure(WritePropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; diff --git a/common/domainadaptor.h b/common/domainadaptor.h index 9182001..2a7924a 100644 --- a/common/domainadaptor.h +++ b/common/domainadaptor.h @@ -40,10 +40,10 @@ */ template flatbuffers::Offset -createBufferPart(const Sink::ApplicationDomain::ApplicationDomainType &domainObject, flatbuffers::FlatBufferBuilder &fbb, const WritePropertyMapper &mapper) +createBufferPart(const Sink::ApplicationDomain::ApplicationDomainType &domainObject, flatbuffers::FlatBufferBuilder &fbb, const WritePropertyMapper &mapper) { // First create a primitives such as strings using the mappings - QList> propertiesToAddToResource; + QList> propertiesToAddToResource; for (const auto &property : domainObject.changedProperties()) { // SinkTrace() << "copying property " << property; const auto value = domainObject.getProperty(property); @@ -57,7 +57,7 @@ createBufferPart(const Sink::ApplicationDomain::ApplicationDomainType &domainObj // Then create all porperties using the above generated builderCalls Builder builder(fbb); for (auto propertyBuilder : propertiesToAddToResource) { - propertyBuilder(builder); + propertyBuilder(&builder); } return builder.Finish(); } @@ -68,7 +68,7 @@ createBufferPart(const Sink::ApplicationDomain::ApplicationDomainType &domainObj * After this the buffer can be extracted from the FlatBufferBuilder object. */ template -static void createBufferPartBuffer(const Sink::ApplicationDomain::ApplicationDomainType &domainObject, flatbuffers::FlatBufferBuilder &fbb, WritePropertyMapper &mapper) +static void createBufferPartBuffer(const Sink::ApplicationDomain::ApplicationDomainType &domainObject, flatbuffers::FlatBufferBuilder &fbb, WritePropertyMapper &mapper) { auto pos = createBufferPart(domainObject, fbb, mapper); // Because we cannot template the following call @@ -178,8 +178,8 @@ public: DomainTypeAdaptorFactory() : mLocalMapper(QSharedPointer::create()), mResourceMapper(QSharedPointer::create()), - mLocalWriteMapper(QSharedPointer>::create()), - mResourceWriteMapper(QSharedPointer>::create()), + mLocalWriteMapper(QSharedPointer::create()), + mResourceWriteMapper(QSharedPointer::create()), mIndexMapper(QSharedPointer::create()) { Sink::ApplicationDomain::TypeImplementation::configure(*mLocalMapper); @@ -238,8 +238,8 @@ public: protected: QSharedPointer mLocalMapper; QSharedPointer mResourceMapper; - QSharedPointer> mLocalWriteMapper; - QSharedPointer> mResourceWriteMapper; + QSharedPointer mLocalWriteMapper; + QSharedPointer mResourceWriteMapper; QSharedPointer mIndexMapper; }; diff --git a/common/propertymapper.h b/common/propertymapper.h index bca45aa..fa4592b 100644 --- a/common/propertymapper.h +++ b/common/propertymapper.h @@ -104,13 +104,12 @@ private: QHash> mReadAccessors; }; -template class WritePropertyMapper { public: virtual ~WritePropertyMapper(){}; - virtual void setProperty(const QByteArray &key, const QVariant &value, QList> &builderCalls, flatbuffers::FlatBufferBuilder &fbb) const + virtual void setProperty(const QByteArray &key, const QVariant &value, QList> &builderCalls, flatbuffers::FlatBufferBuilder &fbb) const { if (mWriteAccessors.contains(key)) { auto accessor = mWriteAccessors.value(key); @@ -123,36 +122,36 @@ public: return mWriteAccessors.contains(key); } - void addMapping(const QByteArray &property, const std::function(const QVariant &, flatbuffers::FlatBufferBuilder &)> &mapping) + void addMapping(const QByteArray &property, const std::function(const QVariant &, flatbuffers::FlatBufferBuilder &)> &mapping) { mWriteAccessors.insert(property, mapping); } - template + template void addMapping(void (BufferBuilder::*f)(uint8_t)) { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { - return [value, f](BufferBuilder &builder) { (builder.*f)(value.value()); }; + addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { + return [value, f](void *builder) { (static_cast(builder)->*f)(value.value()); }; }); } - template + template void addMapping(void (BufferBuilder::*f)(bool)) { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { - return [value, f](BufferBuilder &builder) { (builder.*f)(value.value()); }; + addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { + return [value, f](void *builder) { (static_cast(builder)->*f)(value.value()); }; }); } - template + template void addMapping(void (BufferBuilder::*f)(flatbuffers::Offset)) { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { + addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { auto offset = variantToProperty(value, fbb); - return [offset, f](BufferBuilder &builder) { (builder.*f)(offset); }; + return [offset, f](void *builder) { (static_cast(builder)->*f)(offset); }; }); } private: - QHash(const QVariant &, flatbuffers::FlatBufferBuilder &)>> mWriteAccessors; + QHash(const QVariant &, flatbuffers::FlatBufferBuilder &)>> mWriteAccessors; }; diff --git a/tests/domainadaptortest.cpp b/tests/domainadaptortest.cpp index a17152e..c5f6e3b 100644 --- a/tests/domainadaptortest.cpp +++ b/tests/domainadaptortest.cpp @@ -19,7 +19,7 @@ class TestFactory : public DomainTypeAdaptorFactory>::create(); + mResourceWriteMapper = QSharedPointer::create(); Sink::ApplicationDomain::TypeImplementation::configure(*mResourceWriteMapper); } }; @@ -29,7 +29,7 @@ class TestMailFactory : public DomainTypeAdaptorFactory>::create(); + mResourceWriteMapper = QSharedPointer::create(); Sink::ApplicationDomain::TypeImplementation::configure(*mResourceWriteMapper); } }; @@ -51,7 +51,7 @@ private slots: void testCreateBufferPart() { - auto writeMapper = QSharedPointer>::create(); + auto writeMapper = QSharedPointer::create(); Sink::ApplicationDomain::TypeImplementation::configure(*writeMapper); Sink::ApplicationDomain::Event event; @@ -104,7 +104,7 @@ private slots: void testMail() { - auto writeMapper = QSharedPointer>::create(); + auto writeMapper = QSharedPointer::create(); Sink::ApplicationDomain::TypeImplementation::configure(*writeMapper); Sink::ApplicationDomain::Mail mail; -- cgit v1.2.3 From 5b408ab7e4a2921aea50153782b126e146faeb53 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 28 Apr 2017 12:23:20 +0200 Subject: Removed more unnecessary template arguments --- common/domainadaptor.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/common/domainadaptor.h b/common/domainadaptor.h index 2a7924a..3fbb95f 100644 --- a/common/domainadaptor.h +++ b/common/domainadaptor.h @@ -120,7 +120,6 @@ private: /** * A generic adaptor implementation that uses a property mapper to read/write values. */ -template class DatastoreBufferAdaptor : public Sink::ApplicationDomain::BufferAdaptor { SINK_DEBUG_AREA("bufferadaptor") @@ -155,8 +154,8 @@ public: return mResourceMapper->availableProperties() + mLocalMapper->availableProperties() + mIndexMapper->availableProperties(); } - LocalBuffer const *mLocalBuffer; - ResourceBuffer const *mResourceBuffer; + void const *mLocalBuffer; + void const *mResourceBuffer; QSharedPointer mLocalMapper; QSharedPointer mResourceMapper; QSharedPointer mIndexMapper; @@ -196,7 +195,7 @@ public: */ virtual QSharedPointer createAdaptor(const Sink::Entity &entity, TypeIndex *index = nullptr) Q_DECL_OVERRIDE { - auto adaptor = QSharedPointer>::create(); + auto adaptor = QSharedPointer::create(); adaptor->mLocalBuffer = Sink::EntityBuffer::readBuffer(entity.local()); adaptor->mLocalMapper = mLocalMapper; adaptor->mResourceBuffer = Sink::EntityBuffer::readBuffer(entity.resource()); -- cgit v1.2.3 From a08984c450b1cd2584272b0d57a2f95ae3d074c3 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 28 Apr 2017 15:25:47 +0200 Subject: Removed the resource mapper --- common/domainadaptor.h | 30 ++++++------------------------ examples/dummyresource/domainadaptor.cpp | 3 --- examples/dummyresource/domainadaptor.h | 8 +++----- tests/domainadaptortest.cpp | 8 ++------ tests/testimplementations.h | 4 ++-- 5 files changed, 13 insertions(+), 40 deletions(-) diff --git a/common/domainadaptor.h b/common/domainadaptor.h index 3fbb95f..1db43ca 100644 --- a/common/domainadaptor.h +++ b/common/domainadaptor.h @@ -29,11 +29,9 @@ #include "domain/typeimplementations.h" #include "bufferadaptor.h" #include "entity_generated.h" -#include "metadata_generated.h" #include "entitybuffer.h" #include "propertymapper.h" #include "log.h" -#include "dummy_generated.h" /** * Create a buffer from a domain object using the provided mappings @@ -136,9 +134,7 @@ public: virtual QVariant getProperty(const QByteArray &key) const Q_DECL_OVERRIDE { - if (mResourceBuffer && mResourceMapper->hasMapping(key)) { - return mResourceMapper->getProperty(key, mResourceBuffer); - } else if (mLocalBuffer && mLocalMapper->hasMapping(key)) { + if (mLocalBuffer && mLocalMapper->hasMapping(key)) { return mLocalMapper->getProperty(key, mLocalBuffer); } else if (mIndex && mIndexMapper->hasMapping(key)) { return mIndexMapper->getProperty(key, *mIndex, *this); @@ -151,23 +147,21 @@ public: */ virtual QList availableProperties() const Q_DECL_OVERRIDE { - return mResourceMapper->availableProperties() + mLocalMapper->availableProperties() + mIndexMapper->availableProperties(); + return mLocalMapper->availableProperties() + mIndexMapper->availableProperties(); } void const *mLocalBuffer; - void const *mResourceBuffer; QSharedPointer mLocalMapper; - QSharedPointer mResourceMapper; QSharedPointer mIndexMapper; TypeIndex *mIndex; }; /** - * The factory should define how to go from an entitybuffer (local + resource buffer), to a domain type adapter. + * The factory should define how to go from an entitybuffer (local buffer), to a domain type adapter. * It defines how values are split accross local and resource buffer. * This is required by the facade the read the value, and by the pipeline preprocessors to access the domain values in a generic way. */ -template +template class SINK_EXPORT DomainTypeAdaptorFactory : public DomainTypeAdaptorFactoryInterface { typedef typename Sink::ApplicationDomain::TypeImplementation::Buffer LocalBuffer; @@ -176,9 +170,7 @@ class SINK_EXPORT DomainTypeAdaptorFactory : public DomainTypeAdaptorFactoryInte public: DomainTypeAdaptorFactory() : mLocalMapper(QSharedPointer::create()), - mResourceMapper(QSharedPointer::create()), mLocalWriteMapper(QSharedPointer::create()), - mResourceWriteMapper(QSharedPointer::create()), mIndexMapper(QSharedPointer::create()) { Sink::ApplicationDomain::TypeImplementation::configure(*mLocalMapper); @@ -189,7 +181,7 @@ public: virtual ~DomainTypeAdaptorFactory(){}; /** - * Creates an adaptor for the given domain and resource types. + * Creates an adaptor for the given domain types. * * This returns by default a DatastoreBufferAdaptor initialized with the corresponding property mappers. */ @@ -198,8 +190,6 @@ public: auto adaptor = QSharedPointer::create(); adaptor->mLocalBuffer = Sink::EntityBuffer::readBuffer(entity.local()); adaptor->mLocalMapper = mLocalMapper; - adaptor->mResourceBuffer = Sink::EntityBuffer::readBuffer(entity.resource()); - adaptor->mResourceMapper = mResourceMapper; adaptor->mIndexMapper = mIndexMapper; adaptor->mIndex = index; return adaptor; @@ -214,13 +204,7 @@ public: createBufferPartBuffer(domainObject, localFbb, *mLocalWriteMapper); } - flatbuffers::FlatBufferBuilder resFbb; - if (mResourceWriteMapper) { - // SinkTrace() << "Creating resouce buffer part"; - createBufferPartBuffer(domainObject, resFbb, *mResourceWriteMapper); - } - - Sink::EntityBuffer::assembleEntityBuffer(fbb, metadataData, metadataSize, resFbb.GetBufferPointer(), resFbb.GetSize(), localFbb.GetBufferPointer(), localFbb.GetSize()); + Sink::EntityBuffer::assembleEntityBuffer(fbb, metadataData, metadataSize, 0, 0, localFbb.GetBufferPointer(), localFbb.GetSize()); return true; } @@ -236,9 +220,7 @@ public: protected: QSharedPointer mLocalMapper; - QSharedPointer mResourceMapper; QSharedPointer mLocalWriteMapper; - QSharedPointer mResourceWriteMapper; QSharedPointer mIndexMapper; }; diff --git a/examples/dummyresource/domainadaptor.cpp b/examples/dummyresource/domainadaptor.cpp index dcc08c7..e7a20da 100644 --- a/examples/dummyresource/domainadaptor.cpp +++ b/examples/dummyresource/domainadaptor.cpp @@ -28,9 +28,6 @@ using namespace flatbuffers; DummyEventAdaptorFactory::DummyEventAdaptorFactory() : DomainTypeAdaptorFactory() { - //TODO turn this into initializeReadPropertyMapper as well? - mResourceMapper->addMapping(&DummyEvent::summary); - mResourceWriteMapper->addMapping(&DummyEventBuilder::add_summary); } DummyMailAdaptorFactory::DummyMailAdaptorFactory() diff --git a/examples/dummyresource/domainadaptor.h b/examples/dummyresource/domainadaptor.h index e7098e9..3faaa63 100644 --- a/examples/dummyresource/domainadaptor.h +++ b/examples/dummyresource/domainadaptor.h @@ -22,25 +22,23 @@ #include "event_generated.h" #include "mail_generated.h" #include "folder_generated.h" -#include "dummy_generated.h" -#include "dummycalendar_generated.h" #include "entity_generated.h" -class DummyEventAdaptorFactory : public DomainTypeAdaptorFactory +class DummyEventAdaptorFactory : public DomainTypeAdaptorFactory { public: DummyEventAdaptorFactory(); virtual ~DummyEventAdaptorFactory() {}; }; -class DummyMailAdaptorFactory : public DomainTypeAdaptorFactory +class DummyMailAdaptorFactory : public DomainTypeAdaptorFactory { public: DummyMailAdaptorFactory(); virtual ~DummyMailAdaptorFactory() {}; }; -class DummyFolderAdaptorFactory : public DomainTypeAdaptorFactory +class DummyFolderAdaptorFactory : public DomainTypeAdaptorFactory { public: DummyFolderAdaptorFactory(); diff --git a/tests/domainadaptortest.cpp b/tests/domainadaptortest.cpp index c5f6e3b..2b7455a 100644 --- a/tests/domainadaptortest.cpp +++ b/tests/domainadaptortest.cpp @@ -14,23 +14,19 @@ #include "metadata_generated.h" #include "entity_generated.h" -class TestFactory : public DomainTypeAdaptorFactory +class TestFactory : public DomainTypeAdaptorFactory { public: TestFactory() { - mResourceWriteMapper = QSharedPointer::create(); - Sink::ApplicationDomain::TypeImplementation::configure(*mResourceWriteMapper); } }; -class TestMailFactory : public DomainTypeAdaptorFactory +class TestMailFactory : public DomainTypeAdaptorFactory { public: TestMailFactory() { - mResourceWriteMapper = QSharedPointer::create(); - Sink::ApplicationDomain::TypeImplementation::configure(*mResourceWriteMapper); } }; diff --git a/tests/testimplementations.h b/tests/testimplementations.h index a145265..ff9d9b8 100644 --- a/tests/testimplementations.h +++ b/tests/testimplementations.h @@ -33,7 +33,7 @@ #include "mail_generated.h" #include "createentity_generated.h" -class TestEventAdaptorFactory : public DomainTypeAdaptorFactory +class TestEventAdaptorFactory : public DomainTypeAdaptorFactory { public: TestEventAdaptorFactory() : DomainTypeAdaptorFactory() @@ -43,7 +43,7 @@ public: virtual ~TestEventAdaptorFactory(){}; }; -class TestMailAdaptorFactory : public DomainTypeAdaptorFactory +class TestMailAdaptorFactory : public DomainTypeAdaptorFactory { public: TestMailAdaptorFactory() : DomainTypeAdaptorFactory() -- cgit v1.2.3 From ca5020095abfb76e63bd801e9722c07193eb05f5 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 28 Apr 2017 15:29:18 +0200 Subject: A single propertymapper for both directions --- common/domain/typeimplementations.cpp | 137 +++++++++++----------------------- common/domain/typeimplementations.h | 18 ++--- common/domainadaptor.h | 23 ++---- common/propertymapper.h | 64 ++++++++-------- tests/domainadaptortest.cpp | 4 +- 5 files changed, 89 insertions(+), 157 deletions(-) diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp index 598e5a7..a87e73d 100644 --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -32,6 +32,10 @@ using namespace Sink; using namespace Sink::ApplicationDomain; +#define SINK_REGISTER_SERIALIZER(MAPPER, ENTITYTYPE, PROPERTY, LOWERCASEPROPERTY) \ + MAPPER.addMapping(&Sink::ApplicationDomain::Buffer::ENTITYTYPE::LOWERCASEPROPERTY, &Sink::ApplicationDomain::Buffer::ENTITYTYPE##Builder::add_##LOWERCASEPROPERTY); + + void TypeImplementation::configure(TypeIndex &index) { // index.addProperty(); @@ -61,69 +65,39 @@ void TypeImplementation::configure(IndexPropertyMapper &indexPropertyMappe }); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) -{ - propertyMapper.addMapping(&Buffer::sender); - propertyMapper.addMapping(&Buffer::to); - propertyMapper.addMapping(&Buffer::cc); - propertyMapper.addMapping(&Buffer::bcc); - propertyMapper.addMapping(&Buffer::subject); - propertyMapper.addMapping(&Buffer::date); - propertyMapper.addMapping(&Buffer::unread); - propertyMapper.addMapping(&Buffer::important); - propertyMapper.addMapping(&Buffer::folder); - propertyMapper.addMapping(&Buffer::mimeMessage); - propertyMapper.addMapping(&Buffer::fullPayloadAvailable); - propertyMapper.addMapping(&Buffer::draft); - propertyMapper.addMapping(&Buffer::trash); - propertyMapper.addMapping(&Buffer::sent); - propertyMapper.addMapping(&Buffer::messageId); - propertyMapper.addMapping(&Buffer::parentMessageId); -} - -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) +void TypeImplementation::configure(PropertyMapper &propertyMapper) { - propertyMapper.addMapping(&BufferBuilder::add_sender); - propertyMapper.addMapping(&BufferBuilder::add_to); - propertyMapper.addMapping(&BufferBuilder::add_cc); - propertyMapper.addMapping(&BufferBuilder::add_bcc); - propertyMapper.addMapping(&BufferBuilder::add_subject); - propertyMapper.addMapping(&BufferBuilder::add_date); - propertyMapper.addMapping(&BufferBuilder::add_unread); - propertyMapper.addMapping(&BufferBuilder::add_important); - propertyMapper.addMapping(&BufferBuilder::add_folder); - propertyMapper.addMapping(&BufferBuilder::add_mimeMessage); - propertyMapper.addMapping(&BufferBuilder::add_fullPayloadAvailable); - propertyMapper.addMapping(&BufferBuilder::add_draft); - propertyMapper.addMapping(&BufferBuilder::add_trash); - propertyMapper.addMapping(&BufferBuilder::add_sent); - propertyMapper.addMapping(&BufferBuilder::add_messageId); - propertyMapper.addMapping(&BufferBuilder::add_parentMessageId); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Sender, sender); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, To, to); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Cc, cc); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Bcc, bcc); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Subject, subject); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Date, date); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Unread, unread); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Important, important); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Folder, folder); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, MimeMessage, mimeMessage); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, FullPayloadAvailable, fullPayloadAvailable); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Draft, draft); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Trash, trash); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, Sent, sent); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, MessageId, messageId); + SINK_REGISTER_SERIALIZER(propertyMapper, Mail, ParentMessageId, parentMessageId); } - void TypeImplementation::configure(TypeIndex &index) { index.addProperty(Folder::Parent::name); index.addProperty(Folder::Name::name); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) -{ - propertyMapper.addMapping(&Buffer::parent); - propertyMapper.addMapping(&Buffer::name); - propertyMapper.addMapping(&Buffer::icon); - propertyMapper.addMapping(&Buffer::specialpurpose); - propertyMapper.addMapping(&Buffer::enabled); -} - -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) +void TypeImplementation::configure(PropertyMapper &propertyMapper) { - propertyMapper.addMapping(&BufferBuilder::add_parent); - propertyMapper.addMapping(&BufferBuilder::add_name); - propertyMapper.addMapping(&BufferBuilder::add_icon); - propertyMapper.addMapping(&BufferBuilder::add_specialpurpose); - propertyMapper.addMapping(&BufferBuilder::add_enabled); + SINK_REGISTER_SERIALIZER(propertyMapper, Folder, Parent, parent); + SINK_REGISTER_SERIALIZER(propertyMapper, Folder, Name, name); + SINK_REGISTER_SERIALIZER(propertyMapper, Folder, Icon, icon); + SINK_REGISTER_SERIALIZER(propertyMapper, Folder, SpecialPurpose, specialpurpose); + SINK_REGISTER_SERIALIZER(propertyMapper, Folder, Enabled, enabled); } void TypeImplementation::configure(IndexPropertyMapper &) @@ -137,26 +111,15 @@ void TypeImplementation::configure(TypeIndex &index) index.addProperty(Contact::Uid::name); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) +void TypeImplementation::configure(PropertyMapper &propertyMapper) { - propertyMapper.addMapping(&Buffer::uid); - propertyMapper.addMapping(&Buffer::fn); - propertyMapper.addMapping(&Buffer::emails); - propertyMapper.addMapping(&Buffer::vcard); - propertyMapper.addMapping(&Buffer::addressbook); - propertyMapper.addMapping(&Buffer::firstname); - propertyMapper.addMapping(&Buffer::lastname); -} - -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) -{ - propertyMapper.addMapping(&BufferBuilder::add_uid); - propertyMapper.addMapping(&BufferBuilder::add_fn); - propertyMapper.addMapping(&BufferBuilder::add_emails); - propertyMapper.addMapping(&BufferBuilder::add_vcard); - propertyMapper.addMapping(&BufferBuilder::add_addressbook); - propertyMapper.addMapping(&BufferBuilder::add_firstname); - propertyMapper.addMapping(&BufferBuilder::add_lastname); + SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Uid, uid); + SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Fn, fn); + SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Emails, emails); + SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Vcard, vcard); + SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Addressbook, addressbook); + SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Firstname, firstname); + SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Lastname, lastname); } void TypeImplementation::configure(IndexPropertyMapper &) @@ -171,16 +134,10 @@ void TypeImplementation::configure(TypeIndex &index) index.addProperty(Addressbook::Name::name); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) +void TypeImplementation::configure(PropertyMapper &propertyMapper) { - propertyMapper.addMapping(&Buffer::parent); - propertyMapper.addMapping(&Buffer::name); -} - -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) -{ - propertyMapper.addMapping(&BufferBuilder::add_parent); - propertyMapper.addMapping(&BufferBuilder::add_name); + SINK_REGISTER_SERIALIZER(propertyMapper, Addressbook, Parent, parent); + SINK_REGISTER_SERIALIZER(propertyMapper, Addressbook, Name, name); } void TypeImplementation::configure(IndexPropertyMapper &) @@ -194,20 +151,12 @@ void TypeImplementation::configure(TypeIndex &index) index.addProperty(Event::Uid::name); } -void TypeImplementation::configure(ReadPropertyMapper &propertyMapper) -{ - propertyMapper.addMapping(&Buffer::summary); - propertyMapper.addMapping(&Buffer::description); - propertyMapper.addMapping(&Buffer::uid); - propertyMapper.addMapping(&Buffer::attachment); -} - -void TypeImplementation::configure(WritePropertyMapper &propertyMapper) +void TypeImplementation::configure(PropertyMapper &propertyMapper) { - propertyMapper.addMapping(&BufferBuilder::add_summary); - propertyMapper.addMapping(&BufferBuilder::add_description); - propertyMapper.addMapping(&BufferBuilder::add_uid); - propertyMapper.addMapping(&BufferBuilder::add_attachment); + SINK_REGISTER_SERIALIZER(propertyMapper, Event, Summary, summary); + SINK_REGISTER_SERIALIZER(propertyMapper, Event, Description, description); + SINK_REGISTER_SERIALIZER(propertyMapper, Event, Uid, uid); + SINK_REGISTER_SERIALIZER(propertyMapper, Event, Attachment, attachment); } void TypeImplementation::configure(IndexPropertyMapper &) diff --git a/common/domain/typeimplementations.h b/common/domain/typeimplementations.h index 8acea58..7e4608c 100644 --- a/common/domain/typeimplementations.h +++ b/common/domain/typeimplementations.h @@ -26,8 +26,7 @@ #include "contact_generated.h" #include "addressbook_generated.h" -class ReadPropertyMapper; -class WritePropertyMapper; +class PropertyMapper; class IndexPropertyMapper; class TypeIndex; @@ -46,8 +45,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Mail Buffer; typedef Sink::ApplicationDomain::Buffer::MailBuilder BufferBuilder; static void configure(TypeIndex &index); - static void configure(ReadPropertyMapper &propertyMapper); - static void configure(WritePropertyMapper &propertyMapper); + static void configure(PropertyMapper &propertyMapper); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -57,8 +55,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Folder Buffer; typedef Sink::ApplicationDomain::Buffer::FolderBuilder BufferBuilder; static void configure(TypeIndex &); - static void configure(ReadPropertyMapper &); - static void configure(WritePropertyMapper &); + static void configure(PropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -68,8 +65,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Contact Buffer; typedef Sink::ApplicationDomain::Buffer::ContactBuilder BufferBuilder; static void configure(TypeIndex &); - static void configure(ReadPropertyMapper &); - static void configure(WritePropertyMapper &); + static void configure(PropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -79,8 +75,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Addressbook Buffer; typedef Sink::ApplicationDomain::Buffer::AddressbookBuilder BufferBuilder; static void configure(TypeIndex &); - static void configure(ReadPropertyMapper &); - static void configure(WritePropertyMapper &); + static void configure(PropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; @@ -90,8 +85,7 @@ public: typedef Sink::ApplicationDomain::Buffer::Event Buffer; typedef Sink::ApplicationDomain::Buffer::EventBuilder BufferBuilder; static void configure(TypeIndex &); - static void configure(ReadPropertyMapper &); - static void configure(WritePropertyMapper &); + static void configure(PropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); }; diff --git a/common/domainadaptor.h b/common/domainadaptor.h index 1db43ca..c38b713 100644 --- a/common/domainadaptor.h +++ b/common/domainadaptor.h @@ -38,7 +38,7 @@ */ template flatbuffers::Offset -createBufferPart(const Sink::ApplicationDomain::ApplicationDomainType &domainObject, flatbuffers::FlatBufferBuilder &fbb, const WritePropertyMapper &mapper) +createBufferPart(const Sink::ApplicationDomain::ApplicationDomainType &domainObject, flatbuffers::FlatBufferBuilder &fbb, const PropertyMapper &mapper) { // First create a primitives such as strings using the mappings QList> propertiesToAddToResource; @@ -66,7 +66,7 @@ createBufferPart(const Sink::ApplicationDomain::ApplicationDomainType &domainObj * After this the buffer can be extracted from the FlatBufferBuilder object. */ template -static void createBufferPartBuffer(const Sink::ApplicationDomain::ApplicationDomainType &domainObject, flatbuffers::FlatBufferBuilder &fbb, WritePropertyMapper &mapper) +static void createBufferPartBuffer(const Sink::ApplicationDomain::ApplicationDomainType &domainObject, flatbuffers::FlatBufferBuilder &fbb, PropertyMapper &mapper) { auto pos = createBufferPart(domainObject, fbb, mapper); // Because we cannot template the following call @@ -151,7 +151,7 @@ public: } void const *mLocalBuffer; - QSharedPointer mLocalMapper; + QSharedPointer mLocalMapper; QSharedPointer mIndexMapper; TypeIndex *mIndex; }; @@ -169,12 +169,10 @@ class SINK_EXPORT DomainTypeAdaptorFactory : public DomainTypeAdaptorFactoryInte public: DomainTypeAdaptorFactory() - : mLocalMapper(QSharedPointer::create()), - mLocalWriteMapper(QSharedPointer::create()), + : mPropertyMapper(QSharedPointer::create()), mIndexMapper(QSharedPointer::create()) { - Sink::ApplicationDomain::TypeImplementation::configure(*mLocalMapper); - Sink::ApplicationDomain::TypeImplementation::configure(*mLocalWriteMapper); + Sink::ApplicationDomain::TypeImplementation::configure(*mPropertyMapper); Sink::ApplicationDomain::TypeImplementation::configure(*mIndexMapper); } @@ -189,7 +187,7 @@ public: { auto adaptor = QSharedPointer::create(); adaptor->mLocalBuffer = Sink::EntityBuffer::readBuffer(entity.local()); - adaptor->mLocalMapper = mLocalMapper; + adaptor->mLocalMapper = mPropertyMapper; adaptor->mIndexMapper = mIndexMapper; adaptor->mIndex = index; return adaptor; @@ -199,11 +197,7 @@ public: createBuffer(const Sink::ApplicationDomain::ApplicationDomainType &domainObject, flatbuffers::FlatBufferBuilder &fbb, void const *metadataData = 0, size_t metadataSize = 0) Q_DECL_OVERRIDE { flatbuffers::FlatBufferBuilder localFbb; - if (mLocalWriteMapper) { - // SinkTrace() << "Creating local buffer part"; - createBufferPartBuffer(domainObject, localFbb, *mLocalWriteMapper); - } - + createBufferPartBuffer(domainObject, localFbb, *mPropertyMapper); Sink::EntityBuffer::assembleEntityBuffer(fbb, metadataData, metadataSize, 0, 0, localFbb.GetBufferPointer(), localFbb.GetSize()); return true; } @@ -219,8 +213,7 @@ public: protected: - QSharedPointer mLocalMapper; - QSharedPointer mLocalWriteMapper; + QSharedPointer mPropertyMapper; QSharedPointer mIndexMapper; }; diff --git a/common/propertymapper.h b/common/propertymapper.h index fa4592b..fd24278 100644 --- a/common/propertymapper.h +++ b/common/propertymapper.h @@ -65,10 +65,17 @@ QVariant SINK_EXPORT propertyToVariant(const flatbuffers::Vector + void addMapping(FunctionReturnValue (Buffer::*f)() const, void (BufferBuilder::*f2)(Arg)) + { + addReadMapping(f); + addWriteMapping(f2); + } virtual QVariant getProperty(const QByteArray &key, void const *buffer) const { @@ -79,6 +86,14 @@ public: return QVariant(); } + virtual void setProperty(const QByteArray &key, const QVariant &value, QList> &builderCalls, flatbuffers::FlatBufferBuilder &fbb) const + { + if (mWriteAccessors.contains(key)) { + auto accessor = mWriteAccessors.value(key); + builderCalls << accessor(value, fbb); + } + } + bool hasMapping(const QByteArray &key) const { return mReadAccessors.contains(key); @@ -89,69 +104,50 @@ public: return mReadAccessors.keys(); } - void addMapping(const QByteArray &property, const std::function &mapping) +private: + void addReadMapping(const QByteArray &property, const std::function &mapping) { mReadAccessors.insert(property, mapping); } template - void addMapping(FunctionReturnValue (Buffer::*f)() const) + void addReadMapping(FunctionReturnValue (Buffer::*f)() const) { - addMapping(T::name, [f](void const *buffer) -> QVariant { return propertyToVariant((static_cast(buffer)->*f)()); }); + addReadMapping(T::name, [f](void const *buffer) -> QVariant { return propertyToVariant((static_cast(buffer)->*f)()); }); } -private: - QHash> mReadAccessors; -}; -class WritePropertyMapper -{ -public: - virtual ~WritePropertyMapper(){}; - - virtual void setProperty(const QByteArray &key, const QVariant &value, QList> &builderCalls, flatbuffers::FlatBufferBuilder &fbb) const - { - if (mWriteAccessors.contains(key)) { - auto accessor = mWriteAccessors.value(key); - builderCalls << accessor(value, fbb); - } - } - - bool hasMapping(const QByteArray &key) const - { - return mWriteAccessors.contains(key); - } - - void addMapping(const QByteArray &property, const std::function(const QVariant &, flatbuffers::FlatBufferBuilder &)> &mapping) + void addWriteMapping(const QByteArray &property, const std::function(const QVariant &, flatbuffers::FlatBufferBuilder &)> &mapping) { mWriteAccessors.insert(property, mapping); } template - void addMapping(void (BufferBuilder::*f)(uint8_t)) + void addWriteMapping(void (BufferBuilder::*f)(uint8_t)) { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { + addWriteMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { return [value, f](void *builder) { (static_cast(builder)->*f)(value.value()); }; }); } template - void addMapping(void (BufferBuilder::*f)(bool)) + void addWriteMapping(void (BufferBuilder::*f)(bool)) { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { + addWriteMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { return [value, f](void *builder) { (static_cast(builder)->*f)(value.value()); }; }); } template - void addMapping(void (BufferBuilder::*f)(flatbuffers::Offset)) + void addWriteMapping(void (BufferBuilder::*f)(flatbuffers::Offset)) { - addMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { + addWriteMapping(T::name, [f](const QVariant &value, flatbuffers::FlatBufferBuilder &fbb) -> std::function { auto offset = variantToProperty(value, fbb); return [offset, f](void *builder) { (static_cast(builder)->*f)(offset); }; }); } -private: + QHash> mReadAccessors; QHash(const QVariant &, flatbuffers::FlatBufferBuilder &)>> mWriteAccessors; }; + diff --git a/tests/domainadaptortest.cpp b/tests/domainadaptortest.cpp index 2b7455a..df44819 100644 --- a/tests/domainadaptortest.cpp +++ b/tests/domainadaptortest.cpp @@ -47,7 +47,7 @@ private slots: void testCreateBufferPart() { - auto writeMapper = QSharedPointer::create(); + auto writeMapper = QSharedPointer::create(); Sink::ApplicationDomain::TypeImplementation::configure(*writeMapper); Sink::ApplicationDomain::Event event; @@ -100,7 +100,7 @@ private slots: void testMail() { - auto writeMapper = QSharedPointer::create(); + auto writeMapper = QSharedPointer::create(); Sink::ApplicationDomain::TypeImplementation::configure(*writeMapper); Sink::ApplicationDomain::Mail mail; -- cgit v1.2.3 From 9228b3ba170a0f68dbb432b2455c75d5fff21506 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 3 May 2017 21:29:28 +0200 Subject: Sanity check db names lmdb and sink deal badly with e.g. a string containing a null in the millde as db name. Thus we try to protect better against it. This is an actual problem we triggered: https://phabricator.kde.org/T5880 --- common/storage/entitystore.cpp | 5 +++++ common/storage_lmdb.cpp | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index b7309ab..4cb4641 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -320,6 +320,11 @@ void EntityStore::cleanupEntityRevisionsUntil(qint64 revision) { const auto uid = DataStore::getUidFromRevision(d->transaction, revision); const auto bufferType = DataStore::getTypeFromRevision(d->transaction, revision); + if (bufferType.isEmpty() || uid.isEmpty()) { + SinkErrorCtx(d->logCtx) << "Failed to find revision during cleanup: " << revision; + Q_ASSERT(false); + return; + } SinkTraceCtx(d->logCtx) << "Cleaning up revision " << revision << uid << bufferType; DataStore::mainDatabase(d->transaction, bufferType) .scan(uid, diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index 08eea37..18364ea 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -169,6 +169,26 @@ public: if (const int rc = mdb_dbi_open(transaction, db.constData(), flags, &dbi)) { //Create the db if it is not existing already if (rc == MDB_NOTFOUND && !readOnly) { + //Sanity check db name + { + auto parts = db.split('.'); + for (const auto &p : parts) { + auto containsSpecialCharacter = [] (const QByteArray &p) { + for (int i = 0; i < p.size(); i++) { + const auto c = p.at(i); + //Between 0 and z in the ascii table. Essentially ensures that the name is printable and doesn't contain special chars + if (c < 0x30 || c > 0x7A) { + return true; + } + } + return false; + }; + if (p.isEmpty() || containsSpecialCharacter(p)) { + SinkError() << "Tried to create a db with an invalid name. Hex:" << db.toHex() << " ASCII:" << db; + Q_ASSERT(false); + } + } + } if (const int rc = mdb_dbi_open(transaction, db.constData(), flags | MDB_CREATE, &dbi)) { SinkWarning() << "Failed to create db " << QByteArray(mdb_strerror(rc)); Error error(name.toLatin1(), ErrorCodes::GenericError, "Error while creating database: " + QByteArray(mdb_strerror(rc))); -- cgit v1.2.3 From 00efa772f1b39db010e480bddb08c2e086d7b364 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 3 May 2017 21:31:53 +0200 Subject: Dont pass around buffers to potentially invalid memory. Once the transaction is done or some modification is executed that memory is no longer valid. So we always copy. --- common/storage_common.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/storage_common.cpp b/common/storage_common.cpp index 81a38c7..f648c94 100644 --- a/common/storage_common.cpp +++ b/common/storage_common.cpp @@ -113,7 +113,7 @@ QByteArray DataStore::getUidFromRevision(const DataStore::Transaction &transacti transaction.openDatabase("revisions") .scan(QByteArray::number(revision), [&](const QByteArray &, const QByteArray &value) -> bool { - uid = value; + uid = QByteArray{value.constData(), value.size()}; return false; }, [revision](const Error &error) { SinkWarning() << "Couldn't find uid for revision: " << revision << error.message; }); @@ -126,7 +126,7 @@ QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transact transaction.openDatabase("revisionType") .scan(QByteArray::number(revision), [&](const QByteArray &, const QByteArray &value) -> bool { - type = value; + type = QByteArray{value.constData(), value.size()}; return false; }, [revision](const Error &error) { SinkWarning() << "Couldn't find type for revision " << revision; }); -- cgit v1.2.3 From 544e08ab379fd53d00009a3400aa530520b65e85 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 3 May 2017 21:33:04 +0200 Subject: Catch special error case of empty db name --- common/storage_common.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/storage_common.cpp b/common/storage_common.cpp index f648c94..dfcfc2a 100644 --- a/common/storage_common.cpp +++ b/common/storage_common.cpp @@ -207,6 +207,11 @@ QByteArray DataStore::generateUid() DataStore::NamedDatabase DataStore::mainDatabase(const DataStore::Transaction &t, const QByteArray &type) { + if (type.isEmpty()) { + SinkError() << "Tried to open main database for empty type."; + Q_ASSERT(false); + return {}; + } return t.openDatabase(type + ".main"); } -- cgit v1.2.3 From 9487403a87a93a738955bb9cafd0d12efa295896 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 4 May 2017 07:53:18 +0200 Subject: Cleanup --- common/storage.h | 1 - common/storage/entitystore.cpp | 24 ------------------------ common/storage_lmdb.cpp | 5 ----- 3 files changed, 30 deletions(-) diff --git a/common/storage.h b/common/storage.h index 71e9401..d1dda3e 100644 --- a/common/storage.h +++ b/common/storage.h @@ -132,7 +132,6 @@ public: void abort(); QList getDatabaseNames() const; - bool validateNamedDatabases(); NamedDatabase openDatabase(const QByteArray &name = {"default"}, const std::function &errorHandler = {}, bool allowDuplicates = false) const; diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 4cb4641..04760f7 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -112,7 +112,6 @@ void EntityStore::startTransaction(Sink::Storage::DataStore::AccessMode accessMo Q_ASSERT(!d->transaction); Sink::Storage::DataStore store(Sink::storageLocation(), d->resourceContext.instanceId(), accessMode); d->transaction = store.createTransaction(accessMode); - Q_ASSERT(d->transaction.validateNamedDatabases()); } void EntityStore::commitTransaction() @@ -619,29 +618,6 @@ qint64 EntityStore::maxRevision() return DataStore::maxRevision(d->getTransaction()); } -/* DataStore::Transaction getTransaction() */ -/* { */ -/* Sink::Storage::DataStore::Transaction transaction; */ -/* { */ -/* Sink::Storage::DataStore storage(Sink::storageLocation(), mResourceInstanceIdentifier); */ -/* if (!storage.exists()) { */ -/* //This is not an error if the resource wasn't started before */ -/* SinkLogCtx(d->logCtx) << "Store doesn't exist: " << mResourceInstanceIdentifier; */ -/* return Sink::Storage::DataStore::Transaction(); */ -/* } */ -/* storage.setDefaultErrorHandler([this](const Sink::Storage::DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during query: " << error.store << error.message; }); */ -/* transaction = storage.createTransaction(Sink::Storage::DataStore::ReadOnly); */ -/* } */ - -/* //FIXME this is a temporary measure to recover from a failure to open the named databases correctly. */ -/* //Once the actual problem is fixed it will be enough to simply crash if we open the wrong database (which we check in openDatabase already). */ -/* while (!transaction.validateNamedDatabases()) { */ -/* Sink::Storage::DataStore storage(Sink::storageLocation(), mResourceInstanceIdentifier); */ -/* transaction = storage.createTransaction(Sink::Storage::DataStore::ReadOnly); */ -/* } */ -/* return transaction; */ -/* } */ - Sink::Log::Context EntityStore::logContext() const { return d->logCtx; diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index 18364ea..b5698b1 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -659,11 +659,6 @@ static bool ensureCorrectDb(DataStore::NamedDatabase &database, const QByteArray return !openedTheWrongDatabase; } -bool DataStore::Transaction::validateNamedDatabases() -{ - return true; -} - DataStore::NamedDatabase DataStore::Transaction::openDatabase(const QByteArray &db, const std::function &errorHandler, bool allowDuplicates) const { if (!d) { -- cgit v1.2.3 From a2ecf2a77a38025442da8150156e5b59f107b897 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 4 May 2017 07:53:36 +0200 Subject: Only sync subscribed folders --- examples/imapresource/imapresource.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 8577b8c..8e6d2b1 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -584,6 +584,10 @@ public: //Synchronize folders return KAsync::value(folders) .serialEach([=](const Folder &folder) { + //Skip unsubscribed folders + if (!folder.subscribed) { + return KAsync::null(); + } SinkLog() << "Syncing folder " << folder.path(); //Emit notification that the folder is being synced. //The synchronizer can't do that because it has no concept of the folder filter on a mail sync scope meaning that the folder is being synchronized. -- cgit v1.2.3 From feef3bd5c1562a52c274fa07af51c716e5362054 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 4 May 2017 07:56:51 +0200 Subject: Take sink down if we're not asserting --- common/storage_lmdb.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index b5698b1..e3685a5 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -186,6 +186,7 @@ public: if (p.isEmpty() || containsSpecialCharacter(p)) { SinkError() << "Tried to create a db with an invalid name. Hex:" << db.toHex() << " ASCII:" << db; Q_ASSERT(false); + throw std::runtime_error("Fatal error while creating db."); } } } -- cgit v1.2.3 From 6adf9a4734f15a2c0fa199897f76ded4659b83b7 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 4 May 2017 11:40:24 +0200 Subject: Added progress notification --- common/commands/notification.fbs | 2 ++ common/listener.cpp | 2 ++ common/notification.h | 2 ++ common/resourceaccess.cpp | 2 ++ common/synchronizer.cpp | 18 ++++++++++++++++-- common/synchronizer.h | 10 ++++++++++ examples/imapresource/imapresource.cpp | 4 ++-- 7 files changed, 36 insertions(+), 4 deletions(-) diff --git a/common/commands/notification.fbs b/common/commands/notification.fbs index 517111c..7ced666 100644 --- a/common/commands/notification.fbs +++ b/common/commands/notification.fbs @@ -5,6 +5,8 @@ table Notification { identifier: string; //An identifier that links back to the something related to the notification (e.g. a command id) message: string; code: int = 0; //See notification.h + progress: int = 0; //See notification.h + total: int = 0; //See notification.h entities: [string]; //A list of entities this applies to } diff --git a/common/listener.cpp b/common/listener.cpp index 983e438..ec2bedb 100644 --- a/common/listener.cpp +++ b/common/listener.cpp @@ -420,6 +420,8 @@ void Listener::notify(const Sink::Notification ¬ification) builder.add_identifier(idString); builder.add_message(messageString); builder.add_entities(entities); + builder.add_progress(notification.progress); + builder.add_total(notification.total); auto command = builder.Finish(); Sink::Commands::FinishNotificationBuffer(m_fbb, command); for (Client &client : m_connections) { diff --git a/common/notification.h b/common/notification.h index f5379fd..30e240c 100644 --- a/common/notification.h +++ b/common/notification.h @@ -56,6 +56,8 @@ public: QString message; //A return code. Zero typically indicates success. int code = 0; + int progress = 0; + int total = 0; QByteArray resource; }; } diff --git a/common/resourceaccess.cpp b/common/resourceaccess.cpp index ad8cae9..cf8b2e0 100644 --- a/common/resourceaccess.cpp +++ b/common/resourceaccess.cpp @@ -547,6 +547,8 @@ static Sink::Notification getNotification(const Sink::Commands::Notification *bu } n.type = buffer->type(); n.code = buffer->code(); + n.progress = buffer->progress(); + n.total = buffer->total(); n.entities = BufferUtils::fromVector(*buffer->entities()); return n; } diff --git a/common/synchronizer.cpp b/common/synchronizer.cpp index 3e7bd30..58e5e7a 100644 --- a/common/synchronizer.cpp +++ b/common/synchronizer.cpp @@ -304,9 +304,21 @@ void Synchronizer::emitNotification(Notification::NoticationType type, int code, emit notify(n); } +void Synchronizer::emitProgressNotification(Notification::NoticationType type, int progress, int total, const QByteArray &id, const QByteArrayList &entities) +{ + Sink::Notification n; + n.id = id; + n.type = type; + n.progress = progress; + n.total = total; + n.entities = entities; + emit notify(n); +} + void Synchronizer::reportProgress(int progress, int total) { SinkLogCtx(mLogCtx) << "Progress: " << progress << " out of " << total; + emitProgressNotification(Notification::Progress, progress, total, mCurrentRequest.requestId, mCurrentRequest.applicableEntities); } void Synchronizer::setStatusFromResult(const KAsync::Error &error, const QString &s, const QByteArray &requestId) @@ -465,13 +477,15 @@ KAsync::Job Synchronizer::processSyncQueue() if (request.requestType == Synchronizer::SyncRequest::Synchronization) { setBusy(true, "Synchronization has started.", request.requestId); } else if (request.requestType == Synchronizer::SyncRequest::ChangeReplay) { - setBusy(true, "ChangeReplay has started.", "changereplay"); + setBusy(true, "ChangeReplay has started.", request.requestId); } + mCurrentRequest = request; }) .then(processRequest(request)) .then([this, request](const KAsync::Error &error) { SinkTraceCtx(mLogCtx) << "Sync request processed"; setBusy(false, {}, request.requestId); + mCurrentRequest = {}; mEntityStore->abortTransaction(); mSyncTransaction.abort(); mMessageQueue->commit(); @@ -516,7 +530,7 @@ void Synchronizer::revisionChanged() return; } } - mSyncRequestQueue << Synchronizer::SyncRequest{Synchronizer::SyncRequest::ChangeReplay}; + mSyncRequestQueue << Synchronizer::SyncRequest{Synchronizer::SyncRequest::ChangeReplay, "changereplay"}; processSyncQueue().exec(); } diff --git a/common/synchronizer.h b/common/synchronizer.h index b1ee122..bb24c2b 100644 --- a/common/synchronizer.h +++ b/common/synchronizer.h @@ -131,6 +131,8 @@ protected: RequestFlush }; + SyncRequest() = default; + SyncRequest(const Sink::QueryBase &q, const QByteArray &requestId_ = QByteArray(), RequestOptions o = NoOptions) : requestId(requestId_), requestType(Synchronization), @@ -145,6 +147,12 @@ protected: { } + SyncRequest(RequestType type, const QByteArray &requestId_) + : requestId(requestId_), + requestType(type) + { + } + SyncRequest(RequestType type, int flushType_, const QByteArray &requestId_) : flushType(flushType_), requestId(requestId_), @@ -184,6 +192,7 @@ protected: virtual void mergeIntoQueue(const Synchronizer::SyncRequest &request, QList &queue); void emitNotification(Notification::NoticationType type, int code, const QString &message, const QByteArray &id = QByteArray{}, const QByteArrayList &entiteis = QByteArrayList{}); + void emitProgressNotification(Notification::NoticationType type, int progress, int total, const QByteArray &id, const QByteArrayList &entities); /** * Report progress for current task @@ -211,6 +220,7 @@ private: Sink::Storage::DataStore::Transaction mSyncTransaction; std::function mEnqueue; QList mSyncRequestQueue; + SyncRequest mCurrentRequest; MessageQueue *mMessageQueue; bool mSyncInProgress; QMultiHash mPendingSyncRequests; diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 8e6d2b1..5aa18dd 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -324,7 +324,7 @@ public: synchronizeMails(folderRemoteId, m); }, [this, maxUid, folder](int progress, int total) { - SinkLog() << "Progress: " << progress << " out of " << total; + reportProgress(progress, total); //commit every 10 messages if ((progress % 10) == 0) { commit(); @@ -365,7 +365,7 @@ public: synchronizeMails(folderRemoteId, m); }, [=](int progress, int total) { - SinkLogCtx(mLogCtx) << "Progress: " << progress << " out of " << total; + reportProgress(progress, total); //commit every 100 messages if ((progress % 100) == 0) { commit(); -- cgit v1.2.3 From bb3a79c8b71d6d4e2a4269bdcffb616b2db9d619 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 4 May 2017 15:38:27 +0200 Subject: Limit the buffering on write. Otherwise the system becomes rather unresponsive under load. --- common/commands.cpp | 22 ++++++++++++---------- common/commands.h | 8 ++++---- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/common/commands.cpp b/common/commands.cpp index eeb7f08..24f2017 100644 --- a/common/commands.cpp +++ b/common/commands.cpp @@ -1,5 +1,6 @@ /* * Copyright (C) 2014 Aaron Seigo + * Copyright (C) 2016 Christian Mollekopf * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -20,7 +21,7 @@ #include "commands.h" -#include +#include #include namespace Sink { @@ -73,19 +74,19 @@ int headerSize() return sizeof(int) + (sizeof(uint) * 2); } -void write(QIODevice *device, int messageId, int commandId) +void write(QLocalSocket *device, int messageId, int commandId) { write(device, messageId, commandId, 0, 0); } -static void write(QIODevice *device, const char *buffer, uint size) +static void write(QLocalSocket *device, const char *buffer, uint size) { if (device->write(buffer, size) < 0) { SinkWarningCtx(Sink::Log::Context{"commands"}) << "Error while writing " << device->errorString(); } } -void write(QIODevice *device, int messageId, int commandId, const char *buffer, uint size) +void write(QLocalSocket *device, int messageId, int commandId, const char *buffer, uint size) { if (size > 0 && !buffer) { size = 0; @@ -97,15 +98,16 @@ void write(QIODevice *device, int messageId, int commandId, const char *buffer, if (buffer) { write(device, buffer, size); } + //The default implementation will happily buffer 200k bytes before sending it out which doesn't make the sytem exactly responsive. + //1k is arbitrary, but fits a bunch of messages at least. + if (device->bytesToWrite() > 1000) { + device->flush(); + } } -void write(QIODevice *device, int messageId, int commandId, flatbuffers::FlatBufferBuilder &fbb) +void write(QLocalSocket *device, int messageId, int commandId, flatbuffers::FlatBufferBuilder &fbb) { - const int dataSize = fbb.GetSize(); - write(device, (const char *)&messageId, sizeof(int)); - write(device, (const char *)&commandId, sizeof(int)); - write(device, (const char *)&dataSize, sizeof(int)); - write(device, (const char *)fbb.GetBufferPointer(), dataSize); + write(device, messageId, commandId, (const char *)fbb.GetBufferPointer(), fbb.GetSize()); } } // namespace Commands diff --git a/common/commands.h b/common/commands.h index 6d5d39b..1548eac 100644 --- a/common/commands.h +++ b/common/commands.h @@ -24,7 +24,7 @@ #include #include -class QIODevice; +class QLocalSocket; namespace Sink { @@ -55,9 +55,9 @@ enum CommandIds QByteArray name(int commandId); int SINK_EXPORT headerSize(); -void SINK_EXPORT write(QIODevice *device, int messageId, int commandId); -void SINK_EXPORT write(QIODevice *device, int messageId, int commandId, const char *buffer, uint size); -void SINK_EXPORT write(QIODevice *device, int messageId, int commandId, flatbuffers::FlatBufferBuilder &fbb); +void SINK_EXPORT write(QLocalSocket *device, int messageId, int commandId); +void SINK_EXPORT write(QLocalSocket *device, int messageId, int commandId, const char *buffer, uint size); +void SINK_EXPORT write(QLocalSocket *device, int messageId, int commandId, flatbuffers::FlatBufferBuilder &fbb); } } // namespace Sink -- cgit v1.2.3 From 447383e860d523ca3aab7da266622780f644de6c Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 4 May 2017 16:38:12 +0200 Subject: Avoid unnecessary noise Such as progress 0 out of 0 (happens on sync of already synced folder) --- common/synchronizer.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/synchronizer.cpp b/common/synchronizer.cpp index 58e5e7a..175ed83 100644 --- a/common/synchronizer.cpp +++ b/common/synchronizer.cpp @@ -317,8 +317,10 @@ void Synchronizer::emitProgressNotification(Notification::NoticationType type, i void Synchronizer::reportProgress(int progress, int total) { - SinkLogCtx(mLogCtx) << "Progress: " << progress << " out of " << total; - emitProgressNotification(Notification::Progress, progress, total, mCurrentRequest.requestId, mCurrentRequest.applicableEntities); + if (progress > 0 && total > 0) { + SinkLogCtx(mLogCtx) << "Progress: " << progress << " out of " << total; + emitProgressNotification(Notification::Progress, progress, total, mCurrentRequest.requestId, mCurrentRequest.applicableEntities); + } } void Synchronizer::setStatusFromResult(const KAsync::Error &error, const QString &s, const QByteArray &requestId) -- cgit v1.2.3 From 4bc39b1b771aaf9ae5f7c96aeaddb06f2bd101ac Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 4 May 2017 17:33:32 +0200 Subject: Fixed folder sync When explicitly listing the folder we can't rely on the subscription state, nor should we. --- examples/imapresource/imapresource.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 5aa18dd..e7458f6 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -492,7 +492,7 @@ public: //Otherwise fetch full payload for daterange auto folderList = QSharedPointer>::create(); return imap->fetchFolders([folderList](const Folder &folder) { - if (!folder.noselect) { + if (!folder.noselect && folder.subscribed) { *folderList << folder; } }) @@ -580,14 +580,7 @@ public: bool syncHeaders = query.hasFilter(); //FIXME If we were able to to flush in between we could just query the local store for the folder list. return getFolderList(imap, query) - .then([=] (const QVector &folders) { - //Synchronize folders - return KAsync::value(folders) .serialEach([=](const Folder &folder) { - //Skip unsubscribed folders - if (!folder.subscribed) { - return KAsync::null(); - } SinkLog() << "Syncing folder " << folder.path(); //Emit notification that the folder is being synced. //The synchronizer can't do that because it has no concept of the folder filter on a mail sync scope meaning that the folder is being synchronized. @@ -602,7 +595,6 @@ public: SinkWarning() << "Failed to sync folder: " << folder.path() << "Error: " << error.errorMessage; }); }); - }); } }) .then([=] (const KAsync::Error &error) { -- cgit v1.2.3 From 8eab2b67fdf83c657f996debfc238703a78b337b Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 5 May 2017 10:42:28 +0200 Subject: Don't leak transactions when copying them. Previsouly we would hit the maxreaders limit --- common/storage_lmdb.cpp | 1 + tests/storagetest.cpp | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index e3685a5..cb7bca4 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -563,6 +563,7 @@ DataStore::Transaction::Transaction(Transaction &&other) : d(nullptr) DataStore::Transaction &DataStore::Transaction::operator=(DataStore::Transaction &&other) { if (&other != this) { + abort(); delete d; d = other.d; other.d = nullptr; diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp index 7202628..dfb91ec 100644 --- a/tests/storagetest.cpp +++ b/tests/storagetest.cpp @@ -487,6 +487,24 @@ private slots: } } + void testCopyTransaction() + { + Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); + { + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + transaction.openDatabase("a", nullptr, false); + transaction.openDatabase("b", nullptr, false); + transaction.openDatabase("c", nullptr, false); + transaction.commit(); + } + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); + for (int i = 0; i < 1000; i++) { + transaction.openDatabase("a", nullptr, false); + transaction.openDatabase("b", nullptr, false); + transaction.openDatabase("c", nullptr, false); + transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); + } + } }; QTEST_MAIN(StorageTest) -- cgit v1.2.3 From f52ed4fd64994985f1061c5fcd20dccaa61fbc67 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 6 May 2017 12:37:30 +0200 Subject: A defined table layout --- common/storage.h | 7 +++++++ common/storage/entitystore.cpp | 33 ++++++++++++++++++++++++++++++--- common/storage_lmdb.cpp | 21 ++++++++++++++++----- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/common/storage.h b/common/storage.h index d1dda3e..42cdcac 100644 --- a/common/storage.h +++ b/common/storage.h @@ -25,10 +25,16 @@ #include #include #include +#include namespace Sink { namespace Storage { +struct DbLayout { + QByteArray name; + QMap tables; +}; + class SINK_EXPORT DataStore { public: @@ -151,6 +157,7 @@ public: }; DataStore(const QString &storageRoot, const QString &name, AccessMode mode = ReadOnly); + DataStore(const QString &storageRoot, const DbLayout &layout, AccessMode mode = ReadOnly); ~DataStore(); Transaction createTransaction(AccessMode mode = ReadWrite, const std::function &errorHandler = std::function()); diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 04760f7..9539bec 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -36,6 +36,33 @@ using namespace Sink; using namespace Sink::Storage; +static Sink::Storage::DbLayout dbLayout(const QByteArray &instanceId) +{ + return Sink::Storage::DbLayout { + instanceId, + { + {"folder.main", 0}, + {"folder.index.name", 1}, + {"folder.index.parent", 1}, + {"mail.main", 0}, + {"mail.index.date", 1}, + {"mail.index.folder", 1}, + {"mail.index.folder.sort.date", 0}, + {"mail.index.messageId", 1}, + {"mail.index.messageIdthreadId", 1}, + {"mail.index.parentMessageId", 1}, + {"mail.index.subjectthreadId", 1}, + {"mail.index.threadIdmessageId", 1}, + {"revisionType", 0}, + {"revisions", 0}, + {"uids", 0}, + {"default", 0}, + {"__flagtable", 0} + } + }; +} + + class EntityStore::Private { public: Private(const ResourceContext &context, const Sink::Log::Context &ctx) : resourceContext(context), logCtx(ctx.subContext("entitystore")) {} @@ -47,7 +74,7 @@ public: bool exists() { - return Sink::Storage::DataStore(Sink::storageLocation(), resourceContext.instanceId(), DataStore::ReadOnly).exists(); + return Sink::Storage::DataStore(Sink::storageLocation(), dbLayout(resourceContext.instanceId()), DataStore::ReadOnly).exists(); } DataStore::Transaction &getTransaction() @@ -56,7 +83,7 @@ public: return transaction; } - Sink::Storage::DataStore store(Sink::storageLocation(), resourceContext.instanceId(), DataStore::ReadOnly); + Sink::Storage::DataStore store(Sink::storageLocation(), dbLayout(resourceContext.instanceId()), DataStore::ReadOnly); transaction = store.createTransaction(DataStore::ReadOnly); return transaction; } @@ -110,7 +137,7 @@ void EntityStore::startTransaction(Sink::Storage::DataStore::AccessMode accessMo { SinkTraceCtx(d->logCtx) << "Starting transaction: " << accessMode; Q_ASSERT(!d->transaction); - Sink::Storage::DataStore store(Sink::storageLocation(), d->resourceContext.instanceId(), accessMode); + Sink::Storage::DataStore store(Sink::storageLocation(), dbLayout(d->resourceContext.instanceId()), accessMode); d->transaction = store.createTransaction(accessMode); } diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index cb7bca4..ef4bd17 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -711,7 +711,7 @@ QList DataStore::Transaction::getDatabaseNames() const class DataStore::Private { public: - Private(const QString &s, const QString &n, AccessMode m); + Private(const QString &s, const QString &n, AccessMode m, const DbLayout &layout = {}); ~Private(); QString storageRoot; @@ -721,7 +721,7 @@ public: AccessMode mode; }; -DataStore::Private::Private(const QString &s, const QString &n, AccessMode m) : storageRoot(s), name(n), env(0), mode(m) +DataStore::Private::Private(const QString &s, const QString &n, AccessMode m, const DbLayout &layout) : storageRoot(s), name(n), env(0), mode(m) { const QString fullPath(storageRoot + '/' + name); QFileInfo dirInfo(fullPath); @@ -772,9 +772,16 @@ DataStore::Private::Private(const QString &s, const QString &n, AccessMode m) : bool noLock = true; bool requestedRead = m == ReadOnly; auto t = Transaction(new Transaction::Private(requestedRead, nullptr, name, env, noLock)); - for (const auto &db : t.getDatabaseNames()) { - //Get dbi to store for future use. - t.openDatabase(db); + if (!layout.tables.isEmpty()) { + for (auto it = layout.tables.constBegin(); it != layout.tables.constEnd(); it++) { + bool allowDuplicates = it.value(); + t.openDatabase(it.key(), {}, allowDuplicates); + } + } else { + for (const auto &db : t.getDatabaseNames()) { + //Get dbi to store for future use. + t.openDatabase(db); + } } //To persist the dbis (this is also necessary for read-only transactions) t.commit(); @@ -794,6 +801,10 @@ DataStore::DataStore(const QString &storageRoot, const QString &name, AccessMode { } +DataStore::DataStore(const QString &storageRoot, const DbLayout &dbLayout, AccessMode mode) : d(new Private(storageRoot, dbLayout.name, mode, dbLayout)) +{ +} + DataStore::~DataStore() { delete d; -- cgit v1.2.3 From 3657dcd309f30b704801dcaf3e43b71ef703c0de Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sun, 7 May 2017 11:27:40 +0200 Subject: Somewhat useless stress test... --- tests/CMakeLists.txt | 3 +++ tests/dbwriter.cpp | 45 +++++++++++++++++++++++++++++++++++++++++++++ tests/storagetest.cpp | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 tests/dbwriter.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c77a736..2b3e7b1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,9 @@ add_library(sink_test SHARED testimplementations.cpp getrssusage.cpp mailtest.cp qt5_use_modules(sink_test Core Test Concurrent) target_link_libraries(sink_test sink libhawd KF5::Mime) +add_executable(dbwriter dbwriter.cpp) +target_link_libraries(dbwriter sink) + include(SinkTest) manual_tests ( diff --git a/tests/dbwriter.cpp b/tests/dbwriter.cpp new file mode 100644 index 0000000..902a607 --- /dev/null +++ b/tests/dbwriter.cpp @@ -0,0 +1,45 @@ +#include +#include +#include + +int main(int argc, char *argv[]) +{ + + QByteArrayList arguments; + for (int i = 0; i < argc; i++) { + arguments << argv[i]; + } + auto testDataPath = arguments.value(1); + auto dbName = arguments.value(2); + auto count = arguments.value(3).toInt(); + + if (Sink::Storage::DataStore(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly).exists()) { + Sink::Storage::DataStore(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite).removeFromDisk(); + } + + qWarning() << "Creating db: " << testDataPath << dbName << count; + Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + for (int i = 0; i < count; i++) { + if (!transaction) { + qWarning() << "No valid transaction"; + return -1; + } + transaction.openDatabase("a", nullptr, false).write(QByteArray::number(i), "a"); + transaction.openDatabase("b", nullptr, false).write(QByteArray::number(i), "b"); + transaction.openDatabase("c", nullptr, false).write(QByteArray::number(i), "c"); + transaction.openDatabase("p", nullptr, false).write(QByteArray::number(i), "c"); + transaction.openDatabase("q", nullptr, false).write(QByteArray::number(i), "c"); + if (i > (count/2)) { + for (int d = 0; d < 40; d++) { + transaction.openDatabase("db" + QByteArray::number(d), nullptr, false).write(QByteArray::number(i), "a"); + } + } + if ((i % 1000) == 0) { + transaction.commit(); + transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + } + } + qWarning() << "Creating db done."; + return 0; +} diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp index dfb91ec..9e9bad9 100644 --- a/tests/storagetest.cpp +++ b/tests/storagetest.cpp @@ -505,6 +505,43 @@ private slots: transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); } } + + /* + * This test is meant to find problems with the multi-process architecture and initial database creation. + * If we create named databases dynamically (not all up front), it is possilbe that we violate the rule + * that mdb_open_dbi may only be used by a single thread at a time. + * This test is meant to stress that condition. + * + * However, it yields absolutely nothing. + */ + void testReadDuringExternalProcessWrite() + { + QSKIP("Not running multiprocess test"); + + QList> futures; + for (int i = 0; i < 5; i++) { + futures << QtConcurrent::run([&]() { + QTRY_VERIFY(Sink::Storage::DataStore(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly).exists()); + Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly); + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); + for (int i = 0; i < 100000; i++) { + transaction.openDatabase("a", nullptr, false); + transaction.openDatabase("b", nullptr, false); + transaction.openDatabase("c", nullptr, false); + transaction.openDatabase("p", nullptr, false); + transaction.openDatabase("q", nullptr, false); + } + }); + } + + //Start writing to the db from a separate process + QVERIFY(QProcess::startDetached(QCoreApplication::applicationDirPath() + "/dbwriter", QStringList() << testDataPath << dbName << QString::number(100000))); + + for (auto future : futures) { + future.waitForFinished(); + } + + } }; QTEST_MAIN(StorageTest) -- cgit v1.2.3 From 778b01181604dc2eae2013f2dc37db6d647b526a Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 8 May 2017 14:27:25 +0200 Subject: Gather required databases from index definitions. --- common/domain/typeimplementations.cpp | 78 +++++++++++++---- common/domain/typeimplementations.h | 5 ++ common/domain/typeimplementations_p.h | 154 ++++++++++++++++++++++++++++++++++ common/mail/threadindexer.cpp | 7 ++ common/mail/threadindexer.h | 1 + common/storage/entitystore.cpp | 73 +++++++++++----- 6 files changed, 278 insertions(+), 40 deletions(-) create mode 100644 common/domain/typeimplementations_p.h diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp index a87e73d..4e61bad 100644 --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -28,6 +28,7 @@ #include "entity_generated.h" #include "mail/threadindexer.h" #include "domainadaptor.h" +#include "typeimplementations_p.h" using namespace Sink; using namespace Sink::ApplicationDomain; @@ -35,22 +36,43 @@ using namespace Sink::ApplicationDomain; #define SINK_REGISTER_SERIALIZER(MAPPER, ENTITYTYPE, PROPERTY, LOWERCASEPROPERTY) \ MAPPER.addMapping(&Sink::ApplicationDomain::Buffer::ENTITYTYPE::LOWERCASEPROPERTY, &Sink::ApplicationDomain::Buffer::ENTITYTYPE##Builder::add_##LOWERCASEPROPERTY); +typedef IndexConfig, + ValueIndex, + ValueIndex, + ValueIndex, + SortedIndex, + SecondaryIndex, + SecondaryIndex, + CustomSecondaryIndex + > MailIndexConfig; + +typedef IndexConfig, + ValueIndex + > FolderIndexConfig; + +typedef IndexConfig + > ContactIndexConfig; + +typedef IndexConfig + > AddressbookIndexConfig; + +typedef IndexConfig + > EventIndexConfig; + void TypeImplementation::configure(TypeIndex &index) { - // index.addProperty(); - /* index.addProperty(Mail::SenderName::name); */ - /* index->addProperty(Mail::Subject::name); */ - /* index->addFulltextProperty(Mail::Subject::name); */ - index.addProperty(); - index.addProperty(); - index.addPropertyWithSorting(); - index.addProperty(); - index.addProperty(); + MailIndexConfig::configure(index); +} - index.addSecondaryPropertyIndexer(); - index.addSecondaryProperty(); - index.addSecondaryProperty(); +QMap TypeImplementation::typeDatabases() +{ + return merge(QMap{{QByteArray{Mail::name} + ".main", 0}}, MailIndexConfig::databases()); } void TypeImplementation::configure(IndexPropertyMapper &indexPropertyMapper) @@ -85,10 +107,15 @@ void TypeImplementation::configure(PropertyMapper &propertyMapper) SINK_REGISTER_SERIALIZER(propertyMapper, Mail, ParentMessageId, parentMessageId); } + void TypeImplementation::configure(TypeIndex &index) { - index.addProperty(Folder::Parent::name); - index.addProperty(Folder::Name::name); + FolderIndexConfig::configure(index); +} + +QMap TypeImplementation::typeDatabases() +{ + return merge(QMap{{QByteArray{Folder::name} + ".main", 0}}, FolderIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) @@ -108,7 +135,12 @@ void TypeImplementation::configure(IndexPropertyMapper &) void TypeImplementation::configure(TypeIndex &index) { - index.addProperty(Contact::Uid::name); + ContactIndexConfig::configure(index); +} + +QMap TypeImplementation::typeDatabases() +{ + return merge(QMap{{QByteArray{Contact::name} + ".main", 0}}, ContactIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) @@ -130,8 +162,12 @@ void TypeImplementation::configure(IndexPropertyMapper &) void TypeImplementation::configure(TypeIndex &index) { - index.addProperty(Addressbook::Parent::name); - index.addProperty(Addressbook::Name::name); + AddressbookIndexConfig::configure(index); +} + +QMap TypeImplementation::typeDatabases() +{ + return merge(QMap{{QByteArray{Addressbook::name} + ".main", 0}}, AddressbookIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) @@ -148,7 +184,12 @@ void TypeImplementation::configure(IndexPropertyMapper &) void TypeImplementation::configure(TypeIndex &index) { - index.addProperty(Event::Uid::name); + EventIndexConfig::configure(index); +} + +QMap TypeImplementation::typeDatabases() +{ + return merge(QMap{{QByteArray{Event::name} + ".main", 0}}, EventIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) @@ -163,3 +204,4 @@ void TypeImplementation::configure(IndexPropertyMapper &) { } + diff --git a/common/domain/typeimplementations.h b/common/domain/typeimplementations.h index 7e4608c..d36dfc1 100644 --- a/common/domain/typeimplementations.h +++ b/common/domain/typeimplementations.h @@ -47,6 +47,7 @@ public: static void configure(TypeIndex &index); static void configure(PropertyMapper &propertyMapper); static void configure(IndexPropertyMapper &indexPropertyMapper); + static QMap typeDatabases(); }; template<> @@ -57,6 +58,7 @@ public: static void configure(TypeIndex &); static void configure(PropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); + static QMap typeDatabases(); }; template<> @@ -67,6 +69,7 @@ public: static void configure(TypeIndex &); static void configure(PropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); + static QMap typeDatabases(); }; template<> @@ -77,6 +80,7 @@ public: static void configure(TypeIndex &); static void configure(PropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); + static QMap typeDatabases(); }; template<> @@ -87,6 +91,7 @@ public: static void configure(TypeIndex &); static void configure(PropertyMapper &); static void configure(IndexPropertyMapper &indexPropertyMapper); + static QMap typeDatabases(); }; } diff --git a/common/domain/typeimplementations_p.h b/common/domain/typeimplementations_p.h new file mode 100644 index 0000000..b7a78ac --- /dev/null +++ b/common/domain/typeimplementations_p.h @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "typeindex.h" +#include + +template +void mergeImpl(T &map, First f) +{ + for (auto it = f.constBegin(); it != f.constEnd(); it++) { + map.insert(it.key(), it.value()); + } +} + +template +void mergeImpl(T &map, First f, Tail ...maps) +{ + for (auto it = f.constBegin(); it != f.constEnd(); it++) { + map.insert(it.key(), it.value()); + } + mergeImpl(map, maps...); +} + +template +First merge(First f, Tail ...maps) +{ + First map; + mergeImpl(f, maps...); + return map; +} + +template +class ValueIndex +{ +public: + static void configure(TypeIndex &index) + { + index.addProperty(); + } + + template + static QMap databases() + { + return {{QByteArray{EntityType::name} +".index." + Property::name, 1}}; + } +}; + + +template +class SortedIndex +{ +public: + static void configure(TypeIndex &index) + { + index.addPropertyWithSorting(); + } + + template + static QMap databases() + { + return {{QByteArray{EntityType::name} +".index." + Property::name + ".sort." + SortProperty::name, 1}}; + } +}; + +template +class SecondaryIndex +{ +public: + static void configure(TypeIndex &index) + { + index.addSecondaryProperty(); + } + + template + static QMap databases() + { + return {{QByteArray{EntityType::name} +".index." + Property::name + SecondaryProperty::name, 1}}; + } +}; + +template +class CustomSecondaryIndex +{ +public: + static void configure(TypeIndex &index) + { + index.addSecondaryPropertyIndexer(); + } + + template + static QMap databases() + { + return Indexer::databases(); + } +}; + +template +class IndexConfig +{ + template + static void applyIndex(TypeIndex &index) + { + T::configure(index); + } + + ///Apply recursively for parameter pack + template + static void applyIndex(TypeIndex &index) + { + applyIndex(index); + applyIndex(index); + } + + template + static QMap getDbs() + { + return T::template databases(); + } + + template + static QMap getDbs() + { + return merge(getDbs(), getDbs()); + } + +public: + static void configure(TypeIndex &index) + { + applyIndex(index); + } + + static QMap databases() + { + return getDbs(); + } + +}; + diff --git a/common/mail/threadindexer.cpp b/common/mail/threadindexer.cpp index d91ab5f..af96b94 100644 --- a/common/mail/threadindexer.cpp +++ b/common/mail/threadindexer.cpp @@ -145,3 +145,10 @@ void ThreadIndexer::remove(const ApplicationDomain::ApplicationDomainType &entit } +QMap ThreadIndexer::databases() +{ + return {{"mail.index.messageIdthreadId", 1}, + {"mail.index.subjectthreadId", 1}, + {"mail.index.threadIdmessageId", 1}}; +} + diff --git a/common/mail/threadindexer.h b/common/mail/threadindexer.h index 064ae71..60d0863 100644 --- a/common/mail/threadindexer.h +++ b/common/mail/threadindexer.h @@ -29,6 +29,7 @@ public: virtual void add(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE; virtual void modify(const ApplicationDomain::ApplicationDomainType &old, const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE; virtual void remove(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE; + static QMap databases(); private: void updateThreadingIndex(const QByteArray &identifier, const ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction); }; diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 9539bec..38ff730 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -36,30 +36,59 @@ using namespace Sink; using namespace Sink::Storage; +static QMap baseDbs() +{ + return {{"revisionType", 0}, + {"revisions", 0}, + {"uids", 0}, + {"default", 0}, + {"__flagtable", 0}}; +} + +template +void mergeImpl(T &map, First f) +{ + for (auto it = f.constBegin(); it != f.constEnd(); it++) { + map.insert(it.key(), it.value()); + } +} + +template +void mergeImpl(T &map, First f, Tail ...maps) +{ + for (auto it = f.constBegin(); it != f.constEnd(); it++) { + map.insert(it.key(), it.value()); + } + mergeImpl(map, maps...); +} + +template +First merge(First f, Tail ...maps) +{ + First map; + mergeImpl(f, maps...); + return map; +} + +template +struct DbLayoutHelper { + void operator()(QMap map) const { + mergeImpl(map, ApplicationDomain::TypeImplementation::typeDatabases()); + } +}; + static Sink::Storage::DbLayout dbLayout(const QByteArray &instanceId) { - return Sink::Storage::DbLayout { - instanceId, - { - {"folder.main", 0}, - {"folder.index.name", 1}, - {"folder.index.parent", 1}, - {"mail.main", 0}, - {"mail.index.date", 1}, - {"mail.index.folder", 1}, - {"mail.index.folder.sort.date", 0}, - {"mail.index.messageId", 1}, - {"mail.index.messageIdthreadId", 1}, - {"mail.index.parentMessageId", 1}, - {"mail.index.subjectthreadId", 1}, - {"mail.index.threadIdmessageId", 1}, - {"revisionType", 0}, - {"revisions", 0}, - {"uids", 0}, - {"default", 0}, - {"__flagtable", 0} - } - }; + static auto databases = [] { + QMap map; + mergeImpl(map, ApplicationDomain::TypeImplementation::typeDatabases()); + mergeImpl(map, ApplicationDomain::TypeImplementation::typeDatabases()); + mergeImpl(map, ApplicationDomain::TypeImplementation::typeDatabases()); + mergeImpl(map, ApplicationDomain::TypeImplementation::typeDatabases()); + mergeImpl(map, ApplicationDomain::TypeImplementation::typeDatabases()); + return merge(baseDbs(), map); + }(); + return {instanceId, databases}; } -- cgit v1.2.3 From a818e0471366be77a1ba4abc3ee7647a3050d9df Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 8 May 2017 14:42:55 +0200 Subject: Bumped release version --- dist/sink.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/sink.spec b/dist/sink.spec index 1f48e4f..55ba6ac 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 2%{?dist} +Release: 3%{?dist} Summary: sink Group: Applications/Desktop -- cgit v1.2.3 From fd9a5b1ff3b31f80d72283d6011459127dead282 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 8 May 2017 21:39:04 +0200 Subject: Fixed merge implementation --- common/domain/typeimplementations_p.h | 2 +- common/storage/entitystore.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/domain/typeimplementations_p.h b/common/domain/typeimplementations_p.h index b7a78ac..6f77a2d 100644 --- a/common/domain/typeimplementations_p.h +++ b/common/domain/typeimplementations_p.h @@ -41,7 +41,7 @@ template First merge(First f, Tail ...maps) { First map; - mergeImpl(f, maps...); + mergeImpl(map, f, maps...); return map; } diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 38ff730..33c3c73 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -66,7 +66,7 @@ template First merge(First f, Tail ...maps) { First map; - mergeImpl(f, maps...); + mergeImpl(map, f, maps...); return map; } -- cgit v1.2.3 From 205729e3ab9664c8b2d56cc478daac2c5afd1b28 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 8 May 2017 21:40:13 +0200 Subject: Guard the changereplay callbacks using the new API --- common/changereplay.cpp | 14 ++++++-------- common/changereplay.h | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/common/changereplay.cpp b/common/changereplay.cpp index 7895b66..da36b3e 100644 --- a/common/changereplay.cpp +++ b/common/changereplay.cpp @@ -30,8 +30,7 @@ using namespace Sink; using namespace Sink::Storage; ChangeReplay::ChangeReplay(const ResourceContext &resourceContext, const Sink::Log::Context &ctx) - : mStorage(storageLocation(), resourceContext.instanceId(), DataStore::ReadOnly), mChangeReplayStore(storageLocation(), resourceContext.instanceId() + ".changereplay", DataStore::ReadWrite), mReplayInProgress(false), mLogCtx{ctx.subContext("changereplay")}, - mGuard{new QObject} + : mStorage(storageLocation(), resourceContext.instanceId(), DataStore::ReadOnly), mChangeReplayStore(storageLocation(), resourceContext.instanceId() + ".changereplay", DataStore::ReadWrite), mReplayInProgress(false), mLogCtx{ctx.subContext("changereplay")} { } @@ -84,6 +83,8 @@ KAsync::Job ChangeReplay::replayNextRevision() auto replayStoreTransaction = mChangeReplayStore.createTransaction(DataStore::ReadOnly, [this](const DataStore::Error &error) { SinkWarningCtx(mLogCtx) << error.message; }); + Q_ASSERT(mMainStoreTransaction); + Q_ASSERT(replayStoreTransaction); replayStoreTransaction.openDatabase().scan("lastReplayedRevision", [lastReplayedRevision](const QByteArray &key, const QByteArray &value) -> bool { *lastReplayedRevision = value.toLongLong(); @@ -98,14 +99,11 @@ KAsync::Job ChangeReplay::replayNextRevision() SinkTraceCtx(mLogCtx) << "Changereplay from " << *lastReplayedRevision << " to " << *topRevision; return KAsync::doWhile( [this, lastReplayedRevision, topRevision]() -> KAsync::Job { - if (!mGuard) { - SinkTraceCtx(mLogCtx) << "Exit due to guard"; - return KAsync::value(KAsync::Break); - } if (*lastReplayedRevision >= *topRevision) { SinkTraceCtx(mLogCtx) << "Done replaying" << *lastReplayedRevision << *topRevision; return KAsync::value(KAsync::Break); } + Q_ASSERT(mMainStoreTransaction); KAsync::Job replayJob = KAsync::null(); qint64 revision = *lastReplayedRevision + 1; @@ -164,7 +162,7 @@ KAsync::Job ChangeReplay::replayNextRevision() //We shouldn't ever get here Q_ASSERT(false); return KAsync::value(KAsync::Break); - }); + }).guard(&mGuard); }); }) .then([this](const KAsync::Error &error) { @@ -181,7 +179,7 @@ KAsync::Job ChangeReplay::replayNextRevision() emit changesReplayed(); } } - }); + }).guard(&mGuard); } void ChangeReplay::revisionChanged() diff --git a/common/changereplay.h b/common/changereplay.h index edc4462..ab2d857 100644 --- a/common/changereplay.h +++ b/common/changereplay.h @@ -63,7 +63,7 @@ private: bool mReplayInProgress; Sink::Storage::DataStore::Transaction mMainStoreTransaction; Sink::Log::Context mLogCtx; - QSharedPointer mGuard; + QObject mGuard; }; class NullChangeReplay : public ChangeReplay -- cgit v1.2.3 From 7c7e4e8cd4190f57f2336467615369632aada95b Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 8 May 2017 21:51:44 +0200 Subject: Bumped kasync requirement --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 695676d..53716a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ include(KDEInstallDirs) find_package(Qt5 COMPONENTS REQUIRED Core Network Gui) find_package(KF5 COMPONENTS REQUIRED Mime Contacts) find_package(FlatBuffers REQUIRED 1.4.0) -find_package(KAsync REQUIRED 0.1.0) +find_package(KAsync REQUIRED 0.1.2) find_package(LMDB REQUIRED 0.9) find_program(MEMORYCHECK_COMMAND valgrind) -- cgit v1.2.3 From a213eff8a6ec034d01df9a36d8fddae8f2be2ac1 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 8 May 2017 21:55:55 +0200 Subject: Bumped release number --- dist/sink.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/sink.spec b/dist/sink.spec index 55ba6ac..3c150d1 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 3%{?dist} +Release: 4%{?dist} Summary: sink Group: Applications/Desktop -- cgit v1.2.3 From 32a2d9b61193db0a3f7ab6f45113c7d3d1ce6a92 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 8 May 2017 23:38:36 +0200 Subject: Fixed warnings --- common/pipeline.cpp | 2 ++ tests/hawd/state.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/common/pipeline.cpp b/common/pipeline.cpp index f5cf995..dc6f128 100644 --- a/common/pipeline.cpp +++ b/common/pipeline.cpp @@ -283,6 +283,8 @@ KAsync::Job Pipeline::modifiedEntity(void const *command, size_t size) case Preprocessor::DropModification: SinkTraceCtx(d->logCtx) << "Dropping modification"; return KAsync::error(0); + case Preprocessor::NoAction: + case Preprocessor::DeleteEntity: default: break; } diff --git a/tests/hawd/state.cpp b/tests/hawd/state.cpp index dfeef41..46b5f28 100644 --- a/tests/hawd/state.cpp +++ b/tests/hawd/state.cpp @@ -117,7 +117,7 @@ void State::findGitHash() { #ifdef HAVE_LIBGIT2 git_libgit2_init(); - git_buf root = {0}; + git_buf root; int error = git_repository_discover(&root, projectPath().toStdString().data(), 0, NULL); if (!error) { git_repository *repo = NULL; -- cgit v1.2.3 From 7f4e3f838d2ca27a6f3910208235ec4b96725f14 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 8 May 2017 23:38:52 +0200 Subject: Export DbLayout --- common/storage.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/storage.h b/common/storage.h index 42cdcac..b664e11 100644 --- a/common/storage.h +++ b/common/storage.h @@ -30,9 +30,10 @@ namespace Sink { namespace Storage { -struct DbLayout { +struct SINK_EXPORT DbLayout { + typedef QMap Databases; QByteArray name; - QMap tables; + Databases tables; }; class SINK_EXPORT DataStore -- cgit v1.2.3 From 4da825f0429f1f125abe5f1843cd4e3ac5387346 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 9 May 2017 14:30:29 +0200 Subject: Translate the host not found error --- examples/imapresource/imapresource.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index e7458f6..90383e0 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -506,12 +506,16 @@ public: KAsync::Error getError(const KAsync::Error &error) { if (error) { - if (error.errorCode == Imap::CouldNotConnectError) { - return {ApplicationDomain::ConnectionError, error.errorMessage}; - } else if (error.errorCode == Imap::SslHandshakeError) { - return {ApplicationDomain::LoginError, error.errorMessage}; + switch(error.errorCode) { + case Imap::CouldNotConnectError: + return {ApplicationDomain::ConnectionError, error.errorMessage}; + case Imap::SslHandshakeError: + return {ApplicationDomain::LoginError, error.errorMessage}; + case Imap::HostNotFoundError: + return {ApplicationDomain::NoServerError, error.errorMessage}; + default: + return {ApplicationDomain::UnknownError, error.errorMessage}; } - return {ApplicationDomain::UnknownError, error.errorMessage}; } return {}; } -- cgit v1.2.3 From 0909360cdf270f6074b698bf7c34cf8566e8a71c Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 9 May 2017 22:19:37 +0200 Subject: Set the resource offline on no server host not found is pretty much the same as offline for our purpose. --- common/synchronizer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/synchronizer.cpp b/common/synchronizer.cpp index 175ed83..b9decbd 100644 --- a/common/synchronizer.cpp +++ b/common/synchronizer.cpp @@ -329,6 +329,9 @@ void Synchronizer::setStatusFromResult(const KAsync::Error &error, const QString if (error.errorCode == ApplicationDomain::ConnectionError) { //Couldn't connect, so we assume we don't have a network connection. setStatus(ApplicationDomain::OfflineStatus, s, requestId); + } else if (error.errorCode == ApplicationDomain::NoServerError) { + //Failed to contact the server. + setStatus(ApplicationDomain::OfflineStatus, s, requestId); } else if (error.errorCode == ApplicationDomain::ConfigurationError) { //There is an error with the configuration. setStatus(ApplicationDomain::ErrorStatus, s, requestId); -- cgit v1.2.3 From 6fb340f0a899c9e373cea01c0b986bb39760b67b Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 9 May 2017 22:20:21 +0200 Subject: Removed unused headers --- common/resourcefacade.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/resourcefacade.cpp b/common/resourcefacade.cpp index dee0711..de662c5 100644 --- a/common/resourcefacade.cpp +++ b/common/resourcefacade.cpp @@ -21,11 +21,9 @@ #include "resourceconfig.h" #include "query.h" #include "definitions.h" -#include "storage.h" #include "store.h" #include "resourceaccess.h" #include "resource.h" -#include using namespace Sink; -- cgit v1.2.3 From 89c5405f200bed5f255fbf26602318b3f6426e9d Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 10 May 2017 08:18:37 +0200 Subject: Make the offline status higher priority Some resources always claim to be online, so the ones offline are relevant. --- common/resourcefacade.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/resourcefacade.cpp b/common/resourcefacade.cpp index de662c5..0cf9df2 100644 --- a/common/resourcefacade.cpp +++ b/common/resourcefacade.cpp @@ -371,10 +371,10 @@ QPair, typename Sink::ResultEmitter Date: Thu, 11 May 2017 11:58:15 +0200 Subject: Fixed benchmark --- hawd_defs/mail_query_threadleader | 8 ++++++++ tests/hawd/state.cpp | 2 +- tests/mailquerybenchmark.cpp | 8 ++++---- 3 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 hawd_defs/mail_query_threadleader diff --git a/hawd_defs/mail_query_threadleader b/hawd_defs/mail_query_threadleader new file mode 100644 index 0000000..c555997 --- /dev/null +++ b/hawd_defs/mail_query_threadleader @@ -0,0 +1,8 @@ +{ + "name": "Mail Query performance", + "description": "Measures performance of mail queries", + "columns": [ + { "name": "rows", "type": "int" }, + { "name": "queryResultPerMs", "type": "float", "unit": "result/ms" } + ] +} diff --git a/tests/hawd/state.cpp b/tests/hawd/state.cpp index 46b5f28..dfeef41 100644 --- a/tests/hawd/state.cpp +++ b/tests/hawd/state.cpp @@ -117,7 +117,7 @@ void State::findGitHash() { #ifdef HAVE_LIBGIT2 git_libgit2_init(); - git_buf root; + git_buf root = {0}; int error = git_repository_discover(&root, projectPath().toStdString().data(), 0, NULL); if (!error) { git_repository *repo = NULL; diff --git a/tests/mailquerybenchmark.cpp b/tests/mailquerybenchmark.cpp index b15c8d6..00c156d 100644 --- a/tests/mailquerybenchmark.cpp +++ b/tests/mailquerybenchmark.cpp @@ -78,7 +78,7 @@ class MailQueryBenchmark : public QObject entityStore.commitTransaction(); } - void testLoad(const Sink::Query &query, int count, int expectedSize) + void testLoad(const QByteArray &name, const Sink::Query &query, int count, int expectedSize) { const auto startingRss = getCurrentRSS(); @@ -124,7 +124,7 @@ class MailQueryBenchmark : public QObject std::cout << "Rss without db [kb]: " << rssWithoutDb / 1024 << std::endl; std::cout << "Percentage error: " << percentageRssError << std::endl; - HAWD::Dataset dataset("mail_query", mHawdState); + HAWD::Dataset dataset(QString{"mail_query"} + name, mHawdState); HAWD::Dataset::Row row = dataset.row(); row.setValue("rows", list.size()); row.setValue("queryResultPerMs", (qreal)list.size() / elapsed); @@ -159,7 +159,7 @@ private slots: query.limit(1000); populateDatabase(50000); - testLoad(query, 50000, query.limit()); + testLoad({}, query, 50000, query.limit()); } void test50kThreadleader() @@ -176,7 +176,7 @@ private slots: int count = 50000; populateDatabase(count, mailsPerFolder); - testLoad(query, count, query.limit()); + testLoad("_threadleader", query, count, query.limit()); } }; -- cgit v1.2.3 From 8c63c7ff740e021e25aeb08aa60e437c54bea6e7 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 11 May 2017 12:00:55 +0200 Subject: Avoid recreating the settings object on every log call --- common/log.cpp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/common/log.cpp b/common/log.cpp index 045b3cc..4024545 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -12,12 +12,17 @@ #include #include #include +#include using namespace Sink::Log; -static QSharedPointer config() +QThreadStorage> sSettings; +static QSettings &config() { - return QSharedPointer::create(Sink::configLocation() + "/log.ini", QSettings::IniFormat); + if (!sSettings.hasLocalData()) { + sSettings.setLocalData(QSharedPointer::create(Sink::configLocation() + "/log.ini", QSettings::IniFormat)); + } + return *sSettings.localData(); } static QByteArray sPrimaryComponent; @@ -173,22 +178,22 @@ DebugLevel Sink::Log::debugLevelFromName(const QByteArray &name) void Sink::Log::setDebugOutputLevel(DebugLevel debugLevel) { - config()->setValue("level", debugLevel); + config().setValue("level", debugLevel); } Sink::Log::DebugLevel Sink::Log::debugOutputLevel() { - return static_cast(config()->value("level", Sink::Log::Log).toInt()); + return static_cast(config().value("level", Sink::Log::Log).toInt()); } void Sink::Log::setDebugOutputFilter(FilterType type, const QByteArrayList &filter) { switch (type) { case ApplicationName: - config()->setValue("applicationfilter", QVariant::fromValue(filter)); + config().setValue("applicationfilter", QVariant::fromValue(filter)); break; case Area: - config()->setValue("areafilter", QVariant::fromValue(filter)); + config().setValue("areafilter", QVariant::fromValue(filter)); break; } } @@ -197,9 +202,9 @@ QByteArrayList Sink::Log::debugOutputFilter(FilterType type) { switch (type) { case ApplicationName: - return config()->value("applicationfilter").value(); + return config().value("applicationfilter").value(); case Area: - return config()->value("areafilter").value(); + return config().value("areafilter").value(); default: return QByteArrayList(); } @@ -207,12 +212,12 @@ QByteArrayList Sink::Log::debugOutputFilter(FilterType type) void Sink::Log::setDebugOutputFields(const QByteArrayList &output) { - config()->setValue("outputfields", QVariant::fromValue(output)); + config().setValue("outputfields", QVariant::fromValue(output)); } QByteArrayList Sink::Log::debugOutputFields() { - return config()->value("outputfields").value(); + return config().value("outputfields").value(); } static QByteArray getProgramName() -- cgit v1.2.3 From ed3766450ad2f29a73c05753be9a1563377b52fc Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 11 May 2017 13:04:35 +0200 Subject: Calling mkpath once is enough --- common/definitions.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/common/definitions.cpp b/common/definitions.cpp index 3fc4700..4bf3da4 100644 --- a/common/definitions.cpp +++ b/common/definitions.cpp @@ -40,10 +40,12 @@ QString Sink::configLocation() QString Sink::temporaryFileLocation() { - auto path = dataLocation() + "/temporaryFiles"; - //FIXME create in a singleton on startup? - QDir dir; - dir.mkpath(path); + static auto path = dataLocation() + "/temporaryFiles"; + static bool initialized = false; + if (!initialized) { + QDir{}.mkpath(path); + initialized = true; + } return path; } -- cgit v1.2.3 From 6ef1f74e7a38bff92a338eb7d123c382cd4844ba Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 11 May 2017 14:39:39 +0200 Subject: Centralized Sink::Log::debugStream call --- common/log.h | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/common/log.h b/common/log.h index e85856a..abb1e55 100644 --- a/common/log.h +++ b/common/log.h @@ -90,25 +90,27 @@ SINK_EXPORT inline QDebug operator<<(QDebug d, const TraceTime &time) static const char *getComponentName() { return nullptr; } -#define Trace_area(AREA) Sink::Log::debugStream(Sink::Log::DebugLevel::Trace, __LINE__, __FILE__, Q_FUNC_INFO, AREA) -#define Log_area(AREA) Sink::Log::debugStream(Sink::Log::DebugLevel::Log, __LINE__, __FILE__, Q_FUNC_INFO, AREA) -#define Warning_area(AREA) Sink::Log::debugStream(Sink::Log::DebugLevel::Warning, __LINE__, __FILE__, Q_FUNC_INFO, AREA) -#define Error_area(AREA) Sink::Log::debugStream(Sink::Log::DebugLevel::Error, __LINE__, __FILE__, Q_FUNC_INFO, AREA) - -#define SinkTrace_(COMPONENT, AREA) Sink::Log::debugStream(Sink::Log::DebugLevel::Trace, __LINE__, __FILE__, Q_FUNC_INFO, AREA, COMPONENT) -#define SinkLog_(COMPONENT, AREA) Sink::Log::debugStream(Sink::Log::DebugLevel::Log, __LINE__, __FILE__, Q_FUNC_INFO, AREA, COMPONENT) -#define SinkWarning_(COMPONENT, AREA) Sink::Log::debugStream(Sink::Log::DebugLevel::Warning, __LINE__, __FILE__, Q_FUNC_INFO, AREA, COMPONENT) -#define SinkError_(COMPONENT, AREA) Sink::Log::debugStream(Sink::Log::DebugLevel::Error, __LINE__, __FILE__, Q_FUNC_INFO, AREA, COMPONENT) - -#define SinkTrace() Sink::Log::debugStream(Sink::Log::DebugLevel::Trace, __LINE__, __FILE__, Q_FUNC_INFO, s_sinkDebugArea, getComponentName()) -#define SinkLog() Sink::Log::debugStream(Sink::Log::DebugLevel::Log, __LINE__, __FILE__, Q_FUNC_INFO, s_sinkDebugArea, getComponentName()) -#define SinkWarning() Sink::Log::debugStream(Sink::Log::DebugLevel::Warning, __LINE__, __FILE__, Q_FUNC_INFO, s_sinkDebugArea, getComponentName()) -#define SinkError() Sink::Log::debugStream(Sink::Log::DebugLevel::Error, __LINE__, __FILE__, Q_FUNC_INFO, s_sinkDebugArea, getComponentName()) - -#define SinkTraceCtx(CTX) Sink::Log::debugStream(Sink::Log::DebugLevel::Trace, __LINE__, __FILE__, Q_FUNC_INFO, CTX.name) -#define SinkLogCtx(CTX) Sink::Log::debugStream(Sink::Log::DebugLevel::Log, __LINE__, __FILE__, Q_FUNC_INFO, CTX.name) -#define SinkWarningCtx(CTX) Sink::Log::debugStream(Sink::Log::DebugLevel::Warning, __LINE__, __FILE__, Q_FUNC_INFO, CTX.name) -#define SinkErrorCtx(CTX) Sink::Log::debugStream(Sink::Log::DebugLevel::Error, __LINE__, __FILE__, Q_FUNC_INFO, CTX.name) +#define SINK_DEBUG_STREAM_IMPL(LEVEL, AREA, COMPONENT) Sink::Log::debugStream(LEVEL, __LINE__, __FILE__, Q_FUNC_INFO, AREA, COMPONENT) + +#define Trace_area(AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, AREA, nullptr) +#define Log_area(AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, AREA, nullptr) +#define Warning_area(AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, AREA, nullptr) +#define Error_area(AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, AREA, nullptr) + +#define SinkTrace_(COMPONENT, AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, AREA, COMPONENT) +#define SinkLog_(COMPONENT, AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, AREA, COMPONENT) +#define SinkWarning_(COMPONENT, AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, AREA, COMPONENT) +#define SinkError_(COMPONENT, AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, AREA, COMPONENT) + +#define SinkTrace() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, s_sinkDebugArea, getComponentName()) +#define SinkLog() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, s_sinkDebugArea, getComponentName()) +#define SinkWarning() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, s_sinkDebugArea, getComponentName()) +#define SinkError() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, s_sinkDebugArea, getComponentName()) + +#define SinkTraceCtx(CTX) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, CTX.name, nullptr) +#define SinkLogCtx(CTX) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, CTX.name, nullptr) +#define SinkWarningCtx(CTX) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, CTX.name, nullptr) +#define SinkErrorCtx(CTX) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, CTX.name, nullptr) #define SINK_DEBUG_AREA(AREA) static constexpr const char* s_sinkDebugArea{AREA}; #define SINK_DEBUG_COMPONENT(COMPONENT) const char* getComponentName() const { return COMPONENT; }; -- cgit v1.2.3 From 003adce13169c6874a4399a3345c215a0a758f9b Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 11 May 2017 15:03:01 +0200 Subject: Fixed querytest --- tests/querytest.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/querytest.cpp b/tests/querytest.cpp index f639d94..65004b6 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -503,12 +503,13 @@ private slots: Folder folder2(resource2.identifier()); VERIFYEXEC(Sink::Store::create(folder2)); } + VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource1.identifier())); + VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource2.identifier())); // Test Sink::Query query; query.resourceFilter(account1); - // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto folders = Sink::Store::read(query); QCOMPARE(folders.size(), 1); } @@ -626,6 +627,9 @@ private slots: Folder folder2(resource2.identifier()); VERIFYEXEC(Sink::Store::create(folder2)); + VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource1.identifier())); + VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource2.identifier())); + // Test Sink::Query query; query.resourceContainsFilter("cap1"); -- cgit v1.2.3 From cf28dd267cf8e385539dc84938e4cae61b13fdb0 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 11 May 2017 15:09:37 +0200 Subject: Fixed notification test --- tests/notificationtest.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/notificationtest.cpp b/tests/notificationtest.cpp index e268a17..f0d957e 100644 --- a/tests/notificationtest.cpp +++ b/tests/notificationtest.cpp @@ -70,10 +70,9 @@ private slots: using namespace Sink::ApplicationDomain; { QList expected = { - Status::OfflineStatus, Status::ConnectedStatus, Status::BusyStatus, - Status::ConnectedStatus + Status::ConnectedStatus, }; qInfo() << "Received notifications " << statusNotifications; QVERIFY2(statusNotifications.size() <= expected.size(), "More notifications than expected."); -- cgit v1.2.3 From c427259852a34fea49687b9ba66a6eadbe8ece9c Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 11 May 2017 15:32:27 +0200 Subject: Don't store blobs in directories. Creating the directories is way more expensive than searching through the files on removal. --- common/storage/entitystore.cpp | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 33c3c73..44fd54d 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -94,7 +94,13 @@ static Sink::Storage::DbLayout dbLayout(const QByteArray &instanceId) class EntityStore::Private { public: - Private(const ResourceContext &context, const Sink::Log::Context &ctx) : resourceContext(context), logCtx(ctx.subContext("entitystore")) {} + Private(const ResourceContext &context, const Sink::Log::Context &ctx) : resourceContext(context), logCtx(ctx.subContext("entitystore")) + { + + if (!QDir().mkpath(entityBlobStorageDir())) { + SinkWarningCtx(logCtx) << "Failed to create the directory: " << entityBlobStorageDir(); + } + } ResourceContext resourceContext; DataStore::Transaction transaction; @@ -149,9 +155,14 @@ public: return ApplicationDomain::ApplicationDomainType{resourceContext.instanceId(), uid, revision, adaptor}; } + QString entityBlobStorageDir() + { + return Sink::resourceStorageLocation(resourceContext.instanceId()) + "/blob"; + } + QString entityBlobStoragePath(const QByteArray &id) { - return Sink::resourceStorageLocation(resourceContext.instanceId()) + "/blob/" + id; + return entityBlobStorageDir() +"/"+ id; } }; @@ -193,9 +204,6 @@ bool EntityStore::hasTransaction() const void EntityStore::copyBlobs(ApplicationDomain::ApplicationDomainType &entity, qint64 newRevision) { const auto directory = d->entityBlobStoragePath(entity.identifier()); - if (!QDir().mkpath(directory)) { - SinkWarningCtx(d->logCtx) << "Failed to create the directory: " << directory; - } for (const auto &property : entity.changedProperties()) { const auto value = entity.getProperty(property); @@ -204,7 +212,7 @@ void EntityStore::copyBlobs(ApplicationDomain::ApplicationDomainType &entity, qi //Any blob that is not part of the storage yet has to be moved there. if (blob.isExternal) { auto oldPath = blob.value; - auto filePath = directory + QString("/%1%2.blob").arg(QString::number(newRevision)).arg(QString::fromLatin1(property)); + auto filePath = directory + QString("_%1%2.blob").arg(QString::number(newRevision)).arg(QString::fromLatin1(property)); //In case we hit the same revision again due to a rollback. QFile::remove(filePath); QFile origFile(oldPath); @@ -397,10 +405,10 @@ void EntityStore::cleanupEntityRevisionsUntil(qint64 revision) DataStore::mainDatabase(d->transaction, bufferType).remove(key); } if (isRemoval) { - const auto directory = d->entityBlobStoragePath(uid); - QDir dir(directory); - if (!dir.removeRecursively()) { - SinkErrorCtx(d->logCtx) << "Failed to cleanup: " << directory; + QDir dir{d->entityBlobStorageDir()}; + const auto infoList = dir.entryInfoList(QStringList{} << QString{uid + "*"}); + for (const auto fileInfo : infoList) { + QFile::remove(fileInfo.filePath()); } } //Don't cleanup more than specified -- cgit v1.2.3 From 330f9418cac14664b2f80e0c2a5a89295d8ec51f Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 11 May 2017 15:55:11 +0200 Subject: Starting to get rid of the SINK_DEBUG_AREA --- common/log.h | 8 ++++---- common/storage_lmdb.cpp | 3 --- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/common/log.h b/common/log.h index abb1e55..42193b7 100644 --- a/common/log.h +++ b/common/log.h @@ -102,10 +102,10 @@ static const char *getComponentName() { return nullptr; } #define SinkWarning_(COMPONENT, AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, AREA, COMPONENT) #define SinkError_(COMPONENT, AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, AREA, COMPONENT) -#define SinkTrace() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, s_sinkDebugArea, getComponentName()) -#define SinkLog() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, s_sinkDebugArea, getComponentName()) -#define SinkWarning() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, s_sinkDebugArea, getComponentName()) -#define SinkError() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, s_sinkDebugArea, getComponentName()) +#define SinkTrace() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, __FILE__, getComponentName()) +#define SinkLog() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, __FILE__, getComponentName()) +#define SinkWarning() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, __FILE__, getComponentName()) +#define SinkError() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, __FILE__, getComponentName()) #define SinkTraceCtx(CTX) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, CTX.name, nullptr) #define SinkLogCtx(CTX) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, CTX.name, nullptr) diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index ef4bd17..c3240f6 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -35,9 +35,6 @@ #include #include "log.h" -SINK_DEBUG_AREA("storage") -// SINK_DEBUG_COMPONENT(d->storageRoot.toLatin1() + '/' + d->name.toLatin1()) - namespace Sink { namespace Storage { -- cgit v1.2.3 From 1291ccf13a0b8a1eddb6aaed24b45ceb31a2e01e Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 11 May 2017 16:36:39 +0200 Subject: initEnvironment with double-checked locking --- common/storage_lmdb.cpp | 124 ++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 57 deletions(-) diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index c3240f6..f7999d1 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -716,75 +716,85 @@ public: MDB_env *env; AccessMode mode; + Sink::Log::Context logCtx; + + void initEnvironment(const QString &fullPath, const DbLayout &layout) + { + // Ensure the environment is only created once, and that we only have one environment per process + if (!(env = sEnvironments.value(fullPath))) { + QMutexLocker locker(&sMutex); + if (!(env = sEnvironments.value(fullPath))) { + int rc = 0; + if ((rc = mdb_env_create(&env))) { + SinkWarningCtx(logCtx) << "mdb_env_create: " << rc << " " << mdb_strerror(rc); + qCritical() << "mdb_env_create: " << rc << " " << mdb_strerror(rc); + } else { + //Limit large enough to accomodate all our named dbs. This only starts to matter if the number gets large, otherwise it's just a bunch of extra entries in the main table. + mdb_env_set_maxdbs(env, 50); + const bool readOnly = (mode == ReadOnly); + unsigned int flags = MDB_NOTLS; + if (readOnly) { + flags |= MDB_RDONLY; + } + if ((rc = mdb_env_open(env, fullPath.toStdString().data(), flags, 0664))) { + SinkWarningCtx(logCtx) << "mdb_env_open: " << rc << ":" << mdb_strerror(rc); + mdb_env_close(env); + env = 0; + } else { + if (RUNNING_ON_VALGRIND) { + // In order to run valgrind this size must be smaller than half your available RAM + // https://github.com/BVLC/caffe/issues/2404 + mdb_env_set_mapsize(env, (size_t)10485760 * (size_t)1000); // 1MB * 1000 + } else { + //This is the maximum size of the db (but will not be used directly), so we make it large enough that we hopefully never run into the limit. + mdb_env_set_mapsize(env, (size_t)10485760 * (size_t)100000); // 1MB * 1000 + } + Q_ASSERT(env); + sEnvironments.insert(fullPath, env); + //Open all available dbi's + bool noLock = true; + auto t = Transaction(new Transaction::Private(readOnly, nullptr, name, env, noLock)); + if (!layout.tables.isEmpty()) { + + //TODO upgrade db if the layout has changed: + //* read existing layout + //* if layout is not the same create new layout + //If the db is read only, abort if the db is not yet existing. + //If the db is not read-only but is not existing, ensure we have a layout and create all tables. + + for (auto it = layout.tables.constBegin(); it != layout.tables.constEnd(); it++) { + bool allowDuplicates = it.value(); + t.openDatabase(it.key(), {}, allowDuplicates); + } + } else { + for (const auto &db : t.getDatabaseNames()) { + //Get dbi to store for future use. + t.openDatabase(db); + } + } + //To persist the dbis (this is also necessary for read-only transactions) + t.commit(); + } + } + } + } + } + }; -DataStore::Private::Private(const QString &s, const QString &n, AccessMode m, const DbLayout &layout) : storageRoot(s), name(n), env(0), mode(m) +DataStore::Private::Private(const QString &s, const QString &n, AccessMode m, const DbLayout &layout) : storageRoot(s), name(n), env(0), mode(m), logCtx(n.toLatin1()) { + const QString fullPath(storageRoot + '/' + name); QFileInfo dirInfo(fullPath); if (!dirInfo.exists() && mode == ReadWrite) { QDir().mkpath(fullPath); dirInfo.refresh(); } - Sink::Log::Context logCtx{n.toLatin1()}; if (mode == ReadWrite && !dirInfo.permission(QFile::WriteOwner)) { qCritical() << fullPath << "does not have write permissions. Aborting"; } else if (dirInfo.exists()) { - // Ensure the environment is only created once - QMutexLocker locker(&sMutex); - - /* - * It seems we can only ever have one environment open in the process. - * Otherwise multi-threading breaks. - */ - env = sEnvironments.value(fullPath); - if (!env) { - int rc = 0; - if ((rc = mdb_env_create(&env))) { - // TODO: handle error - SinkWarningCtx(logCtx) << "mdb_env_create: " << rc << " " << mdb_strerror(rc); - } else { - mdb_env_set_maxdbs(env, 50); - unsigned int flags = MDB_NOTLS; - if (mode == ReadOnly) { - flags |= MDB_RDONLY; - } - if ((rc = mdb_env_open(env, fullPath.toStdString().data(), flags, 0664))) { - SinkWarningCtx(logCtx) << "mdb_env_open: " << rc << ":" << mdb_strerror(rc); - mdb_env_close(env); - env = 0; - } else { - if (RUNNING_ON_VALGRIND) { - // In order to run valgrind this size must be smaller than half your available RAM - // https://github.com/BVLC/caffe/issues/2404 - const size_t dbSize = (size_t)10485760 * (size_t)1000; // 1MB * 1000 - mdb_env_set_mapsize(env, dbSize); - } else { - // FIXME: dynamic resize - const size_t dbSize = (size_t)10485760 * (size_t)8000; // 1MB * 8000 - mdb_env_set_mapsize(env, dbSize); - } - sEnvironments.insert(fullPath, env); - //Open all available dbi's - bool noLock = true; - bool requestedRead = m == ReadOnly; - auto t = Transaction(new Transaction::Private(requestedRead, nullptr, name, env, noLock)); - if (!layout.tables.isEmpty()) { - for (auto it = layout.tables.constBegin(); it != layout.tables.constEnd(); it++) { - bool allowDuplicates = it.value(); - t.openDatabase(it.key(), {}, allowDuplicates); - } - } else { - for (const auto &db : t.getDatabaseNames()) { - //Get dbi to store for future use. - t.openDatabase(db); - } - } - //To persist the dbis (this is also necessary for read-only transactions) - t.commit(); - } - } - } + initEnvironment(fullPath, layout); } } -- cgit v1.2.3 From 254f9dad6377311b8115221c252b20726acad0c5 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 11 May 2017 17:47:54 +0200 Subject: Upgrade job that we can eventually use to upgrade the storage --- common/store.cpp | 13 ++++++++++ common/store.h | 12 +++++++++ sinksh/CMakeLists.txt | 1 + sinksh/syntax_modules/sink_upgrade.cpp | 47 ++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 sinksh/syntax_modules/sink_upgrade.cpp diff --git a/common/store.cpp b/common/store.cpp index d266098..0eefe69 100644 --- a/common/store.cpp +++ b/common/store.cpp @@ -277,6 +277,19 @@ KAsync::Job Store::removeDataFromDisk(const QByteArray &identifier) }); } +KAsync::Job Store::upgrade() +{ + SinkLog() << "Upgrading..."; + return fetchAll({}) + .template each([](const ApplicationDomain::SinkResource::Ptr &resource) -> KAsync::Job { + SinkLog() << "Removing caches for " << resource->identifier(); + return removeDataFromDisk(resource->identifier()); + }) + .then([] { + SinkLog() << "Upgrade complete."; + }); +} + static KAsync::Job synchronize(const QByteArray &resource, const Sink::SyncScope &scope) { SinkLog() << "Synchronizing " << resource << scope; diff --git a/common/store.h b/common/store.h index fae76e5..34e14df 100644 --- a/common/store.h +++ b/common/store.h @@ -122,6 +122,18 @@ KAsync::Job SINK_EXPORT synchronize(const Sink::SyncScope &query); */ KAsync::Job SINK_EXPORT removeDataFromDisk(const QByteArray &resourceIdentifier); +/** + * Run upgrade jobs. + * + * Run this to upgrade your local database to a new version. + * Note that this may: + * * take a while + * * remove some/all of your local caches + * + * Note: The initial implementation simply calls removeDataFromDisk for all resources. + */ +KAsync::Job SINK_EXPORT upgrade(); + template KAsync::Job SINK_EXPORT fetchOne(const Sink::Query &query); diff --git a/sinksh/CMakeLists.txt b/sinksh/CMakeLists.txt index 82839ab..bc487c1 100644 --- a/sinksh/CMakeLists.txt +++ b/sinksh/CMakeLists.txt @@ -18,6 +18,7 @@ set(sink_cli_SRCS syntax_modules/sink_trace.cpp syntax_modules/sink_inspect.cpp syntax_modules/sink_drop.cpp + syntax_modules/sink_upgrade.cpp sinksh_utils.cpp repl/repl.cpp repl/replStates.cpp diff --git a/sinksh/syntax_modules/sink_upgrade.cpp b/sinksh/syntax_modules/sink_upgrade.cpp new file mode 100644 index 0000000..c399048 --- /dev/null +++ b/sinksh/syntax_modules/sink_upgrade.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2017 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include // tr() +#include + +#include "common/store.h" + +#include "state.h" +#include "syntaxtree.h" + +namespace SinkUpgrade +{ + +bool upgrade(const QStringList &args, State &state) +{ + state.print(QObject::tr("Upgrading...")); + Sink::Store::upgrade().exec().waitForFinished(); + state.printLine(QObject::tr("done")); + return true; +} + +Syntax::List syntax() +{ + return Syntax::List() << Syntax{"upgrade", QObject::tr("Upgrades your storage to the latest version (be careful!)"), &SinkUpgrade::upgrade, Syntax::NotInteractive}; +} + +REGISTER_SYNTAX(SinkUpgrade) + +} -- cgit v1.2.3 From 380ecbf2866a322de01b28e90dafc1a24e4d948c Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 10 May 2017 21:32:41 +0200 Subject: Don't create folders on invalid config. --- examples/maildirresource/maildirresource.cpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/examples/maildirresource/maildirresource.cpp b/examples/maildirresource/maildirresource.cpp index 40bab37..24681b5 100644 --- a/examples/maildirresource/maildirresource.cpp +++ b/examples/maildirresource/maildirresource.cpp @@ -555,18 +555,20 @@ MaildirResource::MaildirResource(const Sink::ResourceContext &resourceContext) setupPreprocessors(ENTITY_TYPE_FOLDER, QVector() << new FolderPreprocessor(mMaildirPath)); KPIM::Maildir dir(mMaildirPath, true); - SinkTrace() << "Started maildir resource for maildir: " << mMaildirPath; - { - auto draftsFolder = dir.addSubFolder("Drafts"); - auto remoteId = synchronizer->createFolder(draftsFolder, "folder", QByteArrayList() << "drafts"); - auto draftsFolderLocalId = synchronizer->syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, remoteId); - } - { - auto trashFolder = dir.addSubFolder("Trash"); - auto remoteId = synchronizer->createFolder(trashFolder, "folder", QByteArrayList() << "trash"); - auto trashFolderLocalId = synchronizer->syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, remoteId); + if (dir.isValid(false)) { + { + auto draftsFolder = dir.addSubFolder("Drafts"); + auto remoteId = synchronizer->createFolder(draftsFolder, "folder", QByteArrayList() << "drafts"); + auto draftsFolderLocalId = synchronizer->syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, remoteId); + } + { + auto trashFolder = dir.addSubFolder("Trash"); + auto remoteId = synchronizer->createFolder(trashFolder, "folder", QByteArrayList() << "trash"); + auto trashFolderLocalId = synchronizer->syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, remoteId); + } + synchronizer->commit(); } - synchronizer->commit(); + SinkTrace() << "Started maildir resource for maildir: " << mMaildirPath; } -- cgit v1.2.3 From 41f8f9bc05c059cafa780ac8aec27a56cba2c278 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 08:20:43 +0200 Subject: No more SINK_DEBUG_AREA --- common/configstore.cpp | 2 -- common/domain/applicationdomaintype.cpp | 2 -- common/domainadaptor.h | 1 - common/facade.h | 1 - common/genericresource.h | 2 -- common/listener.h | 2 -- common/mail/threadindexer.cpp | 2 -- common/mailpreprocessor.cpp | 2 -- common/messagequeue.cpp | 2 -- common/queryrunner.h | 2 -- common/resourceaccess.h | 2 -- common/resourcecontrol.cpp | 2 -- common/resourcefacade.cpp | 2 -- common/specialpurposepreprocessor.cpp | 2 -- common/storage_common.cpp | 2 -- common/store.cpp | 2 -- common/synchronizerstore.cpp | 2 -- common/test.cpp | 2 -- common/typeindex.cpp | 2 -- examples/davresource/davresource.cpp | 2 -- examples/dummyresource/resourcefactory.cpp | 2 -- examples/imapresource/imapresource.cpp | 2 -- examples/imapresource/imapserverproxy.cpp | 2 -- examples/imapresource/tests/imapmailsyncbenchmark.cpp | 2 -- examples/imapresource/tests/imapserverproxytest.cpp | 2 -- examples/maildirresource/maildirresource.cpp | 2 -- examples/mailtransportresource/mailtransport.cpp | 2 -- examples/mailtransportresource/mailtransportresource.cpp | 2 -- sinksh/state.cpp | 2 -- synchronizer/main.cpp | 2 -- tests/mailsynctest.cpp | 2 -- tests/mailtest.cpp | 2 -- tests/messagequeuetest.cpp | 2 -- 33 files changed, 64 deletions(-) diff --git a/common/configstore.cpp b/common/configstore.cpp index 1ae9da8..6751907 100644 --- a/common/configstore.cpp +++ b/common/configstore.cpp @@ -24,8 +24,6 @@ #include #include -SINK_DEBUG_AREA("configstore") - static QSharedPointer getConfig(const QByteArray &identifier) { return QSharedPointer::create(Sink::configLocation() + "/" + identifier + ".ini", QSettings::IniFormat); diff --git a/common/domain/applicationdomaintype.cpp b/common/domain/applicationdomaintype.cpp index 67d463f..7ce80ee 100644 --- a/common/domain/applicationdomaintype.cpp +++ b/common/domain/applicationdomaintype.cpp @@ -25,8 +25,6 @@ #include "storage.h" //for generateUid() #include -SINK_DEBUG_AREA("applicationdomaintype"); - QDebug Sink::ApplicationDomain::operator<< (QDebug d, const Sink::ApplicationDomain::Mail::Contact &c) { d << "Contact(" << c.name << ", " << c.emailAddress << ")"; diff --git a/common/domainadaptor.h b/common/domainadaptor.h index c38b713..f981a1f 100644 --- a/common/domainadaptor.h +++ b/common/domainadaptor.h @@ -120,7 +120,6 @@ private: */ class DatastoreBufferAdaptor : public Sink::ApplicationDomain::BufferAdaptor { - SINK_DEBUG_AREA("bufferadaptor") public: DatastoreBufferAdaptor() : BufferAdaptor() { diff --git a/common/facade.h b/common/facade.h index 45c718b..0ec2e59 100644 --- a/common/facade.h +++ b/common/facade.h @@ -48,7 +48,6 @@ template class SINK_EXPORT GenericFacade : public Sink::StoreFacade { protected: - SINK_DEBUG_AREA("facade") SINK_DEBUG_COMPONENT(mResourceContext.resourceInstanceIdentifier) public: /** diff --git a/common/genericresource.h b/common/genericresource.h index 558145c..bffc697 100644 --- a/common/genericresource.h +++ b/common/genericresource.h @@ -35,8 +35,6 @@ class CommandProcessor; */ class SINK_EXPORT GenericResource : public Resource { -protected: - SINK_DEBUG_AREA("resource") public: GenericResource(const Sink::ResourceContext &context, const QSharedPointer &pipeline = QSharedPointer()); virtual ~GenericResource(); diff --git a/common/listener.h b/common/listener.h index 8d8b529..f29130d 100644 --- a/common/listener.h +++ b/common/listener.h @@ -55,8 +55,6 @@ public: class SINK_EXPORT Listener : public QObject { Q_OBJECT - SINK_DEBUG_AREA("communication") - public: Listener(const QByteArray &resourceName, const QByteArray &resourceType, QObject *parent = 0); ~Listener(); diff --git a/common/mail/threadindexer.cpp b/common/mail/threadindexer.cpp index af96b94..473f28e 100644 --- a/common/mail/threadindexer.cpp +++ b/common/mail/threadindexer.cpp @@ -21,8 +21,6 @@ #include "typeindex.h" #include "log.h" -SINK_DEBUG_AREA("threadindex") - using namespace Sink; using namespace Sink::ApplicationDomain; diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp index dff3b3d..5c54fbc 100644 --- a/common/mailpreprocessor.cpp +++ b/common/mailpreprocessor.cpp @@ -29,8 +29,6 @@ using namespace Sink; -SINK_DEBUG_AREA("mailpreprocessor") - QString MailPropertyExtractor::getFilePathFromMimeMessagePath(const QString &s) const { return s; diff --git a/common/messagequeue.cpp b/common/messagequeue.cpp index 6e79d89..362ddfd 100644 --- a/common/messagequeue.cpp +++ b/common/messagequeue.cpp @@ -3,8 +3,6 @@ #include #include -SINK_DEBUG_AREA("messagequeue") - MessageQueue::MessageQueue(const QString &storageRoot, const QString &name) : mStorage(storageRoot, name, Sink::Storage::DataStore::ReadWrite) { } diff --git a/common/queryrunner.h b/common/queryrunner.h index 5308eac..11a302f 100644 --- a/common/queryrunner.h +++ b/common/queryrunner.h @@ -32,8 +32,6 @@ class QueryRunnerBase : public QObject { Q_OBJECT -protected: - SINK_DEBUG_AREA("queryrunner") public: typedef std::function ResultTransformation; diff --git a/common/resourceaccess.h b/common/resourceaccess.h index c32566b..b6a0b34 100644 --- a/common/resourceaccess.h +++ b/common/resourceaccess.h @@ -101,7 +101,6 @@ protected: class SINK_EXPORT ResourceAccess : public ResourceAccessInterface { Q_OBJECT - SINK_DEBUG_AREA("communication") public: typedef QSharedPointer Ptr; @@ -158,7 +157,6 @@ private: */ class SINK_EXPORT ResourceAccessFactory { - SINK_DEBUG_AREA("ResourceAccessFactory") public: static ResourceAccessFactory &instance(); Sink::ResourceAccess::Ptr getAccess(const QByteArray &instanceIdentifier, const QByteArray resourceType); diff --git a/common/resourcecontrol.cpp b/common/resourcecontrol.cpp index 70a3f7d..b6a4c0b 100644 --- a/common/resourcecontrol.cpp +++ b/common/resourcecontrol.cpp @@ -30,8 +30,6 @@ #include "log.h" #include "notifier.h" -SINK_DEBUG_AREA("resourcecontrol") - namespace Sink { KAsync::Job ResourceControl::shutdown(const QByteArray &identifier) diff --git a/common/resourcefacade.cpp b/common/resourcefacade.cpp index 0cf9df2..e6f98a9 100644 --- a/common/resourcefacade.cpp +++ b/common/resourcefacade.cpp @@ -27,8 +27,6 @@ using namespace Sink; -SINK_DEBUG_AREA("ResourceFacade") - template ConfigNotifier LocalStorageFacade::sConfigNotifier; diff --git a/common/specialpurposepreprocessor.cpp b/common/specialpurposepreprocessor.cpp index 25a6d1a..54f7f46 100644 --- a/common/specialpurposepreprocessor.cpp +++ b/common/specialpurposepreprocessor.cpp @@ -5,8 +5,6 @@ using namespace Sink; -SINK_DEBUG_AREA("SpecialPurposeProcessor") - static QHash specialPurposeFolders() { QHash hash; diff --git a/common/storage_common.cpp b/common/storage_common.cpp index dfcfc2a..1426cd5 100644 --- a/common/storage_common.cpp +++ b/common/storage_common.cpp @@ -24,8 +24,6 @@ #include "log.h" #include -SINK_DEBUG_AREA("storage") - QDebug& operator<<(QDebug &dbg, const Sink::Storage::DataStore::Error &error) { dbg << error.message << "Code: " << error.code << "Db: " << error.store; diff --git a/common/store.cpp b/common/store.cpp index 0eefe69..4735113 100644 --- a/common/store.cpp +++ b/common/store.cpp @@ -36,8 +36,6 @@ #include "storage.h" #include "log.h" -SINK_DEBUG_AREA("store") - Q_DECLARE_METATYPE(QSharedPointer>) Q_DECLARE_METATYPE(QSharedPointer); Q_DECLARE_METATYPE(std::shared_ptr); diff --git a/common/synchronizerstore.cpp b/common/synchronizerstore.cpp index 5364094..79cd920 100644 --- a/common/synchronizerstore.cpp +++ b/common/synchronizerstore.cpp @@ -25,8 +25,6 @@ using namespace Sink; -SINK_DEBUG_AREA("synchronizerstore") - SynchronizerStore::SynchronizerStore(Sink::Storage::DataStore::Transaction &transaction) : mTransaction(transaction) { diff --git a/common/test.cpp b/common/test.cpp index 90586ba..f9bfa23 100644 --- a/common/test.cpp +++ b/common/test.cpp @@ -29,8 +29,6 @@ #include "resourceconfig.h" #include "definitions.h" -SINK_DEBUG_AREA("test") - using namespace Sink; void Sink::Test::initTest() diff --git a/common/typeindex.cpp b/common/typeindex.cpp index 153aa43..3e66ffe 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -22,8 +22,6 @@ #include "index.h" #include -SINK_DEBUG_AREA("typeindex") - using namespace Sink; static QByteArray getByteArray(const QVariant &value) diff --git a/examples/davresource/davresource.cpp b/examples/davresource/davresource.cpp index 50471ed..6631148 100644 --- a/examples/davresource/davresource.cpp +++ b/examples/davresource/davresource.cpp @@ -42,8 +42,6 @@ #define ENTITY_TYPE_CONTACT "contact" #define ENTITY_TYPE_ADDRESSBOOK "addressbook" -SINK_DEBUG_AREA("davresource") - using namespace Sink; static KAsync::Job runJob(KJob *job) diff --git a/examples/dummyresource/resourcefactory.cpp b/examples/dummyresource/resourcefactory.cpp index c1f536e..dffdfc9 100644 --- a/examples/dummyresource/resourcefactory.cpp +++ b/examples/dummyresource/resourcefactory.cpp @@ -42,8 +42,6 @@ #define ENTITY_TYPE_MAIL "mail" #define ENTITY_TYPE_FOLDER "folder" -SINK_DEBUG_AREA("dummyresource") - using namespace Sink; class DummySynchronizer : public Sink::Synchronizer { diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 90383e0..94ca27a 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -46,8 +46,6 @@ #define ENTITY_TYPE_MAIL "mail" #define ENTITY_TYPE_FOLDER "folder" -SINK_DEBUG_AREA("imapresource") - Q_DECLARE_METATYPE(QSharedPointer) using namespace Imap; diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp index 7fa0b5a..a52c5eb 100644 --- a/examples/imapresource/imapserverproxy.cpp +++ b/examples/imapresource/imapserverproxy.cpp @@ -37,8 +37,6 @@ #include "log.h" #include "test.h" -SINK_DEBUG_AREA("imapserverproxy") - using namespace Imap; const char* Imap::Flags::Seen = "\\Seen"; diff --git a/examples/imapresource/tests/imapmailsyncbenchmark.cpp b/examples/imapresource/tests/imapmailsyncbenchmark.cpp index a53c148..814e325 100644 --- a/examples/imapresource/tests/imapmailsyncbenchmark.cpp +++ b/examples/imapresource/tests/imapmailsyncbenchmark.cpp @@ -31,8 +31,6 @@ using namespace Sink; using namespace Sink::ApplicationDomain; -SINK_DEBUG_AREA("ImapMailSyncBenchmark") - /** * Test of complete system using the imap resource. * diff --git a/examples/imapresource/tests/imapserverproxytest.cpp b/examples/imapresource/tests/imapserverproxytest.cpp index 476066d..271b3d9 100644 --- a/examples/imapresource/tests/imapserverproxytest.cpp +++ b/examples/imapresource/tests/imapserverproxytest.cpp @@ -12,8 +12,6 @@ using namespace Imap; -// SINK_DEBUG_AREA("imapserverproxytest") - /** */ class ImapServerProxyTest : public QObject diff --git a/examples/maildirresource/maildirresource.cpp b/examples/maildirresource/maildirresource.cpp index 24681b5..b406f63 100644 --- a/examples/maildirresource/maildirresource.cpp +++ b/examples/maildirresource/maildirresource.cpp @@ -43,8 +43,6 @@ #define ENTITY_TYPE_MAIL "mail" #define ENTITY_TYPE_FOLDER "folder" -SINK_DEBUG_AREA("maildirresource") - using namespace Sink; static QString getFilePathFromMimeMessagePath(const QString &mimeMessagePath) diff --git a/examples/mailtransportresource/mailtransport.cpp b/examples/mailtransportresource/mailtransport.cpp index 84c1556..afe0257 100644 --- a/examples/mailtransportresource/mailtransport.cpp +++ b/examples/mailtransportresource/mailtransport.cpp @@ -23,8 +23,6 @@ #include #include -SINK_DEBUG_AREA("mailtransport") - extern "C" { #include diff --git a/examples/mailtransportresource/mailtransportresource.cpp b/examples/mailtransportresource/mailtransportresource.cpp index c73219f..3d6f8e4 100644 --- a/examples/mailtransportresource/mailtransportresource.cpp +++ b/examples/mailtransportresource/mailtransportresource.cpp @@ -39,8 +39,6 @@ #define ENTITY_TYPE_MAIL "mail" -SINK_DEBUG_AREA("mailtransportresource") - using namespace Sink; class MailtransportPreprocessor : public Sink::Preprocessor diff --git a/sinksh/state.cpp b/sinksh/state.cpp index 8f4a675..7e04d28 100644 --- a/sinksh/state.cpp +++ b/sinksh/state.cpp @@ -26,8 +26,6 @@ #include "common/log.h" -SINK_DEBUG_AREA("state") - static bool s_hasEventLoop = false; class State::Private diff --git a/synchronizer/main.cpp b/synchronizer/main.cpp index c66a2fb..3c41c67 100644 --- a/synchronizer/main.cpp +++ b/synchronizer/main.cpp @@ -39,8 +39,6 @@ #include "test.h" #include "definitions.h" -SINK_DEBUG_AREA("main") - static Listener *listener = nullptr; //Print a demangled stacktrace diff --git a/tests/mailsynctest.cpp b/tests/mailsynctest.cpp index c8ba9f1..5d51fbc 100644 --- a/tests/mailsynctest.cpp +++ b/tests/mailsynctest.cpp @@ -33,8 +33,6 @@ using namespace Sink; using namespace Sink::ApplicationDomain; -SINK_DEBUG_AREA("mailsynctest") - void MailSyncTest::initTestCase() { Test::initTest(); diff --git a/tests/mailtest.cpp b/tests/mailtest.cpp index c51fc56..e1361e4 100644 --- a/tests/mailtest.cpp +++ b/tests/mailtest.cpp @@ -31,8 +31,6 @@ using namespace Sink; using namespace Sink::ApplicationDomain; -SINK_DEBUG_AREA("mailtest") - void MailTest::initTestCase() { Test::initTest(); diff --git a/tests/messagequeuetest.cpp b/tests/messagequeuetest.cpp index 83fa23f..2e3bd75 100644 --- a/tests/messagequeuetest.cpp +++ b/tests/messagequeuetest.cpp @@ -9,8 +9,6 @@ #include "log.h" #include "test.h" -SINK_DEBUG_AREA("messagequeuetest") - /** * Test of the messagequeue implementation. */ -- cgit v1.2.3 From 35815a20a78fc440bdf79c64ee86cfcd62a23557 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 08:21:31 +0200 Subject: Fixed build --- common/log.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/log.cpp b/common/log.cpp index 4024545..1fbc482 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -16,7 +16,7 @@ using namespace Sink::Log; -QThreadStorage> sSettings; +static QThreadStorage> sSettings; static QSettings &config() { if (!sSettings.hasLocalData()) { -- cgit v1.2.3 From 497b88f48e634a00ae952afc1776e317c9da460a Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 08:23:22 +0200 Subject: Avoid copying --- common/storage/entitystore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 44fd54d..d2161a6 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -407,7 +407,7 @@ void EntityStore::cleanupEntityRevisionsUntil(qint64 revision) if (isRemoval) { QDir dir{d->entityBlobStorageDir()}; const auto infoList = dir.entryInfoList(QStringList{} << QString{uid + "*"}); - for (const auto fileInfo : infoList) { + for (const auto &fileInfo : infoList) { QFile::remove(fileInfo.filePath()); } } -- cgit v1.2.3 From 4fd06c19bf8ca057f4a60f762568158fb0caa8ec Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 08:32:59 +0200 Subject: Try once more to initialize git_buf portably... --- tests/hawd/state.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/hawd/state.cpp b/tests/hawd/state.cpp index dfeef41..a994718 100644 --- a/tests/hawd/state.cpp +++ b/tests/hawd/state.cpp @@ -117,7 +117,7 @@ void State::findGitHash() { #ifdef HAVE_LIBGIT2 git_libgit2_init(); - git_buf root = {0}; + git_buf root = GIT_BUF_INIT_CONST(0, 0); int error = git_repository_discover(&root, projectPath().toStdString().data(), 0, NULL); if (!error) { git_repository *repo = NULL; -- cgit v1.2.3 From e61c2cad0451cddf1f84a32ed818b4c3bc7b6b0f Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 10:03:17 +0200 Subject: Only use the filename as area, not the full path. --- common/log.cpp | 8 +++++++- common/log.h | 8 ++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/common/log.cpp b/common/log.cpp index 1fbc482..e9e16c6 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -305,6 +305,12 @@ static bool caseInsensitiveContains(const QByteArray &pattern, const QByteArrayL return false; } +static QByteArray getFileName(const char *file) +{ + auto filename = QByteArray(file).split('/').last(); + return filename.split('.').first(); +} + QDebug Sink::Log::debugStream(DebugLevel debugLevel, int line, const char *file, const char *function, const char *debugArea, const char *debugComponent) { static NullStream nullstream; @@ -315,7 +321,7 @@ QDebug Sink::Log::debugStream(DebugLevel debugLevel, int line, const char *file, if (sPrimaryComponent.isEmpty()) { sPrimaryComponent = getProgramName(); } - QString fullDebugArea = sPrimaryComponent + "." + (debugComponent ? (QString::fromLatin1(debugComponent) + ".") : "") + (debugArea ? QString::fromLatin1(debugArea) : ""); + QString fullDebugArea = sPrimaryComponent + "." + (debugComponent ? (QString::fromLatin1(debugComponent) + ".") : "") + (debugArea ? QString::fromLatin1(debugArea) : getFileName(file)); collectDebugArea(fullDebugArea); diff --git a/common/log.h b/common/log.h index 42193b7..be5a508 100644 --- a/common/log.h +++ b/common/log.h @@ -102,10 +102,10 @@ static const char *getComponentName() { return nullptr; } #define SinkWarning_(COMPONENT, AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, AREA, COMPONENT) #define SinkError_(COMPONENT, AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, AREA, COMPONENT) -#define SinkTrace() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, __FILE__, getComponentName()) -#define SinkLog() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, __FILE__, getComponentName()) -#define SinkWarning() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, __FILE__, getComponentName()) -#define SinkError() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, __FILE__, getComponentName()) +#define SinkTrace() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, nullptr, getComponentName()) +#define SinkLog() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, nullptr, getComponentName()) +#define SinkWarning() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Warning, nullptr, getComponentName()) +#define SinkError() SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Error, nullptr, getComponentName()) #define SinkTraceCtx(CTX) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, CTX.name, nullptr) #define SinkLogCtx(CTX) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, CTX.name, nullptr) -- cgit v1.2.3 From 8e91505e656743bd69166b3ba1cf29dfa7cfbea5 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 10:33:52 +0200 Subject: Allow substring matches on the trace identifier --- common/log.cpp | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/common/log.cpp b/common/log.cpp index e9e16c6..d1a3a37 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -1,6 +1,7 @@ #include "log.h" #include +#include #include #include #include @@ -282,14 +283,16 @@ static bool containsItemStartingWith(const QByteArray &pattern, const QByteArray for (const auto &item : list) { if (item.startsWith('*')) { auto stripped = item.mid(1); - stripped.endsWith('*'); - stripped.chop(1); + if (stripped.endsWith('*')) { + stripped.chop(1); + } if (pattern.contains(stripped)) { return true; } - } - if (pattern.startsWith(item)) { - return true; + } else { + if (pattern.contains(item)) { + return true; + } } } return false; @@ -307,29 +310,39 @@ static bool caseInsensitiveContains(const QByteArray &pattern, const QByteArrayL static QByteArray getFileName(const char *file) { - auto filename = QByteArray(file).split('/').last(); + static char sep = QDir::separator().toLatin1(); + auto filename = QByteArray(file).split(sep).last(); return filename.split('.').first(); } -QDebug Sink::Log::debugStream(DebugLevel debugLevel, int line, const char *file, const char *function, const char *debugArea, const char *debugComponent) +bool isFiltered(DebugLevel debugLevel, const QByteArray &fullDebugArea) { - static NullStream nullstream; if (debugLevel < debugOutputLevel()) { - return QDebug(&nullstream); + return true; } + auto areas = debugOutputFilter(Sink::Log::Area); + if (debugLevel <= Sink::Log::Trace && !areas.isEmpty()) { + if (!containsItemStartingWith(fullDebugArea, areas)) { + return true; + } + } + return false; +} +QDebug Sink::Log::debugStream(DebugLevel debugLevel, int line, const char *file, const char *function, const char *debugArea, const char *debugComponent) +{ if (sPrimaryComponent.isEmpty()) { sPrimaryComponent = getProgramName(); } - QString fullDebugArea = sPrimaryComponent + "." + (debugComponent ? (QString::fromLatin1(debugComponent) + ".") : "") + (debugArea ? QString::fromLatin1(debugArea) : getFileName(file)); + const QByteArray fullDebugArea = sPrimaryComponent + "." + + (debugComponent ? (QByteArray{debugComponent} + ".") : "") + + (debugArea ? QByteArray{debugArea} : getFileName(file)); collectDebugArea(fullDebugArea); - auto areas = debugOutputFilter(Sink::Log::Area); - if (debugLevel <= Sink::Log::Trace && !areas.isEmpty()) { - if (!containsItemStartingWith(fullDebugArea.toUtf8(), areas)) { - return QDebug(&nullstream); - } + static NullStream nullstream; + if (isFiltered(debugLevel, fullDebugArea)) { + return QDebug(&nullstream); } QString prefix; @@ -378,12 +391,12 @@ QDebug Sink::Log::debugStream(DebugLevel debugLevel, int line, const char *file, } static std::atomic maxDebugAreaSize{25}; maxDebugAreaSize = qMax(fullDebugArea.size(), maxDebugAreaSize.load()); - output += QString(" %1 ").arg(fullDebugArea.leftJustified(maxDebugAreaSize, ' ', false)); + output += QString(" %1 ").arg(QString{fullDebugArea}.leftJustified(maxDebugAreaSize, ' ', false)); if (useColor) { output += resetColor; } if (showFunction) { - output += QString(" %3").arg(fullDebugArea.leftJustified(25, ' ', true)); + output += QString(" %3").arg(QString{fullDebugArea}.leftJustified(25, ' ', true)); } if (showLocation) { const auto filename = QString::fromLatin1(file).split('/').last(); -- cgit v1.2.3 From d425436c268a748381f003646bd02e8d74ef7fba Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 10:41:48 +0200 Subject: Added draft property index. We need this for the composer to efficiently query. Since we don't have any migration code this will require all data to be refetched (sinksh upgrade). --- common/domain/typeimplementations.cpp | 1 + common/typeindex.cpp | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp index 4e61bad..40b0cfd 100644 --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -41,6 +41,7 @@ typedef IndexConfig, ValueIndex, ValueIndex, + ValueIndex, SortedIndex, SecondaryIndex, SecondaryIndex, diff --git a/common/typeindex.cpp b/common/typeindex.cpp index 3e66ffe..b8845b7 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -31,6 +31,10 @@ static QByteArray getByteArray(const QVariant &value) if (result.isEmpty()) { return "nodate"; } + return result; + } + if (value.type() == QVariant::Bool) { + return value.toBool() ? "t" : "f"; } if (value.canConvert()) { const auto ba = value.value().value; @@ -82,6 +86,20 @@ void TypeIndex::addProperty(const QByteArray &property) mProperties << property; } +template <> +void TypeIndex::addProperty(const QByteArray &property) +{ + auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { + if (add) { + Index(indexName(property), transaction).add(getByteArray(value), identifier); + } else { + Index(indexName(property), transaction).remove(getByteArray(value), identifier); + } + }; + mIndexer.insert(property, indexer); + mProperties << property; +} + template <> void TypeIndex::addProperty(const QByteArray &property) { -- cgit v1.2.3 From 8730949269771ad4d6bba9ba2414d272f9a0a799 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 11:31:17 +0200 Subject: Made the test a bit conciser --- tests/querytest.cpp | 76 ++++++++++++++++++----------------------------------- 1 file changed, 26 insertions(+), 50 deletions(-) diff --git a/tests/querytest.cpp b/tests/querytest.cpp index 65004b6..72eca86 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -98,10 +98,7 @@ private slots: void testSingle() { // Setup - { - Mail mail("sink.dummy.instance1"); - VERIFYEXEC(Sink::Store::create(mail)); - } + VERIFYEXEC(Sink::Store::create(Mail("sink.dummy.instance1"))); // Test Sink::Query query; @@ -116,10 +113,7 @@ private slots: void testSingleWithDelay() { // Setup - { - Mail mail("sink.dummy.instance1"); - Sink::Store::create(mail).exec().waitForFinished(); - } + VERIFYEXEC(Sink::Store::create(Mail("sink.dummy.instance1"))); // Test Sink::Query query; @@ -142,13 +136,13 @@ private slots: Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder("folder1"); - Sink::Store::create(mail).exec().waitForFinished(); + VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setFolder("folder2"); - Sink::Store::create(mail).exec().waitForFinished(); + VERIFYEXEC(Sink::Store::create(mail)); } // Test @@ -164,13 +158,13 @@ private slots: auto mail = model->index(0, 0, QModelIndex()).data(Sink::Store::DomainObjectRole).value(); { mail->setFolder("folder2"); - Sink::Store::modify(*mail).exec().waitForFinished(); + VERIFYEXEC(Sink::Store::modify(*mail)); } QTRY_COMPARE(model->rowCount(), 0); { mail->setFolder("folder1"); - Sink::Store::modify(*mail).exec().waitForFinished(); + VERIFYEXEC(Sink::Store::modify(*mail)); } QTRY_COMPARE(model->rowCount(), 1); } @@ -181,8 +175,8 @@ private slots: // Setup { Mail mail("sink.dummy.instance1"); - Sink::Store::create(mail).exec().waitForFinished(); - Sink::Store::create(mail).exec().waitForFinished(); + VERIFYEXEC(Sink::Store::create(mail)); + VERIFYEXEC(Sink::Store::create(mail)); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); @@ -211,7 +205,7 @@ private slots: // Setup { Folder folder("sink.dummy.instance1"); - Sink::Store::create(folder).exec().waitForFinished(); + VERIFYEXEC(Sink::Store::create(folder)); } // Test @@ -622,20 +616,14 @@ private slots: resource2.setResourceType("sink.dummy"); VERIFYEXEC(Store::create(resource2)); - Folder folder1(resource1.identifier()); - VERIFYEXEC(Sink::Store::create(folder1)); - Folder folder2(resource2.identifier()); - VERIFYEXEC(Sink::Store::create(folder2)); + VERIFYEXEC(Sink::Store::create(Folder{resource1.identifier()})); + VERIFYEXEC(Sink::Store::create(Folder{resource2.identifier()})); - VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource1.identifier())); - VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource2.identifier())); - - // Test - Sink::Query query; - query.resourceContainsFilter("cap1"); + VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource1.identifier())); + VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource2.identifier())); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data - auto folders = Sink::Store::read(query); + auto folders = Sink::Store::read(Sink::Query{}.resourceContainsFilter("cap1")); QCOMPARE(folders.size(), 1); } @@ -1063,30 +1051,18 @@ private slots: QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; - { - auto mail1 = Mail::createEntity("sink.dummy.instance1"); - mail1.setExtractedMessageId("mail1"); - mail1.setFolder(folder1); - mail1.setExtractedDate(now); - mail1.setImportant(false); - VERIFYEXEC(Sink::Store::create(mail1)); - } - { - auto mail2 = Mail::createEntity("sink.dummy.instance1"); - mail2.setExtractedMessageId("mail2"); - mail2.setFolder(folder1); - mail2.setExtractedDate(earlier); - mail2.setImportant(false); - VERIFYEXEC(Sink::Store::create(mail2)); - } - { - auto mail3 = Mail::createEntity("sink.dummy.instance1"); - mail3.setExtractedMessageId("mail3"); - mail3.setFolder(folder1); - mail3.setExtractedDate(later); - mail3.setImportant(true); - VERIFYEXEC(Sink::Store::create(mail3)); - } + auto createMail = [] (const QByteArray &messageid, const Folder &folder, const QDateTime &date, bool important) { + auto mail = Mail::createEntity("sink.dummy.instance1"); + mail.setExtractedMessageId(messageid); + mail.setFolder(folder); + mail.setExtractedDate(date); + mail.setImportant(important); + return mail; + }; + + VERIFYEXEC(Sink::Store::create(createMail("mail1", folder1, now, false))); + VERIFYEXEC(Sink::Store::create(createMail("mail2", folder1, earlier, false))); + VERIFYEXEC(Sink::Store::create(createMail("mail3", folder1, later, true))); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); -- cgit v1.2.3 From d3cb4e649177443e8ecfd7f86d136fc0a319e2f3 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 14:00:54 +0200 Subject: Don't add resources that don't match the query during a livequery --- common/resourcefacade.cpp | 43 +++++++++++++++++++++++++++++++------------ common/resourcefacade.h | 18 +++++++++--------- tests/querytest.cpp | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/common/resourcefacade.cpp b/common/resourcefacade.cpp index e6f98a9..dab6aed 100644 --- a/common/resourcefacade.cpp +++ b/common/resourcefacade.cpp @@ -96,17 +96,24 @@ template LocalStorageQueryRunner::LocalStorageQueryRunner(const Query &query, const QByteArray &identifier, const QByteArray &typeName, ConfigNotifier &configNotifier, const Sink::Log::Context &ctx) : mResultProvider(new ResultProvider), mConfigStore(identifier, typeName), mGuard(new QObject), mLogCtx(ctx.subContext("config")) { + + auto matchesTypeAndIds = [query, this] (const QByteArray &type, const QByteArray &id) { + if (query.hasFilter(ApplicationDomain::SinkResource::ResourceType::name) && query.getFilter(ApplicationDomain::SinkResource::ResourceType::name).value.toByteArray() != type) { + SinkTraceCtx(mLogCtx) << "Skipping due to type."; + return false; + } + if (!query.ids().isEmpty() && !query.ids().contains(id)) { + return false; + } + return true; + }; + QObject *guard = new QObject; - mResultProvider->setFetcher([this, query, guard, &configNotifier](const QSharedPointer &) { + mResultProvider->setFetcher([this, query, guard, &configNotifier, matchesTypeAndIds](const QSharedPointer &) { const auto entries = mConfigStore.getEntries(); for (const auto &res : entries.keys()) { const auto type = entries.value(res); - - if (query.hasFilter(ApplicationDomain::SinkResource::ResourceType::name) && query.getFilter(ApplicationDomain::SinkResource::ResourceType::name).value.toByteArray() != type) { - SinkTraceCtx(mLogCtx) << "Skipping due to type."; - continue; - } - if (!query.ids().isEmpty() && !query.ids().contains(res)) { + if (!matchesTypeAndIds(type, res)){ continue; } auto entity = readFromConfig(mConfigStore, res, type, query.requestedProperties); @@ -124,8 +131,14 @@ LocalStorageQueryRunner::LocalStorageQueryRunner(const Query &query, }); if (query.liveQuery()) { { - auto ret = QObject::connect(&configNotifier, &ConfigNotifier::added, guard, [this](const ApplicationDomain::ApplicationDomainType::Ptr &entry) { + auto ret = QObject::connect(&configNotifier, &ConfigNotifier::added, guard, [this, query, matchesTypeAndIds](const ApplicationDomain::ApplicationDomainType::Ptr &entry, const QByteArray &type) { auto entity = entry.staticCast(); + if (!matchesTypeAndIds(type, entity->identifier())){ + return; + } + if (!matchesFilter(query.getBaseFilters(), *entity)){ + return; + } SinkTraceCtx(mLogCtx) << "A new resource has been added: " << entity->identifier(); updateStatus(*entity); mResultProvider->add(entity); @@ -133,8 +146,14 @@ LocalStorageQueryRunner::LocalStorageQueryRunner(const Query &query, Q_ASSERT(ret); } { - auto ret = QObject::connect(&configNotifier, &ConfigNotifier::modified, guard, [this](const ApplicationDomain::ApplicationDomainType::Ptr &entry) { + auto ret = QObject::connect(&configNotifier, &ConfigNotifier::modified, guard, [this, query, matchesTypeAndIds](const ApplicationDomain::ApplicationDomainType::Ptr &entry, const QByteArray &type) { auto entity = entry.staticCast(); + if (!matchesTypeAndIds(type, entity->identifier())){ + return; + } + if (!matchesFilter(query.getBaseFilters(), *entity)){ + return; + } updateStatus(*entity); mResultProvider->modify(entity); }); @@ -218,7 +237,7 @@ KAsync::Job LocalStorageFacade::create(const DomainType &domai } configStore.modify(identifier, configurationValues); } - sConfigNotifier.add(::readFromConfig(configStore, identifier, type, QByteArrayList{})); + sConfigNotifier.add(::readFromConfig(configStore, identifier, type, QByteArrayList{}), type); }); } @@ -247,7 +266,7 @@ KAsync::Job LocalStorageFacade::modify(const DomainType &domai } const auto type = configStore.getEntries().value(identifier); - sConfigNotifier.modify(::readFromConfig(configStore, identifier, type, QByteArrayList{})); + sConfigNotifier.modify(::readFromConfig(configStore, identifier, type, QByteArrayList{}), type); }); } @@ -277,7 +296,7 @@ KAsync::Job LocalStorageFacade::remove(const DomainType &domai SinkTrace() << "Removing: " << identifier; auto configStore = ConfigStore(configStoreIdentifier, typeName); configStore.remove(identifier); - sConfigNotifier.remove(QSharedPointer::create(domainObject)); + sConfigNotifier.remove(QSharedPointer::create(domainObject), typeName); }); } diff --git a/common/resourcefacade.h b/common/resourcefacade.h index 1cc075c..76fadce 100644 --- a/common/resourcefacade.h +++ b/common/resourcefacade.h @@ -36,24 +36,24 @@ class ConfigNotifier : public QObject { Q_OBJECT public: - void add(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account) + void add(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account, const QByteArray &type) { - emit added(account); + emit added(account, type); } - void remove(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account) + void remove(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account, const QByteArray &type) { - emit removed(account); + emit removed(account, type); } - void modify(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account) + void modify(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account, const QByteArray &type) { - emit modified(account); + emit modified(account, type); } signals: - void added(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account); - void removed(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account); - void modified(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account); + void added(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account, const QByteArray &type); + void removed(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account, const QByteArray &type); + void modified(const Sink::ApplicationDomain::ApplicationDomainType::Ptr &account, const QByteArray &type); }; template diff --git a/tests/querytest.cpp b/tests/querytest.cpp index 72eca86..4ff1be8 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -625,6 +625,41 @@ private slots: // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto folders = Sink::Store::read(Sink::Query{}.resourceContainsFilter("cap1")); QCOMPARE(folders.size(), 1); + + //TODO this should be part of the regular cleanup between tests + VERIFYEXEC(Store::remove(resource1)); + VERIFYEXEC(Store::remove(resource2)); + } + + void testFilteredLiveResourceSubQuery() + { + using namespace Sink; + using namespace Sink::ApplicationDomain; + + //Setup + auto resource1 = ApplicationDomainType::createEntity(); + resource1.setResourceType("sink.dummy"); + resource1.setCapabilities(QByteArrayList() << "cap1"); + VERIFYEXEC(Store::create(resource1)); + VERIFYEXEC(Store::create(Folder{resource1.identifier()})); + VERIFYEXEC(ResourceControl::flushMessageQueue(resource1.identifier())); + + auto model = Sink::Store::loadModel(Query{Query::LiveQuery}.resourceContainsFilter("cap1")); + QTRY_COMPARE(model->rowCount(), 1); + + auto resource2 = ApplicationDomainType::createEntity(); + resource2.setCapabilities(QByteArrayList() << "cap2"); + resource2.setResourceType("sink.dummy"); + VERIFYEXEC(Store::create(resource2)); + VERIFYEXEC(Store::create(Folder{resource2.identifier()})); + VERIFYEXEC(ResourceControl::flushMessageQueue(resource2.identifier())); + + //The new resource should be filtered and thus not make it in here + QCOMPARE(model->rowCount(), 1); + + //TODO this should be part of the regular cleanup between tests + VERIFYEXEC(Store::remove(resource1)); + VERIFYEXEC(Store::remove(resource2)); } void testLivequeryUnmatchInThread() -- cgit v1.2.3 From b9af56291f5bfc898ce232b4ebf5065554484962 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 14:25:35 +0200 Subject: We run into this when trying to index a property that is not set. i.e. in tests. --- common/bufferadaptor.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/bufferadaptor.h b/common/bufferadaptor.h index fd4809b..f393388 100644 --- a/common/bufferadaptor.h +++ b/common/bufferadaptor.h @@ -66,8 +66,7 @@ public: virtual QVariant getProperty(const QByteArray &key) const { if (!mValues.contains(key)) { - qWarning() << "Tried to read value that is not available; Did you forget to call Query::request? Property: " << key; - return QVariant{}; + return {}; } return mValues.value(key); } -- cgit v1.2.3 From 85a4d5ac9561b639c47721e4e2305b6ced6be261 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 14:26:07 +0200 Subject: QDateTime::toString is really slow --- common/propertymapper.cpp | 13 ++++++++++--- common/typeindex.cpp | 8 ++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/common/propertymapper.cpp b/common/propertymapper.cpp index c72cf31..c14a62e 100644 --- a/common/propertymapper.cpp +++ b/common/propertymapper.cpp @@ -21,6 +21,7 @@ #include "applicationdomaintype.h" #include +#include #include "mail_generated.h" #include "contact_generated.h" @@ -66,7 +67,10 @@ template <> flatbuffers::uoffset_t variantToProperty(const QVariant &property, flatbuffers::FlatBufferBuilder &fbb) { if (property.isValid()) { - return fbb.CreateString(property.toDateTime().toString().toStdString()).o; + QByteArray ba; + QDataStream ds(&ba, QIODevice::WriteOnly); + ds << property.toDateTime(); + return fbb.CreateString(ba.toStdString()).o; } return 0; } @@ -256,8 +260,11 @@ template <> QVariant propertyToVariant(const flatbuffers::String *property) { if (property) { - // We have to copy the memory, otherwise it would become eventually invalid - return QDateTime::fromString(QString::fromStdString(property->c_str())); + auto ba = QByteArray::fromRawData(property->c_str(), property->size()); + QDateTime dt; + QDataStream ds(&ba, QIODevice::ReadOnly); + ds >> dt; + return dt; } return QVariant(); } diff --git a/common/typeindex.cpp b/common/typeindex.cpp index b8845b7..113c209 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -21,16 +21,16 @@ #include "log.h" #include "index.h" #include +#include using namespace Sink; static QByteArray getByteArray(const QVariant &value) { if (value.type() == QVariant::DateTime) { - const auto result = value.toDateTime().toString().toLatin1(); - if (result.isEmpty()) { - return "nodate"; - } + QByteArray result; + QDataStream ds(&result, QIODevice::WriteOnly); + ds << value.toDateTime(); return result; } if (value.type() == QVariant::Bool) { -- cgit v1.2.3 From 076ae4c6bec75b081fec9cdc7a84d82ffcadc94f Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 14:58:33 +0200 Subject: Bumped release number --- dist/sink.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/sink.spec b/dist/sink.spec index 3c150d1..dbe1007 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 4%{?dist} +Release: 5%{?dist} Summary: sink Group: Applications/Desktop -- cgit v1.2.3 From 6a3bf46334fc4136da480287898d3f19c88261ee Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 16:15:34 +0200 Subject: Avoid redoing stuff over and over that we can easily avoid. --- common/definitions.cpp | 11 +++++++---- common/storage/entitystore.cpp | 10 +++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/common/definitions.cpp b/common/definitions.cpp index 4bf3da4..7f4fbbe 100644 --- a/common/definitions.cpp +++ b/common/definitions.cpp @@ -30,12 +30,14 @@ QString Sink::storageLocation() QString Sink::dataLocation() { - return QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + "/sink"; + static auto location = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + "/sink"; + return location; } QString Sink::configLocation() { - return QStandardPaths::writableLocation(QStandardPaths::GenericConfigLocation) + "/sink"; + static auto location = QStandardPaths::writableLocation(QStandardPaths::GenericConfigLocation) + "/sink"; + return location; } QString Sink::temporaryFileLocation() @@ -43,8 +45,9 @@ QString Sink::temporaryFileLocation() static auto path = dataLocation() + "/temporaryFiles"; static bool initialized = false; if (!initialized) { - QDir{}.mkpath(path); - initialized = true; + if (QDir{}.mkpath(path)) { + initialized = true; + } } return path; } diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index d2161a6..6ff700e 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -96,9 +96,13 @@ class EntityStore::Private { public: Private(const ResourceContext &context, const Sink::Log::Context &ctx) : resourceContext(context), logCtx(ctx.subContext("entitystore")) { - - if (!QDir().mkpath(entityBlobStorageDir())) { - SinkWarningCtx(logCtx) << "Failed to create the directory: " << entityBlobStorageDir(); + static bool initialized = false; + if (!initialized) { + if (QDir{}.mkpath(entityBlobStorageDir())) { + initialized = true; + } else { + SinkWarningCtx(logCtx) << "Failed to create the directory: " << entityBlobStorageDir(); + } } } -- cgit v1.2.3 From 58b9fa88198eecc224597e52d8bbd7f833fca63b Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 17:01:25 +0200 Subject: Completely shortcut the stream evaluation if we're not going to use the output Serializing the data is the expensive part, so we want to completely avoid that for the noisier part if we're not going to use it. Additionally we're now using a stringbuilder for the debugarea to try to further improve the situation with temporary memory allocations. --- common/log.cpp | 37 ++++++++++++++++++++++++------------- common/log.h | 5 ++++- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/common/log.cpp b/common/log.cpp index d1a3a37..5dfb872 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -14,6 +14,7 @@ #include #include #include +#include using namespace Sink::Log; @@ -315,13 +316,24 @@ static QByteArray getFileName(const char *file) return filename.split('.').first(); } -bool isFiltered(DebugLevel debugLevel, const QByteArray &fullDebugArea) +static QString assembleDebugArea(const char *debugArea, const char *debugComponent, const char *file) +{ + if (sPrimaryComponent.isEmpty()) { + sPrimaryComponent = getProgramName(); + } + //Using stringbuilder for fewer allocations + return QLatin1String{sPrimaryComponent} % QLatin1String{"."} % + (debugComponent ? (QLatin1String{debugComponent} + QLatin1String{"."}) : QLatin1String{""}) % + (debugArea ? QLatin1String{debugArea} : QLatin1String{getFileName(file)}); +} + +static bool isFiltered(DebugLevel debugLevel, const QByteArray &fullDebugArea) { if (debugLevel < debugOutputLevel()) { return true; } - auto areas = debugOutputFilter(Sink::Log::Area); - if (debugLevel <= Sink::Log::Trace && !areas.isEmpty()) { + const auto areas = debugOutputFilter(Sink::Log::Area); + if ((debugLevel <= Sink::Log::Trace) && !areas.isEmpty()) { if (!containsItemStartingWith(fullDebugArea, areas)) { return true; } @@ -329,19 +341,18 @@ bool isFiltered(DebugLevel debugLevel, const QByteArray &fullDebugArea) return false; } -QDebug Sink::Log::debugStream(DebugLevel debugLevel, int line, const char *file, const char *function, const char *debugArea, const char *debugComponent) +bool Sink::Log::isFiltered(DebugLevel debugLevel, const char *debugArea, const char *debugComponent, const char *file) { - if (sPrimaryComponent.isEmpty()) { - sPrimaryComponent = getProgramName(); - } - const QByteArray fullDebugArea = sPrimaryComponent + "." + - (debugComponent ? (QByteArray{debugComponent} + ".") : "") + - (debugArea ? QByteArray{debugArea} : getFileName(file)); + return isFiltered(debugLevel, assembleDebugArea(debugArea, debugComponent, file).toLatin1()); +} +QDebug Sink::Log::debugStream(DebugLevel debugLevel, int line, const char *file, const char *function, const char *debugArea, const char *debugComponent) +{ + const auto fullDebugArea = assembleDebugArea(debugArea, debugComponent, file); collectDebugArea(fullDebugArea); static NullStream nullstream; - if (isFiltered(debugLevel, fullDebugArea)) { + if (isFiltered(debugLevel, fullDebugArea.toLatin1())) { return QDebug(&nullstream); } @@ -391,12 +402,12 @@ QDebug Sink::Log::debugStream(DebugLevel debugLevel, int line, const char *file, } static std::atomic maxDebugAreaSize{25}; maxDebugAreaSize = qMax(fullDebugArea.size(), maxDebugAreaSize.load()); - output += QString(" %1 ").arg(QString{fullDebugArea}.leftJustified(maxDebugAreaSize, ' ', false)); + output += QString(" %1 ").arg(fullDebugArea.leftJustified(maxDebugAreaSize, ' ', false)); if (useColor) { output += resetColor; } if (showFunction) { - output += QString(" %3").arg(QString{fullDebugArea}.leftJustified(25, ' ', true)); + output += QString(" %3").arg(fullDebugArea.leftJustified(25, ' ', true)); } if (showLocation) { const auto filename = QString::fromLatin1(file).split('/').last(); diff --git a/common/log.h b/common/log.h index be5a508..8266fdb 100644 --- a/common/log.h +++ b/common/log.h @@ -85,12 +85,15 @@ SINK_EXPORT inline QDebug operator<<(QDebug d, const TraceTime &time) d << time.time << "[ms]"; return d; } + +SINK_EXPORT bool isFiltered(DebugLevel debugLevel, const char *debugArea, const char *debugComponent, const char *file); + } } static const char *getComponentName() { return nullptr; } -#define SINK_DEBUG_STREAM_IMPL(LEVEL, AREA, COMPONENT) Sink::Log::debugStream(LEVEL, __LINE__, __FILE__, Q_FUNC_INFO, AREA, COMPONENT) +#define SINK_DEBUG_STREAM_IMPL(LEVEL, AREA, COMPONENT) if (!Sink::Log::isFiltered(LEVEL, AREA, COMPONENT, __FILE__)) Sink::Log::debugStream(LEVEL, __LINE__, __FILE__, Q_FUNC_INFO, AREA, COMPONENT) #define Trace_area(AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Trace, AREA, nullptr) #define Log_area(AREA) SINK_DEBUG_STREAM_IMPL(Sink::Log::DebugLevel::Log, AREA, nullptr) -- cgit v1.2.3 From 256fe3fc561f1690e5c29640b9081e805ceb5532 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 12 May 2017 18:57:07 +0200 Subject: Don't commit after every replayed revision If we didn't actually do anything we just carry on. Failing to commit is harmless in that case and committing for every revision is rather expensive. --- common/changereplay.cpp | 36 ++++++++++++++++++------------------ common/changereplay.h | 1 + common/synchronizer.h | 2 +- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/common/changereplay.cpp b/common/changereplay.cpp index da36b3e..75b9a4c 100644 --- a/common/changereplay.cpp +++ b/common/changereplay.cpp @@ -105,7 +105,7 @@ KAsync::Job ChangeReplay::replayNextRevision() } Q_ASSERT(mMainStoreTransaction); - KAsync::Job replayJob = KAsync::null(); + auto replayJob = KAsync::null(); qint64 revision = *lastReplayedRevision + 1; while (revision <= *topRevision) { const auto uid = DataStore::getUidFromRevision(mMainStoreTransaction, revision); @@ -129,14 +129,14 @@ KAsync::Job ChangeReplay::replayNextRevision() if (canReplay(type, key, entityBuffer)) { SinkTraceCtx(mLogCtx) << "Replaying " << key; replayJob = replay(type, key, entityBuffer); + //Set the last revision we tried to replay + *lastReplayedRevision = revision; + //Execute replay job and commit + break; } else { - SinkTraceCtx(mLogCtx) << "Cannot replay " << key; + SinkTraceCtx(mLogCtx) << "Not replaying " << key; //We silently skip over revisions that cannot be replayed, as this is not an error. - replayJob = KAsync::null(); } - //Set the last revision we tried to replay - *lastReplayedRevision = revision; - break; } } //Bump the revision if we failed to even attempt to replay. This will simply skip over those revisions, as we can't recover from those situations. @@ -148,20 +148,20 @@ KAsync::Job ChangeReplay::replayNextRevision() SinkWarningCtx(mLogCtx) << "Change replay failed: " << error << "Last replayed revision: " << *lastReplayedRevision; //We're probably not online or so, so postpone retrying return KAsync::value(KAsync::Break); + } + SinkTraceCtx(mLogCtx) << "Replayed until: " << *lastReplayedRevision; + + recordReplayedRevision(*lastReplayedRevision); + reportProgress(*lastReplayedRevision, *topRevision); + + const bool gotMoreToReplay = (*lastReplayedRevision < *topRevision); + if (gotMoreToReplay) { + SinkTraceCtx(mLogCtx) << "Replaying some more..."; + //Replay more if we have more + return KAsync::wait(0).then(KAsync::value(KAsync::Continue)); } else { - SinkTraceCtx(mLogCtx) << "Replayed until: " << *lastReplayedRevision; - recordReplayedRevision(*lastReplayedRevision); - if (*lastReplayedRevision < *topRevision) { - SinkTraceCtx(mLogCtx) << "Replaying some more..."; - //Replay more if we have more - return KAsync::wait(0).then(KAsync::value(KAsync::Continue)); - } else { - return KAsync::value(KAsync::Break); - } + return KAsync::value(KAsync::Break); } - //We shouldn't ever get here - Q_ASSERT(false); - return KAsync::value(KAsync::Break); }).guard(&mGuard); }); }) diff --git a/common/changereplay.h b/common/changereplay.h index ab2d857..c509735 100644 --- a/common/changereplay.h +++ b/common/changereplay.h @@ -54,6 +54,7 @@ public slots: protected: virtual KAsync::Job replay(const QByteArray &type, const QByteArray &key, const QByteArray &value) = 0; virtual bool canReplay(const QByteArray &type, const QByteArray &key, const QByteArray &value) = 0; + virtual void reportProgress(int progress, int total){}; Sink::Storage::DataStore mStorage; KAsync::Job replayNextRevision(); diff --git a/common/synchronizer.h b/common/synchronizer.h index bb24c2b..935c139 100644 --- a/common/synchronizer.h +++ b/common/synchronizer.h @@ -197,7 +197,7 @@ protected: /** * Report progress for current task */ - void reportProgress(int progress, int total); + virtual void reportProgress(int progress, int total) Q_DECL_OVERRIDE; protected: Sink::Log::Context mLogCtx; -- cgit v1.2.3 From 524e405f645edb6231f9b16fafc1f9ca36af8237 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 20 May 2017 11:28:02 +0200 Subject: Avoid notifcations for requests that do nothing, progress with folderid --- common/changereplay.h | 2 +- common/synchronizer.cpp | 19 +++++++++++-------- common/synchronizer.h | 2 +- examples/imapresource/imapresource.cpp | 21 +++++++++++---------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/common/changereplay.h b/common/changereplay.h index c509735..22e26a5 100644 --- a/common/changereplay.h +++ b/common/changereplay.h @@ -54,7 +54,7 @@ public slots: protected: virtual KAsync::Job replay(const QByteArray &type, const QByteArray &key, const QByteArray &value) = 0; virtual bool canReplay(const QByteArray &type, const QByteArray &key, const QByteArray &value) = 0; - virtual void reportProgress(int progress, int total){}; + virtual void reportProgress(int progress, int total, const QByteArrayList &applicableEntities = {}){}; Sink::Storage::DataStore mStorage; KAsync::Job replayNextRevision(); diff --git a/common/synchronizer.cpp b/common/synchronizer.cpp index b9decbd..3ef7eb7 100644 --- a/common/synchronizer.cpp +++ b/common/synchronizer.cpp @@ -315,11 +315,17 @@ void Synchronizer::emitProgressNotification(Notification::NoticationType type, i emit notify(n); } -void Synchronizer::reportProgress(int progress, int total) +void Synchronizer::reportProgress(int progress, int total, const QByteArrayList &entities) { if (progress > 0 && total > 0) { - SinkLogCtx(mLogCtx) << "Progress: " << progress << " out of " << total; - emitProgressNotification(Notification::Progress, progress, total, mCurrentRequest.requestId, mCurrentRequest.applicableEntities); + SinkLogCtx(mLogCtx) << "Progress: " << progress << " out of " << total << mCurrentRequest.requestId << mCurrentRequest.applicableEntities; + const auto applicableEntities = [&] { + if (entities.isEmpty()) { + return mCurrentRequest.applicableEntities; + } + return entities; + }(); + emitProgressNotification(Notification::Progress, progress, total, mCurrentRequest.requestId, applicableEntities); } } @@ -371,6 +377,7 @@ KAsync::Job Synchronizer::processRequest(const SyncRequest &request) } else if (request.requestType == Synchronizer::SyncRequest::Synchronization) { return KAsync::start([this, request] { SinkLogCtx(mLogCtx) << "Synchronizing: " << request.query; + setBusy(true, "Synchronization has started.", request.requestId); emitNotification(Notification::Info, ApplicationDomain::SyncInProgress, {}, {}, request.applicableEntities); }).then(synchronizeWithSource(request.query)).then([this] { //Commit after every request, so implementations only have to commit more if they add a lot of data. @@ -408,6 +415,7 @@ KAsync::Job Synchronizer::processRequest(const SyncRequest &request) return KAsync::null(); } else { return KAsync::start([this, request] { + setBusy(true, "ChangeReplay has started.", request.requestId); SinkLogCtx(mLogCtx) << "Replaying changes."; }) .then(replayNextRevision()) @@ -479,11 +487,6 @@ KAsync::Job Synchronizer::processSyncQueue() mMessageQueue->startTransaction(); mEntityStore->startTransaction(Sink::Storage::DataStore::ReadOnly); mSyncInProgress = true; - if (request.requestType == Synchronizer::SyncRequest::Synchronization) { - setBusy(true, "Synchronization has started.", request.requestId); - } else if (request.requestType == Synchronizer::SyncRequest::ChangeReplay) { - setBusy(true, "ChangeReplay has started.", request.requestId); - } mCurrentRequest = request; }) .then(processRequest(request)) diff --git a/common/synchronizer.h b/common/synchronizer.h index 935c139..cc082be 100644 --- a/common/synchronizer.h +++ b/common/synchronizer.h @@ -197,7 +197,7 @@ protected: /** * Report progress for current task */ - virtual void reportProgress(int progress, int total) Q_DECL_OVERRIDE; + virtual void reportProgress(int progress, int total, const QByteArrayList &entities = {}) Q_DECL_OVERRIDE; protected: Sink::Log::Context mLogCtx; diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 94ca27a..533dea3 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -184,14 +184,12 @@ public: return flags; } - void synchronizeMails(const QByteArray &folderRid, const Message &message) + void synchronizeMails(const QByteArray &folderRid, const QByteArray &folderLocalId, const Message &message) { auto time = QSharedPointer::create(); time->start(); SinkTraceCtx(mLogCtx) << "Importing new mail." << folderRid; - const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRid); - const auto remoteId = assembleMailRid(folderLocalId, message.uid); Q_ASSERT(message.msg); @@ -315,14 +313,15 @@ public: SinkTraceCtx(mLogCtx) << "Uids to fetch: " << filteredAndSorted; bool headersOnly = false; + const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRemoteId); return imap->fetchMessages(folder, filteredAndSorted, headersOnly, [=](const Message &m) { if (*maxUid < m.uid) { *maxUid = m.uid; } - synchronizeMails(folderRemoteId, m); + synchronizeMails(folderRemoteId, folderLocalId, m); }, - [this, maxUid, folder](int progress, int total) { - reportProgress(progress, total); + [=](int progress, int total) { + reportProgress(progress, total, QByteArrayList{} << folderLocalId); //commit every 10 messages if ((progress % 10) == 0) { commit(); @@ -359,11 +358,12 @@ public: SinkLogCtx(mLogCtx) << "Fetching headers for: " << toFetch; bool headersOnly = true; + const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRemoteId); return imap->fetchMessages(folder, toFetch, headersOnly, [=](const Message &m) { - synchronizeMails(folderRemoteId, m); + synchronizeMails(folderRemoteId, folderLocalId, m); }, [=](int progress, int total) { - reportProgress(progress, total); + reportProgress(progress, total, QByteArrayList{} << folderLocalId); //commit every 100 messages if ((progress % 100) == 0) { commit(); @@ -567,11 +567,12 @@ public: } SinkLog() << "Fetching messages: " << toFetch << folderRemoteId; bool headersOnly = false; + const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRemoteId); return imap->fetchMessages(Folder{folderRemoteId}, toFetch, headersOnly, [=](const Message &m) { - synchronizeMails(folderRemoteId, m); + synchronizeMails(folderRemoteId, folderLocalId, m); }, [=](int progress, int total) { - reportProgress(progress, total); + reportProgress(progress, total, QByteArrayList{} << folderLocalId); //commit every 100 messages if ((progress % 100) == 0) { commit(); -- cgit v1.2.3 From 75d51b5ee87f3c1d34586c34f73c2100d09e1d04 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 20 May 2017 14:45:16 +0200 Subject: Reread the location if we enable the test mode --- common/definitions.cpp | 40 +++++++++++++++++++++++++++++++--------- common/definitions.h | 1 + common/test.cpp | 1 + 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/common/definitions.cpp b/common/definitions.cpp index 7f4fbbe..17977bc 100644 --- a/common/definitions.cpp +++ b/common/definitions.cpp @@ -23,6 +23,17 @@ #include #include +static bool rereadDataLocation = true; +static bool rereadConfigLocation = true; +static bool rereadTemporaryFileLocation = true; + +void Sink::clearLocationCache() +{ + rereadDataLocation = true; + rereadConfigLocation = true; + rereadTemporaryFileLocation = true; +} + QString Sink::storageLocation() { return dataLocation() + "/storage"; @@ -30,26 +41,37 @@ QString Sink::storageLocation() QString Sink::dataLocation() { - static auto location = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + "/sink"; + static QString location; + if (rereadDataLocation) { + location = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + "/sink"; + rereadDataLocation = false; + } return location; } QString Sink::configLocation() { - static auto location = QStandardPaths::writableLocation(QStandardPaths::GenericConfigLocation) + "/sink"; + static QString location; + if (rereadConfigLocation) { + location = QStandardPaths::writableLocation(QStandardPaths::GenericConfigLocation) + "/sink"; + rereadConfigLocation = false; + } return location; } QString Sink::temporaryFileLocation() { - static auto path = dataLocation() + "/temporaryFiles"; - static bool initialized = false; - if (!initialized) { - if (QDir{}.mkpath(path)) { - initialized = true; - } + static QString location; + static bool dirCreated = false; + if (rereadTemporaryFileLocation) { + location = dataLocation() + "/temporaryFiles"; + dirCreated = QDir{}.mkpath(location); + rereadTemporaryFileLocation = false; } - return path; + if (!dirCreated && QDir{}.mkpath(location)) { + dirCreated = true; + } + return location; } QString Sink::resourceStorageLocation(const QByteArray &resourceInstanceIdentifier) diff --git a/common/definitions.h b/common/definitions.h index e8cd45e..ce9e794 100644 --- a/common/definitions.h +++ b/common/definitions.h @@ -25,6 +25,7 @@ #include namespace Sink { +void SINK_EXPORT clearLocationCache(); QString SINK_EXPORT storageLocation(); QString SINK_EXPORT dataLocation(); QString SINK_EXPORT configLocation(); diff --git a/common/test.cpp b/common/test.cpp index f9bfa23..237d3bb 100644 --- a/common/test.cpp +++ b/common/test.cpp @@ -81,6 +81,7 @@ void Sink::Test::initTest() void Sink::Test::setTestModeEnabled(bool enabled) { QStandardPaths::setTestModeEnabled(enabled); + Sink::clearLocationCache(); if (enabled) { qputenv("SINK_TESTMODE", "TRUE"); } else { -- cgit v1.2.3 From f78a3ca8515fba5b8d9ad6f4d5bfda3fb83b46e7 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 20 May 2017 14:45:42 +0200 Subject: We don't need the layout just to check for existence --- common/storage/entitystore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 6ff700e..22e5ae3 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -113,7 +113,7 @@ public: bool exists() { - return Sink::Storage::DataStore(Sink::storageLocation(), dbLayout(resourceContext.instanceId()), DataStore::ReadOnly).exists(); + return Sink::Storage::DataStore(Sink::storageLocation(), resourceContext.instanceId(), DataStore::ReadOnly).exists(); } DataStore::Transaction &getTransaction() -- cgit v1.2.3 From cdca3e36eddea7f43787fa6bbc2ad6b669c89185 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 20 May 2017 16:53:53 +0200 Subject: Subscribe to mailboxes in imaptest --- examples/imapresource/tests/populatemailbox.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/imapresource/tests/populatemailbox.sh b/examples/imapresource/tests/populatemailbox.sh index a435df7..800e2e7 100644 --- a/examples/imapresource/tests/populatemailbox.sh +++ b/examples/imapresource/tests/populatemailbox.sh @@ -1,12 +1,23 @@ #!/bin/bash sudo echo "sam user.doe.* cyrus c" | cyradm --auth PLAIN -u cyrus -w admin localhost +#Delete all mailboxes sudo echo "dm user.doe.*" | cyradm --auth PLAIN -u cyrus -w admin localhost +#Create mailboxes sudo echo "cm user.doe.test" | cyradm --auth PLAIN -u cyrus -w admin localhost sudo echo "cm user.doe.Drafts" | cyradm --auth PLAIN -u cyrus -w admin localhost sudo echo "cm user.doe.Trash" | cyradm --auth PLAIN -u cyrus -w admin localhost + +#Set acls so we can create in INBOX sudo echo "sam user.doe cyrus c" | cyradm --auth PLAIN -u cyrus -w admin localhost +#Subscribe to mailboxes +sudo echo "sub INBOX" | cyradm --auth PLAIN -u doe -w doe localhost +sudo echo "sub INBOX.test" | cyradm --auth PLAIN -u doe -w doe localhost +sudo echo "sub INBOX.Drafts" | cyradm --auth PLAIN -u doe -w doe localhost +sudo echo "sub INBOX.Trash" | cyradm --auth PLAIN -u doe -w doe localhost + +#Create a bunch of test messages in the test folder # for i in `seq 1 5000`; # do # # sudo cp /work/source/Sink/examples/imapresource/tests/data/1365777830.R28.localhost.localdomain\:2\,S /var/spool/imap/d/user/doe/test/$i. -- cgit v1.2.3 From d67b886dd2cf51264e4d6da680e268015a0a7031 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 20 May 2017 16:54:34 +0200 Subject: Don't try to replay modifications on nothing. --- examples/imapresource/imapresource.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 533dea3..bfe43bf 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -689,6 +689,12 @@ public: KAsync::Job replay(const ApplicationDomain::Folder &folder, Sink::Operation operation, const QByteArray &oldRemoteId, const QList &changedProperties) Q_DECL_OVERRIDE { + if (operation != Sink::Operation_Creation) { + if(oldRemoteId.isEmpty()) { + Q_ASSERT(false); + return KAsync::error("Tried to replay modification without old remoteId."); + } + } auto imap = QSharedPointer::create(mServer, mPort, &mSessionCache); auto login = imap->login(mUser, mPassword); if (operation == Sink::Operation_Creation) { -- cgit v1.2.3 From 227610b7399905fca38e0f09053d78e2e866f32e Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 20 May 2017 16:55:21 +0200 Subject: Ensure change-replay errors make it through to the correct error handling and are appropriately dealt with. --- common/changereplay.cpp | 7 ++++++- common/synchronizer.cpp | 7 +++++-- examples/imapresource/imapresource.cpp | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/common/changereplay.cpp b/common/changereplay.cpp index 75b9a4c..0adbd78 100644 --- a/common/changereplay.cpp +++ b/common/changereplay.cpp @@ -147,7 +147,7 @@ KAsync::Job ChangeReplay::replayNextRevision() if (error) { SinkWarningCtx(mLogCtx) << "Change replay failed: " << error << "Last replayed revision: " << *lastReplayedRevision; //We're probably not online or so, so postpone retrying - return KAsync::value(KAsync::Break); + return KAsync::value(KAsync::Break).then(KAsync::error(error)); } SinkTraceCtx(mLogCtx) << "Replayed until: " << *lastReplayedRevision; @@ -179,6 +179,11 @@ KAsync::Job ChangeReplay::replayNextRevision() emit changesReplayed(); } } + if (error) { + return KAsync::error(error); + } else { + return KAsync::null(); + } }).guard(&mGuard); } diff --git a/common/synchronizer.cpp b/common/synchronizer.cpp index 3ef7eb7..b1ff29c 100644 --- a/common/synchronizer.cpp +++ b/common/synchronizer.cpp @@ -629,11 +629,14 @@ KAsync::Job Synchronizer::replay(const QByteArray &type, const QByteArray } }) .then([this](const KAsync::Error &error) { + //We need to commit here otherwise the next change-replay step will abort the transaction + mSyncStore.clear(); + mSyncTransaction.commit(); if (error) { SinkWarningCtx(mLogCtx) << "Failed to replay change: " << error.errorMessage; + return KAsync::error(error); } - mSyncStore.clear(); - mSyncTransaction.commit(); + return KAsync::null(); }); } diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index bfe43bf..29ff03b 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -680,7 +680,7 @@ public: if (error) { SinkWarning() << "Error during changereplay: " << error.errorMessage; return imap->logout() - .then(KAsync::error(error)); + .then(KAsync::error(getError(error))); } return imap->logout() .then(KAsync::value(remoteId)); -- cgit v1.2.3 From 83eb22fded7a475e5640ff415f9de360880a567c Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 20 May 2017 16:56:18 +0200 Subject: no need to hardcode this --- common/synchronizer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/synchronizer.cpp b/common/synchronizer.cpp index b1ff29c..3b32e68 100644 --- a/common/synchronizer.cpp +++ b/common/synchronizer.cpp @@ -420,7 +420,7 @@ KAsync::Job Synchronizer::processRequest(const SyncRequest &request) }) .then(replayNextRevision()) .then([this, request](const KAsync::Error &error) { - setStatusFromResult(error, "Changereplay has ended.", "changereplay"); + setStatusFromResult(error, "Changereplay has ended.", request.requestId); if (error) { SinkWarningCtx(mLogCtx) << "Changereplay failed: " << error.errorMessage; return KAsync::error(error); -- cgit v1.2.3 From 18bd12302c39345f1a1a3fa29ddfd3e6ac558c24 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 6 Jun 2017 11:25:50 +0200 Subject: Specfile cleanup --- dist/sink.spec | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dist/sink.spec b/dist/sink.spec index dbe1007..1522890 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -38,11 +38,6 @@ Development headers for sink %prep %setup -q -sed -i \ - -e '/inspectiontest/d' \ - -e '/maildirresourcetest/d' \ - tests/CMakeLists.txt - %build mkdir -p build/ pushd build @@ -59,7 +54,6 @@ pushd build %make_install popd -#rm -rf %{buildroot}%{_prefix}/mkspecs/modules/qt_KMime.pri rm %{buildroot}%{_prefix}/bin/resetmailbox.sh rm %{buildroot}%{_prefix}/bin/populatemailbox.sh rm %{buildroot}%{_prefix}/bin/sink_smtp_test -- cgit v1.2.3 From be98090bbbda26825ea44777c3d847399b2e6e5c Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 6 Jun 2017 16:34:50 +0200 Subject: A stab at the undefined reference to QMap::QMap() error ...that I only get on copr and nowhere else. --- common/storage.h | 2 ++ common/storage_common.cpp | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/common/storage.h b/common/storage.h index b664e11..8c129df 100644 --- a/common/storage.h +++ b/common/storage.h @@ -32,6 +32,8 @@ namespace Storage { struct SINK_EXPORT DbLayout { typedef QMap Databases; + DbLayout(); + DbLayout(const QByteArray &, const Databases &); QByteArray name; Databases tables; }; diff --git a/common/storage_common.cpp b/common/storage_common.cpp index 1426cd5..8603787 100644 --- a/common/storage_common.cpp +++ b/common/storage_common.cpp @@ -36,6 +36,18 @@ namespace Storage { static const char *s_internalPrefix = "__internal"; static const int s_internalPrefixSize = strlen(s_internalPrefix); +DbLayout::DbLayout() +{ + +} + +DbLayout::DbLayout(const QByteArray &n, const Databases &t) + : name(n), + tables(t) +{ + +} + void errorHandler(const DataStore::Error &error) { if (error.code == DataStore::TransactionError) { -- cgit v1.2.3 From f254153ac1571d5ab31a4f17fba2db9256bc24d3 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 6 Jun 2017 16:39:12 +0200 Subject: Link to qt::core as well --- synchronizer/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synchronizer/CMakeLists.txt b/synchronizer/CMakeLists.txt index ff9ec86..e049d64 100644 --- a/synchronizer/CMakeLists.txt +++ b/synchronizer/CMakeLists.txt @@ -8,5 +8,5 @@ set(sinksynchronizer_SRCS add_executable(${PROJECT_NAME} ${sinksynchronizer_SRCS}) target_link_libraries(${PROJECT_NAME} sink KAsync ${CMAKE_DL_LIBS}) -qt5_use_modules(${PROJECT_NAME} Network) +qt5_use_modules(${PROJECT_NAME} Core Network) install(TARGETS ${PROJECT_NAME} ${KDE_INSTALL_TARGETS_DEFAULT_ARGS}) -- cgit v1.2.3 From e382924f1a90b5a27eba2e8c5981f6a4fe7892c9 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 10 Jun 2017 09:37:08 +0200 Subject: Fixed incremental queries The incremental querying broke as soon as a revision update came in since it would nuke the base-set. This fixes it, but it's definitely not pretty. --- common/datastorequery.cpp | 41 +++++++++++++++++++++++++++++++---------- common/datastorequery.h | 1 + common/queryrunner.cpp | 6 +++--- tests/querytest.cpp | 27 ++++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 14 deletions(-) diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp index 2e0c348..4c95606 100644 --- a/common/datastorequery.cpp +++ b/common/datastorequery.cpp @@ -43,6 +43,8 @@ class Source : public FilterBase { QVector mIds; QVector::ConstIterator mIt; + QVector mIncrementalIds; + QVector::ConstIterator mIncrementalIt; Source (const QVector &ids, DataStoreQuery *store) : FilterBase(store), @@ -63,21 +65,36 @@ class Source : public FilterBase { void add(const QVector &ids) { - mIds = ids; - mIt = mIds.constBegin(); + mIncrementalIds = ids; + mIncrementalIt = mIncrementalIds.constBegin(); } bool next(const std::function &callback) Q_DECL_OVERRIDE { - if (mIt == mIds.constEnd()) { - return false; + if (!mIncrementalIds.isEmpty()) { + if (mIncrementalIt == mIncrementalIds.constEnd()) { + return false; + } + readEntity(*mIncrementalIt, [this, callback](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { + SinkTraceCtx(mDatastore->mLogCtx) << "Source: Read entity: " << entity.identifier() << operationName(operation); + callback({entity, operation}); + }); + mIncrementalIt++; + if (mIncrementalIt == mIncrementalIds.constEnd()) { + return false; + } + return true; + } else { + if (mIt == mIds.constEnd()) { + return false; + } + readEntity(*mIt, [this, callback](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { + SinkTraceCtx(mDatastore->mLogCtx) << "Source: Read entity: " << entity.identifier() << operationName(operation); + callback({entity, operation}); + }); + mIt++; + return mIt != mIds.constEnd(); } - readEntity(*mIt, [this, callback](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { - SinkTraceCtx(mDatastore->mLogCtx) << "Source: Read entity: " << entity.identifier() << operationName(operation); - callback({entity, operation}); - }); - mIt++; - return mIt != mIds.constEnd(); } }; @@ -599,6 +616,10 @@ ResultSet DataStoreQuery::update(qint64 baseRevision) return ResultSet(generator, [this]() { mCollector->skip(); }); } +void DataStoreQuery::updateComplete() +{ + mSource->mIncrementalIds.clear(); +} ResultSet DataStoreQuery::execute() { diff --git a/common/datastorequery.h b/common/datastorequery.h index ee5f99e..de4ae26 100644 --- a/common/datastorequery.h +++ b/common/datastorequery.h @@ -50,6 +50,7 @@ public: ~DataStoreQuery(); ResultSet execute(); ResultSet update(qint64 baseRevision); + void updateComplete(); State::Ptr getState(); diff --git a/common/queryrunner.cpp b/common/queryrunner.cpp index 43f48c0..f196965 100644 --- a/common/queryrunner.cpp +++ b/common/queryrunner.cpp @@ -231,11 +231,11 @@ ReplayResult QueryWorker::executeIncrementalQuery(const Sink::Query auto replayResult = resultSet.replaySet(0, 0, [this, query, &resultProvider](const ResultSet::Result &result) { resultProviderCallback(query, resultProvider, result); }); - + preparedQuery.updateComplete(); SinkTraceCtx(mLogCtx) << "Replayed " << replayResult.replayedEntities << " results.\n" << (replayResult.replayedAll ? "Replayed all available results.\n" : "") << "Incremental query took: " << Log::TraceTime(time.elapsed()); - return {entityStore.maxRevision(), replayResult.replayedEntities, replayResult.replayedAll, preparedQuery.getState()}; + return {entityStore.maxRevision(), replayResult.replayedEntities, false, preparedQuery.getState()}; } template @@ -264,7 +264,7 @@ ReplayResult QueryWorker::executeInitialQuery( return DataStoreQuery{modifiedQuery, ApplicationDomain::getTypeName(), entityStore}; } }(); - auto resultSet = preparedQuery.execute();; + auto resultSet = preparedQuery.execute(); SinkTraceCtx(mLogCtx) << "Filtered set retrieved." << Log::TraceTime(time.elapsed()); auto replayResult = resultSet.replaySet(0, batchsize, [this, query, &resultProvider](const ResultSet::Result &result) { diff --git a/tests/querytest.cpp b/tests/querytest.cpp index 4ff1be8..714e549 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -381,6 +381,7 @@ private slots: { // Setup Folder::Ptr folderEntity; + const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0)); { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); @@ -398,7 +399,6 @@ private slots: folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); - const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0)); { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testSecond"); @@ -428,6 +428,11 @@ private slots: query.filter(*folderEntity); query.sort(); query.limit(1); + query.setFlags(Query::LiveQuery); + query.reduce(Query::Reduce::Selector::max()) + .count("count") + .collect("unreadCollected") + .collect("importantCollected"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); @@ -443,6 +448,26 @@ private slots: QCOMPARE(model->rowCount(), 2); // We can't make any assumptions about the order of the indexes // QCOMPARE(model->index(1, 0).data(Sink::Store::DomainObjectRole).value()->getProperty("messageId").toByteArray(), QByteArray("testSecond")); + + //New revisions always go through + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedMessageId("testInjected"); + mail.setFolder(folderEntity->identifier()); + mail.setExtractedDate(date.addDays(-2)); + Sink::Store::create(mail).exec().waitForFinished(); + } + QTRY_COMPARE(model->rowCount(), 3); + + //Ensure we can continue fetching after the incremental update + model->fetchMore(QModelIndex()); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 4); + + //Ensure we have fetched all + model->fetchMore(QModelIndex()); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 4); } void testReactToNewResource() -- cgit v1.2.3 From e663b740394c316dd53aa566d62931bce0b47150 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 10 Jun 2017 09:43:07 +0200 Subject: Bumped release version --- dist/sink.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/sink.spec b/dist/sink.spec index 1522890..f705069 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 5%{?dist} +Release: 6%{?dist} Summary: sink Group: Applications/Desktop -- cgit v1.2.3 From e97fb6d6de7c9bda6e8f911c5f471f4059ae7470 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sun, 11 Jun 2017 22:55:50 +0200 Subject: Disabled automatic syncing of folders I triggers a lot of work, and as we currently can't abort sync tasks and have no priority lane for requests it's rather intrusive. A sync will still be triggered when a folder is selected, so we're not loosing a lot except the "pull everything offline" case. --- examples/imapresource/imapresource.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 29ff03b..a6c0343 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -415,8 +415,9 @@ public: list << Synchronizer::SyncRequest{query}; } else { list << Synchronizer::SyncRequest{Sink::QueryBase(ApplicationDomain::getTypeName())}; + //TODO automatic syncing off all folders is disabled for the time being //This request depends on the previous one so we flush first. - list << Synchronizer::SyncRequest{applyMailDefaults(Sink::QueryBase(ApplicationDomain::getTypeName())), QByteArray{}, Synchronizer::SyncRequest::RequestFlush}; + //list << Synchronizer::SyncRequest{applyMailDefaults(Sink::QueryBase(ApplicationDomain::getTypeName())), QByteArray{}, Synchronizer::SyncRequest::RequestFlush}; } return list; } -- cgit v1.2.3 From d1f6d879ce29f732e32c0aa7bb2e40f187d9d8d3 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sun, 11 Jun 2017 23:01:45 +0200 Subject: Bumped release --- dist/sink.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/sink.spec b/dist/sink.spec index f705069..59e8052 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 6%{?dist} +Release: 7%{?dist} Summary: sink Group: Applications/Desktop -- cgit v1.2.3 From e179ec896bb5b55e33419da8bea2e891517697f6 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 12 Jun 2017 11:48:34 +0200 Subject: Avoid the extra parsing step. We only need the content, we'll parse later on when processing the pipeline. --- dist/sink.spec | 2 +- examples/imapresource/CMakeLists.txt | 2 +- examples/imapresource/imapresource.cpp | 2 +- examples/imapresource/imapserverproxy.cpp | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dist/sink.spec b/dist/sink.spec index 59e8052..b39fdc4 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 7%{?dist} +Release: 8%{?dist} Summary: sink Group: Applications/Desktop diff --git a/examples/imapresource/CMakeLists.txt b/examples/imapresource/CMakeLists.txt index 15ff135..5d2d38b 100644 --- a/examples/imapresource/CMakeLists.txt +++ b/examples/imapresource/CMakeLists.txt @@ -4,7 +4,7 @@ add_definitions(-DQT_PLUGIN) include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) find_package(KF5 COMPONENTS REQUIRED Mime) -find_package(KIMAP2 0.1.1 REQUIRED) +find_package(KIMAP2 0.2 REQUIRED) include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index a6c0343..5a9c36f 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -193,7 +193,7 @@ public: const auto remoteId = assembleMailRid(folderLocalId, message.uid); Q_ASSERT(message.msg); - SinkTraceCtx(mLogCtx) << "Found a mail " << remoteId << message.msg->subject(true)->asUnicodeString() << message.flags; + SinkTraceCtx(mLogCtx) << "Found a mail " << remoteId << message.flags; auto mail = Sink::ApplicationDomain::Mail::create(mResourceInstanceIdentifier); mail.setFolder(folderLocalId); diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp index a52c5eb..538105c 100644 --- a/examples/imapresource/imapserverproxy.cpp +++ b/examples/imapresource/imapserverproxy.cpp @@ -314,6 +314,7 @@ KAsync::Job ImapServerProxy::fetch(const KIMAP2::ImapSet &set, KIMAP2::Fet fetch->setSequenceSet(set); fetch->setUidBased(true); fetch->setScope(scope); + fetch->setAvoidParsing(true); QObject::connect(fetch, &KIMAP2::FetchJob::resultReceived, callback); return runJob(fetch); } -- cgit v1.2.3 From 7dc97cc6e338e1d756734e1620a062cdb08635ca Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 12 Jun 2017 16:09:29 +0200 Subject: There shouldn't be any conversions necessary at this point. --- examples/imapresource/imapresource.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 5a9c36f..945962f 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -615,10 +615,10 @@ public: auto login = imap->login(mUser, mPassword); KAsync::Job job = KAsync::null(); if (operation == Sink::Operation_Creation) { - QString mailbox = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, mail.getFolder()); - QByteArray content = KMime::LFtoCRLF(mail.getMimeMessage()); - auto flags = getFlags(mail); - QDateTime internalDate = mail.getDate(); + const QString mailbox = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, mail.getFolder()); + const QByteArray content = mail.getMimeMessage(); + const auto flags = getFlags(mail); + const QDateTime internalDate = mail.getDate(); job = login.then(imap->append(mailbox, content, flags, internalDate)) .addToContext(imap) .then([mail](qint64 uid) { @@ -649,11 +649,11 @@ public: const bool messageMoved = changedProperties.contains(ApplicationDomain::Mail::Folder::name); const bool messageChanged = changedProperties.contains(ApplicationDomain::Mail::MimeMessage::name); if (messageChanged || messageMoved) { - SinkTrace() << "Replacing message."; const auto folderId = folderIdFromMailRid(oldRemoteId); const QString oldMailbox = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, folderId); - QByteArray content = KMime::LFtoCRLF(mail.getMimeMessage()); - QDateTime internalDate = mail.getDate(); + const QByteArray content = mail.getMimeMessage(); + const QDateTime internalDate = mail.getDate(); + SinkTrace() << "Replacing message. Old mailbox: " << oldMailbox << "New mailbox: " << mailbox << "Flags: " << flags << "Content: " << content; KIMAP2::ImapSet set; set.add(uid); job = login.then(imap->append(mailbox, content, flags, internalDate)) -- cgit v1.2.3 From ae1c5a0a53d1fd351b6fd33e8a46ad1034874489 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 14 Jun 2017 12:39:29 +0200 Subject: Deal with both CRLF and LF mime messages. IMAP always requires CRLF, and so does the MIME standard, KMIME expects LF-only. We now just try to always use CRLF on disk, but convert LF-only messages should we have to (e.g. because copied over from maildir or so). --- examples/imapresource/imapresource.cpp | 15 ++++++++++++--- .../mailtransportresource/tests/mailtransporttest.cpp | 4 ++-- tests/interresourcemovetest.cpp | 2 +- tests/mailsynctest.cpp | 2 +- tests/mailtest.cpp | 18 +++++++++--------- tests/mailthreadtest.cpp | 6 +++--- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 945962f..25d9534 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -197,7 +197,7 @@ public: auto mail = Sink::ApplicationDomain::Mail::create(mResourceInstanceIdentifier); mail.setFolder(folderLocalId); - mail.setMimeMessage(message.msg->encodedContent()); + mail.setMimeMessage(message.msg->encodedContent(true)); mail.setExtractedFullPayloadAvailable(message.fullPayload); setFlags(mail, message.flags); @@ -608,6 +608,15 @@ public: } return KAsync::error("Nothing to do"); } + static QByteArray ensureCRLF(const QByteArray &data) { + auto index = data.indexOf('\n'); + if (index > 0 && data.at(index - 1) == '\r') { //First line is LF-only terminated + //Convert back and forth in case there's a mix. We don't want to expand CRLF into CRCRLF. + return KMime::LFtoCRLF(KMime::CRLFtoLF(data)); + } else { + return data; + } + } KAsync::Job replay(const ApplicationDomain::Mail &mail, Sink::Operation operation, const QByteArray &oldRemoteId, const QList &changedProperties) Q_DECL_OVERRIDE { @@ -616,7 +625,7 @@ public: KAsync::Job job = KAsync::null(); if (operation == Sink::Operation_Creation) { const QString mailbox = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, mail.getFolder()); - const QByteArray content = mail.getMimeMessage(); + const auto content = ensureCRLF(mail.getMimeMessage()); const auto flags = getFlags(mail); const QDateTime internalDate = mail.getDate(); job = login.then(imap->append(mailbox, content, flags, internalDate)) @@ -651,7 +660,7 @@ public: if (messageChanged || messageMoved) { const auto folderId = folderIdFromMailRid(oldRemoteId); const QString oldMailbox = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, folderId); - const QByteArray content = mail.getMimeMessage(); + const auto content = ensureCRLF(mail.getMimeMessage()); const QDateTime internalDate = mail.getDate(); SinkTrace() << "Replacing message. Old mailbox: " << oldMailbox << "New mailbox: " << mailbox << "Flags: " << flags << "Content: " << content; KIMAP2::ImapSet set; diff --git a/examples/mailtransportresource/tests/mailtransporttest.cpp b/examples/mailtransportresource/tests/mailtransporttest.cpp index e4cc447..2a831ed 100644 --- a/examples/mailtransportresource/tests/mailtransporttest.cpp +++ b/examples/mailtransportresource/tests/mailtransporttest.cpp @@ -64,7 +64,7 @@ private slots: message->assemble(); auto mail = ApplicationDomain::Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message->encodedContent()); + mail.setMimeMessage(message->encodedContent(true)); VERIFYEXEC(Store::create(mail)); VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier)); @@ -92,7 +92,7 @@ private slots: message->assemble(); auto mail = ApplicationDomain::Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message->encodedContent()); + mail.setMimeMessage(message->encodedContent(true)); VERIFYEXEC(Store::create(mail)); VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier)); diff --git a/tests/interresourcemovetest.cpp b/tests/interresourcemovetest.cpp index 3ac6ad4..5438bc7 100644 --- a/tests/interresourcemovetest.cpp +++ b/tests/interresourcemovetest.cpp @@ -48,7 +48,7 @@ class InterResourceMoveTest : public QObject m.subject(true)->fromUnicodeString(subject, "utf8"); m.messageID(true)->setIdentifier(uid); m.assemble(); - return m.encodedContent(); + return m.encodedContent(true); } private slots: diff --git a/tests/mailsynctest.cpp b/tests/mailsynctest.cpp index 5d51fbc..27b29a5 100644 --- a/tests/mailsynctest.cpp +++ b/tests/mailsynctest.cpp @@ -275,7 +275,7 @@ void MailSyncTest::testListMails() QVERIFY(!data.isEmpty()); KMime::Message m; - m.setContent(data); + m.setContent(KMime::CRLFtoLF(data)); m.parse(); QCOMPARE(mail->getSubject(), m.subject(true)->asUnicodeString()); QVERIFY(!mail->getFolder().isEmpty()); diff --git a/tests/mailtest.cpp b/tests/mailtest.cpp index e1361e4..cbb56d5 100644 --- a/tests/mailtest.cpp +++ b/tests/mailtest.cpp @@ -152,7 +152,7 @@ void MailTest::testCreateModifyDeleteMail() message->assemble(); auto mail = Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message->encodedContent()); + mail.setMimeMessage(message->encodedContent(true)); mail.setFolder(folder); VERIFYEXEC(Store::create(mail)); @@ -166,7 +166,7 @@ void MailTest::testCreateModifyDeleteMail() QCOMPARE(mail.getFolder(), folder.identifier()); QVERIFY(QFile(mail.getMimeMessagePath()).exists()); KMime::Message m; - m.setContent(mail.getMimeMessage()); + m.setContent(KMime::CRLFtoLF(mail.getMimeMessage())); m.parse(); QCOMPARE(m.subject(true)->asUnicodeString(), subject); }); @@ -182,7 +182,7 @@ void MailTest::testCreateModifyDeleteMail() auto message2 = KMime::Message::Ptr::create(); message2->subject(true)->fromUnicodeString(subject2, "utf8"); message2->assemble(); - mail.setMimeMessage(message2->encodedContent()); + mail.setMimeMessage(message2->encodedContent(true)); VERIFYEXEC(Store::modify(mail)); VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier)); @@ -195,7 +195,7 @@ void MailTest::testCreateModifyDeleteMail() QCOMPARE(mail.getFolder(), folder.identifier()); QVERIFY(QFile(mail.getMimeMessagePath()).exists()); KMime::Message m; - m.setContent(mail.getMimeMessage()); + m.setContent(KMime::CRLFtoLF(mail.getMimeMessage())); m.parse(); QCOMPARE(m.subject(true)->asUnicodeString(), subject2); }); @@ -237,7 +237,7 @@ void MailTest::testMoveMail() message->assemble(); auto mail = Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message->encodedContent()); + mail.setMimeMessage(message->encodedContent(true)); mail.setFolder(folder); VERIFYEXEC(Store::create(mail)); @@ -289,7 +289,7 @@ void MailTest::testMarkMailAsRead() message->assemble(); auto mail = Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message->encodedContent()); + mail.setMimeMessage(message->encodedContent(true)); mail.setFolder(folder); mail.setUnread(true); VERIFYEXEC(Store::create(mail)); @@ -341,7 +341,7 @@ void MailTest::testCreateDraft() message->assemble(); auto mail = ApplicationDomain::Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message->encodedContent()); + mail.setMimeMessage(message->encodedContent(true)); mail.setDraft(true); VERIFYEXEC(Store::create(mail)); @@ -391,7 +391,7 @@ void MailTest::testModifyMailToDraft() message->assemble(); auto mail = ApplicationDomain::Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message->encodedContent()); + mail.setMimeMessage(message->encodedContent(true)); mail.setDraft(false); mail.setFolder(folder); @@ -438,7 +438,7 @@ void MailTest::testModifyMailToTrash() message->assemble(); auto mail = ApplicationDomain::Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message->encodedContent()); + mail.setMimeMessage(message->encodedContent(true)); mail.setTrash(false); mail.setFolder(folder); diff --git a/tests/mailthreadtest.cpp b/tests/mailthreadtest.cpp index 02b29f6..741eb78 100644 --- a/tests/mailthreadtest.cpp +++ b/tests/mailthreadtest.cpp @@ -117,7 +117,7 @@ void MailThreadTest::testIndexInMixedOrder() { auto mail = Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message3->encodedContent()); + mail.setMimeMessage(message3->encodedContent(true)); mail.setFolder(folder); VERIFYEXEC(Store::create(mail)); } @@ -140,7 +140,7 @@ void MailThreadTest::testIndexInMixedOrder() { auto mail = Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message2->encodedContent()); + mail.setMimeMessage(message2->encodedContent(true)); mail.setFolder(folder); VERIFYEXEC(Store::create(mail)); } @@ -156,7 +156,7 @@ void MailThreadTest::testIndexInMixedOrder() { auto mail = Mail::create(mResourceInstanceIdentifier); - mail.setMimeMessage(message1->encodedContent()); + mail.setMimeMessage(message1->encodedContent(true)); mail.setFolder(folder); VERIFYEXEC(Store::create(mail)); } -- cgit v1.2.3 From c6422cbbb24542a48e26b5f2448def3fcfc8694d Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 14 Jun 2017 13:23:48 +0200 Subject: We now have to manually parse for inspections ..since we turn of parsing for regular fetching. --- examples/imapresource/imapresource.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 25d9534..9fb83ab 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -834,6 +834,10 @@ protected: .then(imap->select(folderRemoteId)) .then([](Imap::SelectResult){}) .then(imap->fetch(set, scope, [imap, messageByUid](const Imap::Message &message) { + //We avoid parsing normally, so we have to do it explicitly here + if (message.msg) { + message.msg->parse(); + } messageByUid->insert(message.uid, message); })); -- cgit v1.2.3 From ead1720e78ca49837e1365e4b5dfb1b3f43632bc Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 14 Jun 2017 13:24:43 +0200 Subject: Use asyncverify where we can Otherwise the test is not aborted because the job doesn't get any error set --- tests/mailsynctest.cpp | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/tests/mailsynctest.cpp b/tests/mailsynctest.cpp index 27b29a5..75454c0 100644 --- a/tests/mailsynctest.cpp +++ b/tests/mailsynctest.cpp @@ -224,7 +224,8 @@ void MailSyncTest::testListNewSubFolder() for (const auto &folder : folders) { names << folder->getName(); } - QVERIFY(names.contains("sub1")); + ASYNCVERIFY(names.contains("sub1")); + return KAsync::null(); }); VERIFYEXEC(job); } @@ -252,7 +253,8 @@ void MailSyncTest::testListRemovedSubFolder() for (const auto &folder : folders) { names << folder->getName(); } - QVERIFY(!names.contains("sub1")); + ASYNCVERIFY(!names.contains("sub1")); + return KAsync::null(); }); VERIFYEXEC(job); } @@ -268,18 +270,19 @@ void MailSyncTest::testListMails() VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier)); auto job = Store::fetchAll(query).then([](const QList &mails) { - QCOMPARE(mails.size(), 1); + ASYNCCOMPARE(mails.size(), 1); auto mail = mails.first(); - QVERIFY(mail->getSubject().startsWith(QString("[Nepomuk] Jenkins build is still unstable"))); + ASYNCVERIFY(mail->getSubject().startsWith(QString("[Nepomuk] Jenkins build is still unstable"))); const auto data = mail->getMimeMessage(); - QVERIFY(!data.isEmpty()); + ASYNCVERIFY(!data.isEmpty()); KMime::Message m; m.setContent(KMime::CRLFtoLF(data)); m.parse(); - QCOMPARE(mail->getSubject(), m.subject(true)->asUnicodeString()); - QVERIFY(!mail->getFolder().isEmpty()); - QVERIFY(mail->getDate().isValid()); + ASYNCCOMPARE(mail->getSubject(), m.subject(true)->asUnicodeString()); + ASYNCVERIFY(!mail->getFolder().isEmpty()); + ASYNCVERIFY(mail->getDate().isValid()); + return KAsync::null(); }); VERIFYEXEC(job); } @@ -300,10 +303,11 @@ void MailSyncTest::testResyncMails() VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier)); auto job = Store::fetchAll(query).then([](const QList &mails) { - QCOMPARE(mails.size(), 1); + ASYNCCOMPARE(mails.size(), 1); auto mail = mails.first(); - QVERIFY(!mail->getSubject().isEmpty()); - QVERIFY(!mail->getMimeMessagePath().isEmpty()); + ASYNCVERIFY(!mail->getSubject().isEmpty()); + ASYNCVERIFY(!mail->getMimeMessagePath().isEmpty()); + return KAsync::null(); }); VERIFYEXEC(job); } @@ -328,7 +332,8 @@ void MailSyncTest::testFetchNewRemovedMessages() { auto job = Store::fetchAll(query).then([](const QList &mails) { - QCOMPARE(mails.size(), 2); + ASYNCCOMPARE(mails.size(), 2); + return KAsync::null(); }); VERIFYEXEC(job); } @@ -340,7 +345,8 @@ void MailSyncTest::testFetchNewRemovedMessages() { auto job = Store::fetchAll(query).then([](const QList &mails) { - QCOMPARE(mails.size(), 1); + ASYNCCOMPARE(mails.size(), 1); + return KAsync::null(); }); VERIFYEXEC(job); } @@ -363,7 +369,8 @@ void MailSyncTest::testFlagChange() { auto job = Store::fetchAll(query).then([](const QList &mails) { - QCOMPARE(mails.size(), 0); + ASYNCCOMPARE(mails.size(), 0); + return KAsync::null(); }); VERIFYEXEC(job); } @@ -376,8 +383,9 @@ void MailSyncTest::testFlagChange() { auto job = Store::fetchAll(query).then([](const QList &mails) { - QCOMPARE(mails.size(), 1); - QVERIFY(mails.first()->getImportant()); + ASYNCCOMPARE(mails.size(), 1); + ASYNCVERIFY(mails.first()->getImportant()); + return KAsync::null(); }); VERIFYEXEC(job); } @@ -392,8 +400,9 @@ void MailSyncTest::testSyncSingleFolder() Folder::Ptr folder; { auto job = Store::fetchAll(Sink::Query{}.resourceFilter(mResourceInstanceIdentifier).filter("test")).template then([&](const QList &folders) { - QCOMPARE(folders.size(), 1); + ASYNCCOMPARE(folders.size(), 1); folder = folders.first(); + return KAsync::null(); }); VERIFYEXEC(job); } @@ -417,11 +426,13 @@ void MailSyncTest::testSyncSingleMail() Mail::Ptr mail; { auto job = Store::fetchAll(Sink::Query{}.resourceFilter(mResourceInstanceIdentifier)).template then([&](const QList &mails) { - QVERIFY(mails.size() >= 1); + ASYNCVERIFY(mails.size() >= 1); mail = mails.first(); + return KAsync::null(); }); VERIFYEXEC(job); } + QVERIFY(mail); auto syncScope = Sink::SyncScope{ApplicationDomain::getTypeName()}; syncScope.resourceFilter(mResourceInstanceIdentifier); -- cgit v1.2.3 From 71ee6fc9b75bfb158ba507d9bd42b64d750029e0 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 14 Jun 2017 14:20:02 +0200 Subject: Cleanup --- examples/imapresource/imapresource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 9fb83ab..b28ecf1 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -584,7 +584,7 @@ public: bool syncHeaders = query.hasFilter(); //FIXME If we were able to to flush in between we could just query the local store for the folder list. return getFolderList(imap, query) - .serialEach([=](const Folder &folder) { + .serialEach([=](const Folder &folder) { SinkLog() << "Syncing folder " << folder.path(); //Emit notification that the folder is being synced. //The synchronizer can't do that because it has no concept of the folder filter on a mail sync scope meaning that the folder is being synchronized. -- cgit v1.2.3 From 78ff8574c3f481f8b0be248b153d55fe8c33eda5 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 14 Jun 2017 14:20:09 +0200 Subject: The mailsynctest relies on subscribed mailboxes --- examples/imapresource/tests/resetmailbox.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/imapresource/tests/resetmailbox.sh b/examples/imapresource/tests/resetmailbox.sh index 6ed198e..63d3478 100644 --- a/examples/imapresource/tests/resetmailbox.sh +++ b/examples/imapresource/tests/resetmailbox.sh @@ -3,8 +3,11 @@ sudo echo "sam user.doe.* cyrus c" | cyradm --auth PLAIN -u cyrus -w admin localhost sudo echo "dm user.doe.*" | cyradm --auth PLAIN -u cyrus -w admin localhost sudo echo "cm user.doe.test" | cyradm --auth PLAIN -u cyrus -w admin localhost +sudo echo "subscribe INBOX.test" | cyradm --auth PLAIN -u doe -w doe localhost sudo echo "cm user.doe.Drafts" | cyradm --auth PLAIN -u cyrus -w admin localhost +sudo echo "subscribe INBOX.Drafts" | cyradm --auth PLAIN -u doe -w doe localhost sudo echo "cm user.doe.Trash" | cyradm --auth PLAIN -u cyrus -w admin localhost +sudo echo "subscribe INBOX.Trash" | cyradm --auth PLAIN -u doe -w doe localhost sudo echo "sam user.doe cyrus c" | cyradm --auth PLAIN -u cyrus -w admin localhost sudo cp /work/source/Sink/examples/imapresource/tests/data/1365777830.R28.localhost.localdomain\:2\,S /var/spool/imap/d/user/doe/test/1. sudo chown cyrus:mail /var/spool/imap/d/user/doe/test/1. -- cgit v1.2.3 From 192cb00abb964a07df2403513f527598b6791df2 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 15 Jun 2017 10:06:28 +0200 Subject: No more threading by subject It seems to do more harm than good, creating huge threads, and the webclient doesn't do it either. --- common/mail/threadindexer.cpp | 74 ++----------------------------------------- 1 file changed, 2 insertions(+), 72 deletions(-) diff --git a/common/mail/threadindexer.cpp b/common/mail/threadindexer.cpp index 473f28e..ea2cf71 100644 --- a/common/mail/threadindexer.cpp +++ b/common/mail/threadindexer.cpp @@ -24,69 +24,12 @@ using namespace Sink; using namespace Sink::ApplicationDomain; -static QString stripOffPrefixes(const QString &subject) -{ - //TODO this hardcoded list is probably not good enough (especially regarding internationalization) - //TODO this whole routine, including internationalized re/fwd ... should go into some library. - //We'll require the same for generating reply/forward subjects in kube - static QStringList defaultReplyPrefixes = QStringList() << QLatin1String("Re\\s*:") - << QLatin1String("Re\\[\\d+\\]:") - << QLatin1String("Re\\d+:"); - - static QStringList defaultForwardPrefixes = QStringList() << QLatin1String("Fwd:") - << QLatin1String("FW:"); - - QStringList replyPrefixes; // = GlobalSettings::self()->replyPrefixes(); - if (replyPrefixes.isEmpty()) { - replyPrefixes = defaultReplyPrefixes; - } - - QStringList forwardPrefixes; // = GlobalSettings::self()->forwardPrefixes(); - if (forwardPrefixes.isEmpty()) { - forwardPrefixes = defaultReplyPrefixes; - } - - const QStringList prefixRegExps = replyPrefixes + forwardPrefixes; - - // construct a big regexp that - // 1. is anchored to the beginning of str (sans whitespace) - // 2. matches at least one of the part regexps in prefixRegExps - const QString bigRegExp = QString::fromLatin1("^(?:\\s+|(?:%1))+\\s*").arg(prefixRegExps.join(QLatin1String(")|(?:"))); - - static QString regExpPattern; - static QRegExp regExp; - - regExp.setCaseSensitivity(Qt::CaseInsensitive); - if (regExpPattern != bigRegExp) { - // the prefixes have changed, so update the regexp - regExpPattern = bigRegExp; - regExp.setPattern(regExpPattern); - } - - if(regExp.isValid()) { - QString tmp = subject; - if (regExp.indexIn( tmp ) == 0) { - return tmp.remove(0, regExp.matchedLength()); - } - } else { - SinkWarning() << "bigRegExp = \"" - << bigRegExp << "\"\n" - << "prefix regexp is invalid!"; - } - - return subject; -} - - void ThreadIndexer::updateThreadingIndex(const QByteArray &identifier, const ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction) { auto messageId = entity.getProperty(Mail::MessageId::name); auto parentMessageId = entity.getProperty(Mail::ParentMessageId::name); - const auto subject = entity.getProperty(Mail::Subject::name); - const auto normalizedSubject = stripOffPrefixes(subject.toString()).toUtf8(); if (messageId.toByteArray().isEmpty()) { SinkWarning() << "Found an email without messageId. This is illegal and threading will break. Entity id: " << identifier; - SinkWarning() << "Subject: " << subject; } QVector thread; @@ -99,18 +42,9 @@ void ThreadIndexer::updateThreadingIndex(const QByteArray &identifier, const App thread = index().secondaryLookup(parentMessageId); SinkTrace() << "Found parent: " << thread; } - if (thread.isEmpty()) { - //Try to lookup the thread by subject if not empty - if ( !normalizedSubject.isEmpty()) { - thread = index().secondaryLookup(normalizedSubject); - } - if (thread.isEmpty()) { - thread << QUuid::createUuid().toByteArray(); - SinkTrace() << "Created a new thread: " << thread; - } else { - SinkTrace() << "Found thread by subject: " << thread; - } + thread << QUuid::createUuid().toByteArray(); + SinkTrace() << "Created a new thread: " << thread; } Q_ASSERT(!thread.isEmpty()); @@ -122,9 +56,6 @@ void ThreadIndexer::updateThreadingIndex(const QByteArray &identifier, const App } index().index(messageId, thread.first(), transaction); index().index(thread.first(), messageId, transaction); - if (!normalizedSubject.isEmpty()) { - index().index(normalizedSubject, thread.first(), transaction); - } } @@ -146,7 +77,6 @@ void ThreadIndexer::remove(const ApplicationDomain::ApplicationDomainType &entit QMap ThreadIndexer::databases() { return {{"mail.index.messageIdthreadId", 1}, - {"mail.index.subjectthreadId", 1}, {"mail.index.threadIdmessageId", 1}}; } -- cgit v1.2.3 From 01d470b8a94c2e022eb1ca620064732bf2ddece3 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 15 Jun 2017 10:28:50 +0200 Subject: Revert this change, we can just only sync folders in kube. --- examples/imapresource/imapresource.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index b28ecf1..81c808b 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -415,9 +415,8 @@ public: list << Synchronizer::SyncRequest{query}; } else { list << Synchronizer::SyncRequest{Sink::QueryBase(ApplicationDomain::getTypeName())}; - //TODO automatic syncing off all folders is disabled for the time being //This request depends on the previous one so we flush first. - //list << Synchronizer::SyncRequest{applyMailDefaults(Sink::QueryBase(ApplicationDomain::getTypeName())), QByteArray{}, Synchronizer::SyncRequest::RequestFlush}; + list << Synchronizer::SyncRequest{applyMailDefaults(Sink::QueryBase(ApplicationDomain::getTypeName())), QByteArray{}, Synchronizer::SyncRequest::RequestFlush}; } return list; } -- cgit v1.2.3 From 1963f3444e92932f3a31526dc91760d98d027bea Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 15 Jun 2017 10:58:46 +0200 Subject: Bumped release --- dist/sink.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/sink.spec b/dist/sink.spec index b39fdc4..da661a5 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 8%{?dist} +Release: 9%{?dist} Summary: sink Group: Applications/Desktop -- cgit v1.2.3 From 1ef92b9d681e614d65b0ca0a2abdfd81a943e778 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 24 May 2017 13:30:34 +0200 Subject: Added threading index cleanup --- common/mail/threadindexer.cpp | 5 ++++- common/typeindex.cpp | 12 ++++++++++++ common/typeindex.h | 9 +++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/common/mail/threadindexer.cpp b/common/mail/threadindexer.cpp index ea2cf71..4171d85 100644 --- a/common/mail/threadindexer.cpp +++ b/common/mail/threadindexer.cpp @@ -71,7 +71,10 @@ void ThreadIndexer::modify(const ApplicationDomain::ApplicationDomainType &old, void ThreadIndexer::remove(const ApplicationDomain::ApplicationDomainType &entity) { - + auto messageId = entity.getProperty(Mail::MessageId::name); + auto thread = index().secondaryLookup(messageId); + index().unindex(messageId.toByteArray(), thread.first(), transaction()); + index().unindex(thread.first(), messageId.toByteArray(), transaction()); } QMap ThreadIndexer::databases() diff --git a/common/typeindex.cpp b/common/typeindex.cpp index 113c209..5a19839 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -281,6 +281,18 @@ void TypeIndex::index(const QByteArray &leftName, const QBy Index(indexName(leftName + rightName), transaction).add(getByteArray(leftValue), getByteArray(rightValue)); } +template <> +void TypeIndex::unindex(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) +{ + Index(indexName(leftName + rightName), transaction).remove(getByteArray(leftValue), getByteArray(rightValue)); +} + +template <> +void TypeIndex::unindex(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) +{ + Index(indexName(leftName + rightName), transaction).remove(getByteArray(leftValue), getByteArray(rightValue)); +} + template <> QVector TypeIndex::secondaryLookup(const QByteArray &leftName, const QByteArray &rightName, const QVariant &value) { diff --git a/common/typeindex.h b/common/typeindex.h index 1f216a7..890c3db 100644 --- a/common/typeindex.h +++ b/common/typeindex.h @@ -95,6 +95,15 @@ public: template void index(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction); + template + void unindex(const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) + { + index(Left::name, Right::name, leftValue, rightValue, transaction); + } + + template + void unindex(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction); + private: friend class Sink::Storage::EntityStore; -- cgit v1.2.3 From 69954149a432f7a77f2613e348a2089bf5bb2012 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Wed, 21 Jun 2017 07:57:20 +0200 Subject: We can run into this on empty models --- common/modelresult.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/common/modelresult.cpp b/common/modelresult.cpp index b12216b..58703ab 100644 --- a/common/modelresult.cpp +++ b/common/modelresult.cpp @@ -224,7 +224,6 @@ QModelIndex ModelResult::index(int row, int column, const QModelIndex &p return createIndex(row, column, childId); } SinkWarningCtx(mLogCtx) << "Index not available " << row << column << parent; - Q_ASSERT(false); return QModelIndex(); } -- cgit v1.2.3 From 5cabc167e646e9d80b70d6617a3f031d77453979 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 23 Jun 2017 10:20:03 +0200 Subject: Build with flatbuffers 1.7 There's a new template version that expects methods that QByteArray doesn't have but breaks the implicit conversion to const char *, std::string is safer anyways (doesn't require a null terminated string). --- common/bufferutils.h | 2 +- common/pipeline.cpp | 8 ++++---- common/resourceaccess.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/bufferutils.h b/common/bufferutils.h index 6763ced..f0460b7 100644 --- a/common/bufferutils.h +++ b/common/bufferutils.h @@ -38,7 +38,7 @@ static flatbuffers::Offset> modifiedPropertiesList; for (const auto &change : list) { - auto s = fbb.CreateString(change); + auto s = fbb.CreateString(change.toStdString()); modifiedPropertiesList.push_back(s); } return fbb.CreateVector(modifiedPropertiesList); diff --git a/common/pipeline.cpp b/common/pipeline.cpp index dc6f128..f29c7db 100644 --- a/common/pipeline.cpp +++ b/common/pipeline.cpp @@ -303,8 +303,8 @@ KAsync::Job Pipeline::modifiedEntity(void const *command, size_t size) SinkTraceCtx(d->logCtx) << "Move of " << current.identifier() << "was successfull"; if (isMove) { flatbuffers::FlatBufferBuilder fbb; - auto entityId = fbb.CreateString(current.identifier()); - auto type = fbb.CreateString(bufferType); + auto entityId = fbb.CreateString(current.identifier().toStdString()); + auto type = fbb.CreateString(bufferType.toStdString()); auto location = Sink::Commands::CreateDeleteEntity(fbb, current.revision(), entityId, type, true); Sink::Commands::FinishDeleteEntityBuffer(fbb, location); const auto data = BufferUtils::extractBuffer(fbb); @@ -450,8 +450,8 @@ void Preprocessor::createEntity(const Sink::ApplicationDomain::ApplicationDomain const auto entityBuffer = BufferUtils::extractBuffer(entityFbb); flatbuffers::FlatBufferBuilder fbb; - auto entityId = fbb.CreateString(entity.identifier()); - auto type = fbb.CreateString(typeName); + auto entityId = fbb.CreateString(entity.identifier().toStdString()); + auto type = fbb.CreateString(typeName.toStdString()); auto delta = Sink::EntityBuffer::appendAsVector(fbb, entityBuffer.constData(), entityBuffer.size()); auto location = Sink::Commands::CreateCreateEntity(fbb, entityId, type, delta); Sink::Commands::FinishCreateEntityBuffer(fbb, location); diff --git a/common/resourceaccess.cpp b/common/resourceaccess.cpp index cf8b2e0..808d892 100644 --- a/common/resourceaccess.cpp +++ b/common/resourceaccess.cpp @@ -477,7 +477,7 @@ void ResourceAccess::connected() { flatbuffers::FlatBufferBuilder fbb; - auto name = fbb.CreateString(QString("PID: %1 ResourceAccess: %2").arg(QCoreApplication::applicationPid()).arg(reinterpret_cast(this)).toLatin1()); + auto name = fbb.CreateString(QString("PID: %1 ResourceAccess: %2").arg(QCoreApplication::applicationPid()).arg(reinterpret_cast(this)).toLatin1().toStdString()); auto command = Sink::Commands::CreateHandshake(fbb, name); Sink::Commands::FinishHandshakeBuffer(fbb, command); Commands::write(d->socket.data(), ++d->messageId, Commands::HandshakeCommand, fbb); -- cgit v1.2.3 From b24f01b4644bb4d7c1e58a23acdf1fcaef0698b0 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 23 Jun 2017 17:06:52 +0200 Subject: Ensure revision updates are instantaneous. We could do some event compression with a timer if necessary, but for the time being we'll just flush after every notification. This is necessary so i.e. a mail in the outbox appears immediately as it is created in the resource. --- common/listener.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/common/listener.cpp b/common/listener.cpp index ec2bedb..c9fd9d3 100644 --- a/common/listener.cpp +++ b/common/listener.cpp @@ -405,6 +405,7 @@ void Listener::updateClientsWithRevision(qint64 revision) SinkTrace() << "Sending revision update for " << client.name << revision; Sink::Commands::write(client.socket, ++m_messageId, Sink::Commands::RevisionUpdateCommand, m_fbb); + client.socket->flush(); } m_fbb.Clear(); } -- cgit v1.2.3 From 3357b1473efa0ccfa06cb4c2f3ab387c33038274 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sun, 25 Jun 2017 16:10:38 +0200 Subject: Register the right facade. --- examples/davresource/davresource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/davresource/davresource.cpp b/examples/davresource/davresource.cpp index 6631148..88b7338 100644 --- a/examples/davresource/davresource.cpp +++ b/examples/davresource/davresource.cpp @@ -269,7 +269,7 @@ Sink::Resource *DavResourceFactory::createResource(const ResourceContext &contex void DavResourceFactory::registerFacades(const QByteArray &name, Sink::FacadeFactory &factory) { factory.registerFacade>(name); - factory.registerFacade>(name); + factory.registerFacade>(name); } void DavResourceFactory::registerAdaptorFactories(const QByteArray &name, Sink::AdaptorFactoryRegistry ®istry) -- cgit v1.2.3 From 0f888b18bf7e6eecc0a2aa5aaa015cf3b9c755be Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 27 Jun 2017 15:46:54 +0200 Subject: Photo support --- common/contactpreprocessor.cpp | 3 +++ common/domain/applicationdomaintype.cpp | 1 + common/domain/applicationdomaintype.h | 1 + common/domain/contact.fbs | 1 + common/domain/typeimplementations.cpp | 1 + 5 files changed, 7 insertions(+) diff --git a/common/contactpreprocessor.cpp b/common/contactpreprocessor.cpp index ac2c3bc..68a8acb 100644 --- a/common/contactpreprocessor.cpp +++ b/common/contactpreprocessor.cpp @@ -35,6 +35,9 @@ void updatedProperties(Sink::ApplicationDomain::Contact &contact, const KContact emails << Sink::ApplicationDomain::Contact::Email{Sink::ApplicationDomain::Contact::Email::Undefined, email}; } contact.setEmails(emails); + + const auto photo = addressee.photo().rawData(); + contact.setPhoto(photo); } ContactPropertyExtractor::~ContactPropertyExtractor() diff --git a/common/domain/applicationdomaintype.cpp b/common/domain/applicationdomaintype.cpp index 7ce80ee..3718f77 100644 --- a/common/domain/applicationdomaintype.cpp +++ b/common/domain/applicationdomaintype.cpp @@ -112,6 +112,7 @@ SINK_REGISTER_PROPERTY(Contact, Lastname); SINK_REGISTER_PROPERTY(Contact, Emails); SINK_REGISTER_PROPERTY(Contact, Vcard); SINK_REGISTER_PROPERTY(Contact, Addressbook); +SINK_REGISTER_PROPERTY(Contact, Photo); SINK_REGISTER_ENTITY(Addressbook); SINK_REGISTER_PROPERTY(Addressbook, Name); diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h index e5aa46e..602d54c 100644 --- a/common/domain/applicationdomaintype.h +++ b/common/domain/applicationdomaintype.h @@ -377,6 +377,7 @@ struct SINK_EXPORT Contact : public Entity { SINK_PROPERTY(QString, Lastname, lastname); SINK_PROPERTY(QList, Emails, emails); SINK_PROPERTY(QByteArray, Vcard, vcard); + SINK_PROPERTY(QByteArray, Photo, photo); SINK_REFERENCE_PROPERTY(Addressbook, Addressbook, addressbook); }; diff --git a/common/domain/contact.fbs b/common/domain/contact.fbs index d941d5a..e689756 100644 --- a/common/domain/contact.fbs +++ b/common/domain/contact.fbs @@ -13,6 +13,7 @@ table Contact { addressbook:string; emails: [ContactEmail]; vcard: string; + photo: string; } root_type Contact; diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp index 40b0cfd..47a9cf7 100644 --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -153,6 +153,7 @@ void TypeImplementation::configure(PropertyMapper &propertyMapper) SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Addressbook, addressbook); SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Firstname, firstname); SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Lastname, lastname); + SINK_REGISTER_SERIALIZER(propertyMapper, Contact, Photo, photo); } void TypeImplementation::configure(IndexPropertyMapper &) -- cgit v1.2.3 From 06f996a139a5ac660e98163fac796f94c1a6468f Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 27 Jun 2017 20:57:31 +0200 Subject: Ensure we can deal with non-null terminated strings. --- common/contactpreprocessor.cpp | 3 +-- common/propertymapper.cpp | 16 +++++++------ tests/domainadaptortest.cpp | 51 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/common/contactpreprocessor.cpp b/common/contactpreprocessor.cpp index 68a8acb..d331421 100644 --- a/common/contactpreprocessor.cpp +++ b/common/contactpreprocessor.cpp @@ -36,8 +36,7 @@ void updatedProperties(Sink::ApplicationDomain::Contact &contact, const KContact } contact.setEmails(emails); - const auto photo = addressee.photo().rawData(); - contact.setPhoto(photo); + contact.setPhoto(addressee.photo().rawData()); } ContactPropertyExtractor::~ContactPropertyExtractor() diff --git a/common/propertymapper.cpp b/common/propertymapper.cpp index c14a62e..dbf93a3 100644 --- a/common/propertymapper.cpp +++ b/common/propertymapper.cpp @@ -58,7 +58,9 @@ template <> flatbuffers::uoffset_t variantToProperty(const QVariant &property, flatbuffers::FlatBufferBuilder &fbb) { if (property.isValid()) { - return fbb.CreateString(property.toByteArray().toStdString()).o; + const auto ba = property.toByteArray(); + const auto s = fbb.CreateString(ba.constData(), ba.size()); + return s.o; } return 0; } @@ -135,7 +137,7 @@ QString propertyToString(const flatbuffers::String *property) { if (property) { // We have to copy the memory, otherwise it would become eventually invalid - return QString::fromStdString(property->c_str()); + return QString::fromStdString(property->str()); } return QString(); } @@ -145,7 +147,7 @@ QVariant propertyToVariant(const flatbuffers::String *property) { if (property) { // We have to copy the memory, otherwise it would become eventually invalid - return QString::fromStdString(property->c_str()); + return QString::fromStdString(property->str()); } return QVariant(); } @@ -155,7 +157,7 @@ QVariant propertyToVariant(const flatbuffers::Str { if (property) { // We have to copy the memory, otherwise it would become eventually invalid - auto s = QString::fromStdString(property->c_str()); + auto s = QString::fromStdString(property->str()); auto ext = s.endsWith(":ext"); s.chop(4); @@ -171,7 +173,7 @@ QVariant propertyToVariant(const flatbuffers { if (property) { // We have to copy the memory, otherwise it would become eventually invalid - return QVariant::fromValue(Sink::ApplicationDomain::Reference{QString::fromStdString(property->c_str()).toUtf8()}); + return QVariant::fromValue(Sink::ApplicationDomain::Reference{QString::fromStdString(property->str()).toUtf8()}); } return QVariant(); } @@ -181,7 +183,7 @@ QVariant propertyToVariant(const flatbuffers::String *property) { if (property) { // We have to copy the memory, otherwise it would become eventually invalid - return QString::fromStdString(property->c_str()).toUtf8(); + return QByteArray(property->c_str(), property->Length()); } return QVariant(); } @@ -203,7 +205,7 @@ QVariant propertyToVariant(const flatbuffers::Vectorbegin(); it != property->end();) { // We have to copy the memory, otherwise it would become eventually invalid - list << QString::fromStdString((*it)->c_str()).toUtf8(); + list << QString::fromStdString((*it)->str()).toUtf8(); it.operator++(); } return QVariant::fromValue(list); diff --git a/tests/domainadaptortest.cpp b/tests/domainadaptortest.cpp index df44819..2aed0a9 100644 --- a/tests/domainadaptortest.cpp +++ b/tests/domainadaptortest.cpp @@ -17,17 +17,19 @@ class TestFactory : public DomainTypeAdaptorFactory { public: - TestFactory() - { - } + TestFactory() = default; }; class TestMailFactory : public DomainTypeAdaptorFactory { public: - TestMailFactory() - { - } + TestMailFactory() = default; +}; + +class TestContactFactory : public DomainTypeAdaptorFactory +{ +public: + TestContactFactory() = default; }; /** @@ -135,6 +137,43 @@ private slots: } } + + void testContact() + { + auto writeMapper = QSharedPointer::create(); + Sink::ApplicationDomain::TypeImplementation::configure(*writeMapper); + + auto binaryData = QByteArray::fromRawData("\xEF\xBF\xBD\x00\xEF\xBF", 5); + + Sink::ApplicationDomain::Contact contact; + contact.setPhoto(binaryData); + QVERIFY(!contact.getPhoto().isEmpty()); + + flatbuffers::FlatBufferBuilder metadataFbb; + auto metadataBuilder = Sink::MetadataBuilder(metadataFbb); + metadataBuilder.add_revision(1); + auto metadataBuffer = metadataBuilder.Finish(); + Sink::FinishMetadataBuffer(metadataFbb, metadataBuffer); + + flatbuffers::FlatBufferBuilder mailFbb; + auto pos = createBufferPart(contact, mailFbb, *writeMapper); + Sink::ApplicationDomain::Buffer::FinishContactBuffer(mailFbb, pos); + + flatbuffers::FlatBufferBuilder fbb; + Sink::EntityBuffer::assembleEntityBuffer( + fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize(), mailFbb.GetBufferPointer(), mailFbb.GetSize(), mailFbb.GetBufferPointer(), mailFbb.GetSize()); + + { + std::string data(reinterpret_cast(fbb.GetBufferPointer()), fbb.GetSize()); + Sink::EntityBuffer buffer((void *)(data.data()), data.size()); + + TestContactFactory factory; + auto adaptor = factory.createAdaptor(buffer.entity()); + Sink::ApplicationDomain::Contact readContact{QByteArray{}, QByteArray{}, 0, adaptor}; + QCOMPARE(readContact.getPhoto(), contact.getPhoto()); + } + + } }; QTEST_MAIN(DomainAdaptorTest) -- cgit v1.2.3 From f3eb08a93f093ebe4ac97221993103c4f12c6212 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 27 Jun 2017 23:15:27 +0200 Subject: Ensure we have the right facade --- common/facade.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/common/facade.cpp b/common/facade.cpp index 38ec1dc..9c14a23 100644 --- a/common/facade.cpp +++ b/common/facade.cpp @@ -104,6 +104,7 @@ KAsync::Job GenericFacade::remove(const DomainType &domainObje template QPair, typename ResultEmitter::Ptr> GenericFacade::load(const Sink::Query &query, const Log::Context &ctx) { + Q_ASSERT(DomainType::name == query.type() || query.type().isEmpty()); // The runner lives for the lifetime of the query auto runner = new QueryRunner(query, mResourceContext, bufferTypeForDomainType(), ctx); runner->setResultTransformation(mResultTransformation); -- cgit v1.2.3 From 26c27fd619857d0945b6c110a863bdc8b4a5e073 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 29 Jun 2017 12:09:11 +0200 Subject: Ported to KDAV2, enabled the dav resource by default. --- CMakeLists.txt | 2 +- examples/davresource/CMakeLists.txt | 4 ++-- examples/davresource/davresource.cpp | 38 ++++++++++++++++++------------------ examples/davresource/davresource.h | 4 ++-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 53716a0..7666a8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_policy(SET CMP0028 NEW) project(sink VERSION 0.3) option(BUILD_MAILDIR "BUILD_MAILDIR" ON) -option(BUILD_DAV "BUILD_DAV" OFF) +option(BUILD_DAV "BUILD_DAV" ON) option(AVOID_BINDING_REBUILD "AVOID_BINDING_REBUILD" OFF) option(CATCH_ERRORS "CATCH_ERRORS" OFF) option(ENABLE_MEMCHECK "Build valgrind tests" OFF) diff --git a/examples/davresource/CMakeLists.txt b/examples/davresource/CMakeLists.txt index 28829d5..7091edc 100644 --- a/examples/davresource/CMakeLists.txt +++ b/examples/davresource/CMakeLists.txt @@ -3,10 +3,10 @@ project(sink_resource_dav) add_definitions(-DQT_PLUGIN) include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -find_package(KPimKDAV REQUIRED) +find_package(KPimKDAV2 REQUIRED) add_library(${PROJECT_NAME} SHARED davresource.cpp) qt5_use_modules(${PROJECT_NAME} Core Network) -target_link_libraries(${PROJECT_NAME} sink KPim::KDAV) +target_link_libraries(${PROJECT_NAME} sink KPim::KDAV2) install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION ${SINK_RESOURCE_PLUGINS_PATH}) diff --git a/examples/davresource/davresource.cpp b/examples/davresource/davresource.cpp index 88b7338..1d45f20 100644 --- a/examples/davresource/davresource.cpp +++ b/examples/davresource/davresource.cpp @@ -31,12 +31,12 @@ #include "contactpreprocessor.h" -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include //This is the resources entity type, and not the domain type #define ENTITY_TYPE_CONTACT "contact" @@ -85,7 +85,7 @@ public: return remoteId; } - void synchronizeAddressbooks(const KDAV::DavCollection::List &addressbookList) + void synchronizeAddressbooks(const KDAV2::DavCollection::List &addressbookList) { const QByteArray bufferType = ENTITY_TYPE_ADDRESSBOOK; SinkTrace() << "Found addressbooks " << addressbookList.size(); @@ -119,12 +119,12 @@ public: return list; } - static QByteArray getRid(const KDAV::DavItem &item) + static QByteArray getRid(const KDAV2::DavItem &item) { return item.url().toDisplayString().toUtf8(); } - static QByteArray getRid(const KDAV::DavCollection &item) + static QByteArray getRid(const KDAV2::DavCollection &item) { return item.url().toDisplayString().toUtf8(); } @@ -133,7 +133,7 @@ public: { if (query.type() == ApplicationDomain::getTypeName()) { SinkLogCtx(mLogCtx) << "Synchronizing addressbooks:" << mResourceUrl.url(); - auto collectionsFetchJob = new KDAV::DavCollectionsFetchJob(mResourceUrl); + auto collectionsFetchJob = new KDAV2::DavCollectionsFetchJob(mResourceUrl); auto job = runJob(collectionsFetchJob).then([this, collectionsFetchJob] (const KAsync::Error &error) { if (error) { SinkWarningCtx(mLogCtx) << "Failed to synchronize addressbooks." << collectionsFetchJob->errorString(); @@ -145,29 +145,29 @@ public: } else if (query.type() == ApplicationDomain::getTypeName()) { SinkLogCtx(mLogCtx) << "Synchronizing contacts."; auto ridList = QSharedPointer::create(); - auto collectionsFetchJob = new KDAV::DavCollectionsFetchJob(mResourceUrl); + auto collectionsFetchJob = new KDAV2::DavCollectionsFetchJob(mResourceUrl); auto job = runJob(collectionsFetchJob).then([this, collectionsFetchJob] { synchronizeAddressbooks(collectionsFetchJob ->collections()); return collectionsFetchJob->collections(); }) - .serialEach([this, ridList](const KDAV::DavCollection &collection) { + .serialEach([this, ridList](const KDAV2::DavCollection &collection) { auto collId = getRid(collection); const auto addressbookLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_ADDRESSBOOK, collId); auto ctag = collection.CTag().toLatin1(); if (ctag != syncStore().readValue(collId + "_ctagXX")) { SinkTraceCtx(mLogCtx) << "Syncing " << collId; - auto cache = std::shared_ptr(new KDAV::EtagCache()); - auto davItemsListJob = new KDAV::DavItemsListJob(collection.url(), cache); + auto cache = std::shared_ptr(new KDAV2::EtagCache()); + auto davItemsListJob = new KDAV2::DavItemsListJob(collection.url(), cache); const QByteArray bufferType = ENTITY_TYPE_CONTACT; QHash mergeCriteria; auto colljob = runJob(davItemsListJob).then([davItemsListJob] { return KAsync::value(davItemsListJob->items()); }) - .serialEach([=] (const KDAV::DavItem &item) { + .serialEach([=] (const KDAV2::DavItem &item) { QByteArray rid = getRid(item); if (item.etag().toLatin1() != syncStore().readValue(rid + "_etag")){ SinkTrace() << "Updating " << rid; - auto davItemFetchJob = new KDAV::DavItemFetchJob(item); + auto davItemFetchJob = new KDAV2::DavItemFetchJob(item); auto itemjob = runJob(davItemFetchJob) .then([=] { const auto item = davItemFetchJob->item(); @@ -178,7 +178,7 @@ public: createOrModify(bufferType, rid, contact, mergeCriteria); return item; }) - .then([this, ridList] (const KDAV::DavItem &item) { + .then([this, ridList] (const KDAV2::DavItem &item) { const auto rid = getRid(item); syncStore().writeValue(rid + "_etag", item.etag().toLatin1()); ridList->append(rid); @@ -225,7 +225,7 @@ KAsync::Job replay(const ApplicationDomain::Contact &contact, Sink:: } public: - KDAV::DavUrl mResourceUrl; + KDAV2::DavUrl mResourceUrl; }; @@ -242,7 +242,7 @@ DavResource::DavResource(const Sink::ResourceContext &resourceContext) resourceUrl.setUserName(config.value("username").toString()); resourceUrl.setPassword(config.value("password").toString()); - mResourceUrl = KDAV::DavUrl(resourceUrl, KDAV::CardDav); + mResourceUrl = KDAV2::DavUrl(resourceUrl, KDAV2::CardDav); auto synchronizer = QSharedPointer::create(resourceContext); synchronizer->mResourceUrl = mResourceUrl; diff --git a/examples/davresource/davresource.h b/examples/davresource/davresource.h index 1ce66ea..db175a4 100644 --- a/examples/davresource/davresource.h +++ b/examples/davresource/davresource.h @@ -21,7 +21,7 @@ #include "common/genericresource.h" -#include +#include #include #include @@ -48,7 +48,7 @@ public: private: QStringList listAvailableFolders(); - KDAV::DavUrl mResourceUrl; + KDAV2::DavUrl mResourceUrl; }; class DavResourceFactory : public Sink::ResourceFactory -- cgit v1.2.3 From 7088853437e5fb41be8d1e6268764dfc31d9a2f3 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 29 Jun 2017 12:11:20 +0200 Subject: Updated spec-file --- dist/sink.spec | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dist/sink.spec b/dist/sink.spec index da661a5..f5132b7 100644 --- a/dist/sink.spec +++ b/dist/sink.spec @@ -1,7 +1,7 @@ Name: sink Version: 0.3 -Release: 9%{?dist} +Release: 10%{?dist} Summary: sink Group: Applications/Desktop @@ -18,12 +18,12 @@ BuildRequires: kf5-kcoreaddons-devel BuildRequires: kf5-kcontacts-devel BuildRequires: kf5-kmime-devel BuildRequires: kimap2-devel +BuildRequires: kdav2-devel BuildRequires: libcurl-devel BuildRequires: libgit2-devel BuildRequires: lmdb-devel BuildRequires: qt5-qtbase-devel BuildRequires: readline-devel -BuildRequires: kdav-devel %description sink @@ -43,7 +43,6 @@ mkdir -p build/ pushd build %{cmake} \ -DQT_PLUGIN_INSTALL_DIR:PATH=%{_libdir}/qt5/plugins/ \ - -DBUILD_DAV=ON \ .. make %{?_smp_mflags} -- cgit v1.2.3 From 682b90cc4138ce7d4c2a4eb4b28c8a17b08cd6de Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 29 Jun 2017 12:13:22 +0200 Subject: KDE_FORK_SLAVES is no longer necessary --- examples/davresource/davresource.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/examples/davresource/davresource.cpp b/examples/davresource/davresource.cpp index 1d45f20..465220f 100644 --- a/examples/davresource/davresource.cpp +++ b/examples/davresource/davresource.cpp @@ -232,11 +232,6 @@ public: DavResource::DavResource(const Sink::ResourceContext &resourceContext) : Sink::GenericResource(resourceContext) { - /* - * Fork KIO slaves (used in kdav), instead of starting them via klauncher. - * Otherwise we have yet another runtime dependency that will i.e. not work in the docker container. - */ - qputenv("KDE_FORK_SLAVES", "TRUE"); auto config = ResourceConfig::getConfiguration(resourceContext.instanceId()); auto resourceUrl = QUrl::fromUserInput(config.value("server").toString()); resourceUrl.setUserName(config.value("username").toString()); -- cgit v1.2.3 From 288946f1694c2abe1d2c5800c87339d1e8780e4b Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 1 Jul 2017 21:04:42 +0200 Subject: Release of 0.3.0 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7666a8c..6690c4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.0) cmake_policy(SET CMP0048 NEW) cmake_policy(SET CMP0028 NEW) -project(sink VERSION 0.3) +project(sink VERSION 0.3.0) option(BUILD_MAILDIR "BUILD_MAILDIR" ON) option(BUILD_DAV "BUILD_DAV" ON) -- cgit v1.2.3