Xapian based fulltext indexing

This cuts into the sync performance by about 40%,
but gives us fast fulltext searching for all local content.

24 files changed, 519 insertions, 64 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index bb6df73..3327afd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -36,6 +36,7 @@ find_package(KF5 COMPONENTS REQUIRED Mime Contacts)
 find_package(FlatBuffers REQUIRED 1.4.0)
 find_package(KAsync REQUIRED 0.1.2)
 find_package(LMDB REQUIRED 0.9)
+find_package(Xapian REQUIRED 1.4)
 
 if (${ENABLE_MEMCHECK})
     message("Enabled memcheck")
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
index ec83f6f..76579dd 100644
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -75,11 +75,13 @@ set(command_SRCS
     storage/entitystore.cpp
     indexer.cpp
     mail/threadindexer.cpp
+    mail/fulltextindexer.cpp
     notification.cpp
     commandprocessor.cpp
     inspector.cpp
     propertyparser.cpp
     utils.cpp
+    fulltextindex.cpp
     ${storage_SRCS})
 
 add_library(${PROJECT_NAME} SHARED ${command_SRCS})
@@ -127,6 +129,7 @@ PRIVATE
     Qt5::Gui
     KF5::Mime
     KF5::Contacts
+    ${XAPIAN_LIBRARIES}
 )
 install(TARGETS ${PROJECT_NAME}
     EXPORT SinkTargets ${KDE_INSTALL_TARGETS_DEFAULT_ARGS} ${LIBRARY_NAMELINK} )
diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp
index 870daf8..3218d1a 100644
--- a/common/datastorequery.cpp
+++ b/common/datastorequery.cpp
@@ -160,6 +160,10 @@ public:
         for (const auto &filterProperty : propertyFilter.keys()) {
             const auto property = entity.getProperty(filterProperty);
             const auto comparator = propertyFilter.value(filterProperty);
+            //We can't deal with a fulltext filter
+            if (comparator.comparator == QueryBase::Comparator::Fulltext) {
+                continue;
+            }
             if (!comparator.matches(property)) {
                 SinkTraceCtx(mDatastore->mLogCtx) << "Filtering entity due to property mismatch on filter: " << entity.identifier() << "Property: " << filterProperty << property << " Filter:" << comparator.value;
                 return false;
diff --git a/common/definitions.cpp b/common/definitions.cpp
index b22137a..642b68c 100644
--- a/common/definitions.cpp
+++ b/common/definitions.cpp
@@ -90,5 +90,5 @@ QString Sink::resourceStorageLocation(const QByteArray &resourceInstanceIdentifi
 
 qint64 Sink::latestDatabaseVersion()
 {
-    return 1;
+    return 2;
 }
diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp
index 47a9cf7..29da7ea 100644
--- a/common/domain/typeimplementations.cpp
+++ b/common/domain/typeimplementations.cpp
@@ -27,6 +27,7 @@
 #include "entitybuffer.h"
 #include "entity_generated.h"
 #include "mail/threadindexer.h"
+#include "mail/fulltextindexer.h"
 #include "domainadaptor.h"
 #include "typeimplementations_p.h"
 
@@ -45,7 +46,8 @@ typedef IndexConfig<Mail,
         SortedIndex<Mail::Folder, Mail::Date>,
         SecondaryIndex<Mail::MessageId, Mail::ThreadId>,
         SecondaryIndex<Mail::ThreadId, Mail::MessageId>,
-        CustomSecondaryIndex<Mail::MessageId, Mail::ThreadId, ThreadIndexer>
+        CustomSecondaryIndex<Mail::MessageId, Mail::ThreadId, ThreadIndexer>,
+        CustomSecondaryIndex<Mail::Subject, Mail::Subject, FulltextIndexer>
     > MailIndexConfig;
 
 typedef IndexConfig<Folder,
diff --git a/common/fulltextindex.cpp b/common/fulltextindex.cpp
new file mode 100644
index 0000000..33972b7
--- /dev/null
+++ b/common/fulltextindex.cpp
@@ -0,0 +1,149 @@
+/*
+ *   Copyright (C) 2018 Christian Mollekopf <mollekopf@kolabsys.com>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+//xapian.h needs to be included first to build
+#include <xapian.h>
+#include "fulltextindex.h"
+
+#include <QFile>
+#include <QDir>
+
+#include "log.h"
+#include "definitions.h"
+
+FulltextIndex::FulltextIndex(const QByteArray &resourceInstanceIdentifier, Sink::Storage::DataStore::AccessMode accessMode)
+    : mName("fulltext"),
+    mDbPath{QFile::encodeName(Sink::resourceStorageLocation(resourceInstanceIdentifier) + '/' + "fulltext")}
+{
+    try {
+        if (QDir{}.mkpath(mDbPath)) {
+            if (accessMode == Sink::Storage::DataStore::ReadWrite) {
+                mDb = new Xapian::WritableDatabase(mDbPath.toStdString(), Xapian::DB_CREATE_OR_OPEN);
+            } else {
+                mDb = new Xapian::Database(mDbPath.toStdString(), Xapian::DB_OPEN);
+            }
+        } else {
+            SinkError() << "Failed to open database" << mDbPath;
+        }
+    } catch (const Xapian::DatabaseError& e) {
+        SinkError() << "Failed to open database" << mDbPath << ":" << QString::fromStdString(e.get_msg());
+    }
+}
+
+FulltextIndex::~FulltextIndex()
+{
+    delete mDb;
+}
+
+static std::string idTerm(const QByteArray &key)
+{
+    return "Q" + key.toStdString();
+}
+
+void FulltextIndex::add(const QByteArray &key, const QString &value)
+{
+    add(key, {{{}, value}});
+}
+
+void FulltextIndex::add(const QByteArray &key, const QList<QPair<QString, QString>> &values)
+{
+    if (!mDb) {
+        return;
+    }
+    Xapian::TermGenerator generator;
+    Xapian::Document document;
+    generator.set_document(document);
+
+    for (const auto &entry : values) {
+        if (!entry.second.isEmpty()) {
+            generator.index_text(entry.second.toStdString());
+        }
+    }
+    document.add_value(0, key.toStdString());
+
+    const auto idterm = idTerm(key);
+    document.add_boolean_term(idterm);
+
+    writableDatabase()->replace_document(idterm, document);
+}
+
+void FulltextIndex::commitTransaction()
+{
+    if (mHasTransactionOpen) {
+        Q_ASSERT(mDb);
+        writableDatabase()->commit_transaction();
+        mHasTransactionOpen = false;
+    }
+}
+
+void FulltextIndex::abortTransaction()
+{
+    if (mHasTransactionOpen) {
+        Q_ASSERT(mDb);
+        writableDatabase()->cancel_transaction();
+        mHasTransactionOpen = false;
+    }
+}
+
+Xapian::WritableDatabase* FulltextIndex::writableDatabase()
+{
+    Q_ASSERT(dynamic_cast<Xapian::WritableDatabase*>(mDb));
+    auto db = static_cast<Xapian::WritableDatabase*>(mDb);
+    if (!mHasTransactionOpen) {
+        db->begin_transaction();
+        mHasTransactionOpen = true;
+    }
+    return db;
+}
+
+void FulltextIndex::remove(const QByteArray &key)
+{
+    if (!mDb) {
+        return;
+    }
+    writableDatabase()->delete_document(idTerm(key));
+}
+
+QVector<QByteArray> FulltextIndex::lookup(const QString &searchTerm)
+{
+    if (!mDb) {
+        return {};
+    }
+    QVector<QByteArray> results;
+
+    try {
+        Xapian::QueryParser parser;
+        auto query = parser.parse_query(searchTerm.toStdString(), Xapian::QueryParser::FLAG_WILDCARD|Xapian::QueryParser::FLAG_PHRASE|Xapian::QueryParser::FLAG_BOOLEAN|Xapian::QueryParser::FLAG_LOVEHATE);
+        Xapian::Enquire enquire(*mDb);
+        enquire.set_query(query);
+
+        auto limit = 1000;
+        Xapian::MSet mset = enquire.get_mset(0, limit);
+        Xapian::MSetIterator it = mset.begin();
+        for (;it != mset.end(); it++) {
+            auto doc = it.get_document();
+            const auto data = doc.get_value(0);
+            results << QByteArray{data.c_str(), int(data.length())};
+        }
+    }
+    catch (const Xapian::Error &error) {
+        // Nothing to do, move along
+    }
+    return results;
+}
+
diff --git a/common/fulltextindex.h b/common/fulltextindex.h
new file mode 100644
index 0000000..e06f29d
--- /dev/null
+++ b/common/fulltextindex.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#include "sink_export.h"
+
+#include <string>
+#include <functional>
+#include <QString>
+#include <memory>
+#include "storage.h"
+#include "log.h"
+
+namespace Xapian {
+    class Database;
+    class WritableDatabase;
+};
+
+class SINK_EXPORT FulltextIndex
+{
+public:
+    FulltextIndex(const QByteArray &resourceInstanceIdentifier, Sink::Storage::DataStore::AccessMode mode = Sink::Storage::DataStore::ReadOnly);
+    ~FulltextIndex();
+
+    void add(const QByteArray &key, const QString &value);
+    void add(const QByteArray &key, const QList<QPair<QString, QString>> &values);
+    void remove(const QByteArray &key);
+
+    void commitTransaction();
+    void abortTransaction();
+
+    QVector<QByteArray> lookup(const QString &key);
+
+private:
+    Xapian::WritableDatabase* writableDatabase();
+    Q_DISABLE_COPY(FulltextIndex);
+    Xapian::Database *mDb{nullptr};
+    QString mName;
+    QString mDbPath;
+    bool mHasTransactionOpen{false};
+};
diff --git a/common/indexer.cpp b/common/indexer.cpp
index 1b223b3..c18170c 100644
--- a/common/indexer.cpp
+++ b/common/indexer.cpp
@@ -20,10 +20,11 @@
 
 using namespace Sink;
 
-void Indexer::setup(TypeIndex *index, Storage::DataStore::Transaction *transaction)
+void Indexer::setup(TypeIndex *index, Storage::DataStore::Transaction *transaction, const QByteArray &resourceId)
 {
     mTypeIndex = index;
     mTransaction = transaction;
+    mResourceInstanceIdentifier = resourceId;
 }
 
 Storage::DataStore::Transaction &Indexer::transaction()
diff --git a/common/indexer.h b/common/indexer.h
index 26887fb..f0b32f5 100644
--- a/common/indexer.h
+++ b/common/indexer.h
@@ -33,16 +33,18 @@ public:
     virtual ~Indexer() = default;
     typedef QSharedPointer<Indexer> Ptr;
     virtual void add(const ApplicationDomain::ApplicationDomainType &entity) = 0;
-    virtual void modify(const ApplicationDomain::ApplicationDomainType &old, const ApplicationDomain::ApplicationDomainType &entity) = 0;
     virtual void remove(const ApplicationDomain::ApplicationDomainType &entity) = 0;
+    virtual void commitTransaction() {};
+    virtual void abortTransaction() {};
 
 protected:
     Storage::DataStore::Transaction &transaction();
     TypeIndex &index();
+    QByteArray mResourceInstanceIdentifier;
 
 private:
     friend class ::TypeIndex;
-    void setup(TypeIndex *, Storage::DataStore::Transaction *);
+    void setup(TypeIndex *, Storage::DataStore::Transaction *, const QByteArray &resourceId);
     Storage::DataStore::Transaction *mTransaction;
     TypeIndex *mTypeIndex;
 };
diff --git a/common/mail/fulltextindexer.cpp b/common/mail/fulltextindexer.cpp
new file mode 100644
index 0000000..a980752
--- /dev/null
+++ b/common/mail/fulltextindexer.cpp
@@ -0,0 +1,64 @@
+/*
+ *   Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+#include "fulltextindexer.h"
+
+#include "typeindex.h"
+#include "fulltextindex.h"
+#include "log.h"
+#include "utils.h"
+
+using namespace Sink;
+using namespace Sink::ApplicationDomain;
+
+
+void FulltextIndexer::add(const ApplicationDomain::ApplicationDomainType &entity)
+{
+    if (!index) {
+        index.reset(new FulltextIndex{mResourceInstanceIdentifier, Storage::DataStore::ReadWrite});
+    }
+    index->add(entity.identifier(), entity.getProperty("index").value<QList<QPair<QString, QString>>>());
+}
+
+void FulltextIndexer::remove(const ApplicationDomain::ApplicationDomainType &entity)
+{
+    if (!index) {
+        index.reset(new FulltextIndex{mResourceInstanceIdentifier, Storage::DataStore::ReadWrite});
+    }
+    index->remove(entity.identifier());
+}
+
+void FulltextIndexer::commitTransaction()
+{
+    if (index) {
+        index->commitTransaction();
+    }
+}
+
+void FulltextIndexer::abortTransaction()
+{
+    if (index) {
+        index->abortTransaction();
+    }
+}
+
+QMap<QByteArray, int> FulltextIndexer::databases()
+{
+    return {};
+}
+
diff --git a/common/mail/fulltextindexer.h b/common/mail/fulltextindexer.h
new file mode 100644
index 0000000..ef41455
--- /dev/null
+++ b/common/mail/fulltextindexer.h
@@ -0,0 +1,39 @@
+/*
+ *   Copyright (C) 2015 Christian Mollekopf <chrigi_1@fastmail.fm>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the
+ *   Free Software Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+ */
+#pragma once
+
+#include "indexer.h"
+
+class FulltextIndex;
+namespace Sink {
+
+class FulltextIndexer : public Indexer
+{
+public:
+    typedef QSharedPointer<FulltextIndexer> Ptr;
+    virtual void add(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE;
+    virtual void remove(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE;
+    virtual void commitTransaction() Q_DECL_OVERRIDE;
+    virtual void abortTransaction() Q_DECL_OVERRIDE;
+    static QMap<QByteArray, int> databases();
+private:
+    QSharedPointer<FulltextIndex> index;
+};
+
+}
diff --git a/common/mail/threadindexer.cpp b/common/mail/threadindexer.cpp
index 2e6a6e7..fb47118 100644
--- a/common/mail/threadindexer.cpp
+++ b/common/mail/threadindexer.cpp
@@ -98,11 +98,6 @@ void ThreadIndexer::add(const ApplicationDomain::ApplicationDomainType &entity)
     updateThreadingIndex(entity.identifier(), entity, transaction());
 }
 
-void ThreadIndexer::modify(const ApplicationDomain::ApplicationDomainType &old, const ApplicationDomain::ApplicationDomainType &entity)
-{
-
-}
-
 void ThreadIndexer::remove(const ApplicationDomain::ApplicationDomainType &entity)
 {
     auto messageId = entity.getProperty(Mail::MessageId::name);
diff --git a/common/mail/threadindexer.h b/common/mail/threadindexer.h
index 60d0863..b2e939a 100644
--- a/common/mail/threadindexer.h
+++ b/common/mail/threadindexer.h
@@ -27,7 +27,6 @@ class ThreadIndexer : public Indexer
 public:
     typedef QSharedPointer<ThreadIndexer> Ptr;
     virtual void add(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE;
-    virtual void modify(const ApplicationDomain::ApplicationDomainType &old, const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE;
     virtual void remove(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE;
     static QMap<QByteArray, int> databases();
 private:
diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp
index 8f5a77d..58cb15b 100644
--- a/common/mailpreprocessor.cpp
+++ b/common/mailpreprocessor.cpp
@@ -21,9 +21,11 @@
 
 #include <QFile>
 #include <QDir>
+#include <QTextDocument>
 #include <KMime/KMime/KMimeMessage>
 
 #include "pipeline.h"
+#include "fulltextindex.h"
 #include "definitions.h"
 #include "applicationdomaintype.h"
 
@@ -45,13 +47,34 @@ static QList<Sink::ApplicationDomain::Mail::Contact> getContactList(const KMime:
     return list;
 }
 
+static QList<QPair<QString, QString>> processPart(KMime::Content* content)
+{
+    if (KMime::Headers::ContentType* type = content->contentType(false)) {
+        if (type->isMultipart() && !type->isSubtype("encrypted")) {
+            QList<QPair<QString, QString>> list;
+            for (const auto c : content->contents()) {
+                list << processPart(c);
+            }
+            return list;
+        } else if (type->isHTMLText()) {
+            // Only get HTML content, if no plain text content
+            QTextDocument doc;
+            doc.setHtml(content->decodedText());
+            return {{{}, {doc.toPlainText()}}};
+        } else if (type->isEmpty()) {
+            return {{{}, {content->decodedText()}}};
+        }
+    }
+    return {};
+}
+
 void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data)
 {
     if (data.isEmpty()) {
         return;
     }
     auto msg = KMime::Message::Ptr(new KMime::Message);
-    msg->setHead(KMime::CRLFtoLF(data));
+    msg->setContent(KMime::CRLFtoLF(data));
     msg->parse();
     if (!msg) {
         return;
@@ -103,6 +126,20 @@ void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Ma
     if (!parentMessageId.isEmpty()) {
         mail.setExtractedParentMessageId(parentMessageId);
     }
+    QList<QPair<QString, QString>> contentToIndex;
+    contentToIndex.append({{}, msg->subject()->asUnicodeString()});
+    if (KMime::Content* mainBody = msg->mainBodyPart("text/plain")) {
+        contentToIndex.append({{}, mainBody->decodedText()});
+    } else {
+        contentToIndex << processPart(msg.data());
+    }
+    contentToIndex.append({{}, msg->from(true)->asUnicodeString()});
+    contentToIndex.append({{}, msg->to(true)->asUnicodeString()});
+    contentToIndex.append({{}, msg->cc(true)->asUnicodeString()});
+    contentToIndex.append({{}, msg->bcc(true)->asUnicodeString()});
+
+    //Prepare content for indexing;
+    mail.setProperty("index", QVariant::fromValue(contentToIndex));
 }
 
 void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail)
@@ -114,4 +151,3 @@ void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &
 {
     updatedIndexedProperties(newMail, newMail.getMimeMessage());
 }
-
diff --git a/common/mailpreprocessor.h b/common/mailpreprocessor.h
index d2e79ca..c0eacaf 100644
--- a/common/mailpreprocessor.h
+++ b/common/mailpreprocessor.h
@@ -29,4 +29,3 @@ public:
 protected:
     static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data);
 };
-
diff --git a/common/query.cpp b/common/query.cpp
index 3dc8f99..5f6d095 100644
--- a/common/query.cpp
+++ b/common/query.cpp
@@ -34,6 +34,8 @@ QDebug operator<<(QDebug dbg, const Sink::QueryBase::Comparator &c)
         dbg.nospace() << "contains " << c.value;
     } else if (c.comparator == Sink::Query::Comparator::In) {
         dbg.nospace() << "in " << c.value;
+    } else if (c.comparator == Sink::Query::Comparator::Fulltext) {
+        dbg.nospace() << "fulltext contains " << c.value;
     } else {
         dbg.nospace() << "unknown comparator: " << c.value;
     }
@@ -169,6 +171,7 @@ bool QueryBase::Comparator::matches(const QVariant &v) const
                 return false;
             }
             return value.value<QByteArrayList>().contains(v.toByteArray());
+        case Fulltext:
         case Invalid:
         default:
             break;
diff --git a/common/query.h b/common/query.h
index 5b37cdd..1e7b41d 100644
--- a/common/query.h
+++ b/common/query.h
@@ -35,7 +35,8 @@ public:
             Invalid,
             Equals,
             Contains,
-            In
+            In,
+            Fulltext
         };
 
         Comparator();
diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp
index 4ad3eaf..8fbc2ad 100644
--- a/common/storage/entitystore.cpp
+++ b/common/storage/entitystore.cpp
@@ -162,23 +162,27 @@ void EntityStore::startTransaction(Sink::Storage::DataStore::AccessMode accessMo
 {
     SinkTraceCtx(d->logCtx) << "Starting transaction: " << accessMode;
     Q_ASSERT(!d->transaction);
-    Sink::Storage::DataStore store(Sink::storageLocation(), dbLayout(d->resourceContext.instanceId()), accessMode);
-    d->transaction = store.createTransaction(accessMode);
+    d->transaction = Sink::Storage::DataStore(Sink::storageLocation(), dbLayout(d->resourceContext.instanceId()), accessMode).createTransaction(accessMode);
 }
 
 void EntityStore::commitTransaction()
 {
     SinkTraceCtx(d->logCtx) << "Committing transaction";
+
+    for (const auto &type : d->indexByType.keys()) {
+        d->typeIndex(type).commitTransaction();
+    }
+
     Q_ASSERT(d->transaction);
     d->transaction.commit();
-    d->transaction = Storage::DataStore::Transaction();
+    d->transaction = {};
 }
 
 void EntityStore::abortTransaction()
 {
     SinkTraceCtx(d->logCtx) << "Aborting transaction";
     d->transaction.abort();
-    d->transaction = Storage::DataStore::Transaction();
+    d->transaction = {};
 }
 
 bool EntityStore::hasTransaction() const
@@ -195,7 +199,7 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomain::ApplicationDoma
 
     SinkTraceCtx(d->logCtx) << "New entity " << entity;
 
-    d->typeIndex(type).add(entity.identifier(), entity, d->transaction);
+    d->typeIndex(type).add(entity.identifier(), entity, d->transaction, d->resourceContext.instanceId());
 
     //The maxRevision may have changed meanwhile if the entity created sub-entities
     const qint64 newRevision = maxRevision() + 1;
@@ -262,8 +266,8 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::Applic
 {
     SinkTraceCtx(d->logCtx) << "Modified entity: " << newEntity;
 
-    d->typeIndex(type).remove(current.identifier(), current, d->transaction);
-    d->typeIndex(type).add(newEntity.identifier(), newEntity, d->transaction);
+    d->typeIndex(type).remove(current.identifier(), current, d->transaction, d->resourceContext.instanceId());
+    d->typeIndex(type).add(newEntity.identifier(), newEntity, d->transaction, d->resourceContext.instanceId());
 
     const qint64 newRevision = DataStore::maxRevision(d->transaction) + 1;
 
@@ -304,7 +308,7 @@ bool EntityStore::remove(const QByteArray &type, const Sink::ApplicationDomain::
         return false;
     }
 
-    d->typeIndex(type).remove(current.identifier(), current, d->transaction);
+    d->typeIndex(type).remove(current.identifier(), current, d->transaction, d->resourceContext.instanceId());
 
     SinkTraceCtx(d->logCtx) << "Removed entity " << current;
 
@@ -422,7 +426,7 @@ QVector<QByteArray> EntityStore::indexLookup(const QByteArray &type, const Query
         SinkTraceCtx(d->logCtx) << "Database is not existing: " << type;
         return QVector<QByteArray>();
     }
-    return d->typeIndex(type).query(query, appliedFilters, appliedSorting, d->getTransaction());
+    return d->typeIndex(type).query(query, appliedFilters, appliedSorting, d->getTransaction(), d->resourceContext.instanceId());
 }
 
 QVector<QByteArray> EntityStore::indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value)
diff --git a/common/typeindex.cpp b/common/typeindex.cpp
index 0228ecb..f2c67a1 100644
--- a/common/typeindex.cpp
+++ b/common/typeindex.cpp
@@ -20,6 +20,7 @@
 
 #include "log.h"
 #include "index.h"
+#include "fulltextindex.h"
 #include <QDateTime>
 #include <QDataStream>
 
@@ -158,7 +159,7 @@ void TypeIndex::addPropertyWithSorting<ApplicationDomain::Reference, QDateTime>(
     addPropertyWithSorting<QByteArray, QDateTime>(property, sortProperty);
 }
 
-void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction)
+void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
 {
     for (const auto &property : mProperties) {
         const auto value = entity.getProperty(property);
@@ -172,7 +173,7 @@ void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::
         indexer(add, identifier, value, sortValue, transaction);
     }
     for (const auto &indexer : mCustomIndexer) {
-        indexer->setup(this, &transaction);
+        indexer->setup(this, &transaction, resourceInstanceId);
         if (add) {
             indexer->add(entity);
         } else {
@@ -182,14 +183,28 @@ void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::
 
 }
 
-void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction)
+void TypeIndex::commitTransaction()
 {
-    updateIndex(true, identifier, entity, transaction);
+    for (const auto &indexer : mCustomIndexer) {
+        indexer->commitTransaction();
+    }
 }
 
-void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction)
+void TypeIndex::abortTransaction()
 {
-    updateIndex(false, identifier, entity, transaction);
+    for (const auto &indexer : mCustomIndexer) {
+        indexer->abortTransaction();
+    }
+}
+
+void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
+{
+    updateIndex(true, identifier, entity, transaction, resourceInstanceId);
+}
+
+void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
+{
+    updateIndex(false, identifier, entity, transaction, resourceInstanceId);
 }
 
 static QVector<QByteArray> indexLookup(Index &index, QueryBase::Comparator filter)
@@ -211,13 +226,22 @@ static QVector<QByteArray> indexLookup(Index &index, QueryBase::Comparator filte
     return keys;
 }
 
-QVector<QByteArray> TypeIndex::query(const Sink::QueryBase &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction)
+QVector<QByteArray> TypeIndex::query(const Sink::QueryBase &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
 {
-    QVector<QByteArray> keys;
+    const auto baseFilters = query.getBaseFilters();
+    for (auto it = baseFilters.constBegin(); it != baseFilters.constEnd(); it++) {
+        if (it.value().comparator == QueryBase::Comparator::Fulltext) {
+            FulltextIndex fulltextIndex{resourceInstanceId};
+            const auto keys = fulltextIndex.lookup(it.value().value.toString());
+            appliedFilters << it.key();
+            return keys;
+        }
+    }
+
     for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) {
         if (query.hasFilter(it.key()) && query.sortProperty() == it.value()) {
             Index index(indexName(it.key(), it.value()), transaction);
-            keys << indexLookup(index, query.getFilter(it.key()));
+            const auto keys = indexLookup(index, query.getFilter(it.key()));
             appliedFilters << it.key();
             appliedSorting = it.value();
             SinkTraceCtx(mLogCtx) << "Index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys.";
@@ -227,14 +251,14 @@ QVector<QByteArray> TypeIndex::query(const Sink::QueryBase &query, QSet<QByteArr
     for (const auto &property : mProperties) {
         if (query.hasFilter(property)) {
             Index index(indexName(property), transaction);
-            keys << indexLookup(index, query.getFilter(property));
+            const auto keys = indexLookup(index, query.getFilter(property));
             appliedFilters << property;
             SinkTraceCtx(mLogCtx) << "Index lookup on " << property << " found " << keys.size() << " keys.";
             return keys;
         }
     }
     SinkTraceCtx(mLogCtx) << "No matching index";
-    return keys;
+    return {};
 }
 
 QVector<QByteArray> TypeIndex::lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction)
diff --git a/common/typeindex.h b/common/typeindex.h
index 890c3db..b8b4d52 100644
--- a/common/typeindex.h
+++ b/common/typeindex.h
@@ -71,10 +71,10 @@ public:
         mCustomIndexer << CustomIndexer::Ptr::create();
     }
 
-    void add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction);
-    void remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction);
+    void add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
+    void remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
 
-    QVector<QByteArray> query(const Sink::QueryBase &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction);
+    QVector<QByteArray> query(const Sink::QueryBase &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
     QVector<QByteArray> lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction);
 
     template <typename Left, typename Right>
@@ -104,10 +104,13 @@ public:
     template <typename LeftType, typename RightType>
     void unindex(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction);
 
+    void commitTransaction();
+    void abortTransaction();
+
 
 private:
     friend class Sink::Storage::EntityStore;
-    void updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction);
+    void updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
     QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const;
     Sink::Log::Context mLogCtx;
     QByteArray mType;
diff --git a/synchronizer/CMakeLists.txt b/synchronizer/CMakeLists.txt
index 2f8b128..9c422e6 100644
--- a/synchronizer/CMakeLists.txt
+++ b/synchronizer/CMakeLists.txt
@@ -10,6 +10,7 @@ add_executable(${PROJECT_NAME} ${sinksynchronizer_SRCS})
 target_link_libraries(${PROJECT_NAME}
     sink
     Qt5::Core
+    Qt5::Gui
     Qt5::Network
     KAsync
     ${CMAKE_DL_LIBS}
diff --git a/synchronizer/main.cpp b/synchronizer/main.cpp
index 3f79207..f4bac73 100644
--- a/synchronizer/main.cpp
+++ b/synchronizer/main.cpp
@@ -17,7 +17,7 @@
  *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  */
 
-#include <QCoreApplication>
+#include <QGuiApplication>
 #include <QLockFile>
 #include <QDir>
 
@@ -226,7 +226,7 @@ int main(int argc, char *argv[])
 
     qInstallMessageHandler(qtMessageHandler);
 
-    QCoreApplication app(argc, argv);
+    QGuiApplication app(argc, argv);
     app.setQuitLockEnabled(false);
 
     QByteArrayList arguments;
diff --git a/tests/dummyresourcewritebenchmark.cpp b/tests/dummyresourcewritebenchmark.cpp
index 07f57f6..e0ec503 100644
--- a/tests/dummyresourcewritebenchmark.cpp
+++ b/tests/dummyresourcewritebenchmark.cpp
@@ -20,6 +20,7 @@
 #include "hawd/formatter.h"
 
 #include "event_generated.h"
+#include "mail_generated.h"
 #include "entity_generated.h"
 #include "metadata_generated.h"
 #include "createentity_generated.h"
@@ -27,38 +28,36 @@
 #include "getrssusage.h"
 #include "utils.h"
 
+#include <KMime/Message>
+
 static QByteArray createEntityBuffer(size_t attachmentSize, int &bufferSize)
 {
-    uint8_t rawData[attachmentSize];
     flatbuffers::FlatBufferBuilder eventFbb;
     eventFbb.Clear();
     {
-        uint8_t *rawDataPtr = Q_NULLPTR;
-        auto data = eventFbb.CreateUninitializedVector<uint8_t>(attachmentSize, &rawDataPtr);
-        auto summary = eventFbb.CreateString("summary");
-        Sink::ApplicationDomain::Buffer::EventBuilder eventBuilder(eventFbb);
-        eventBuilder.add_summary(summary);
-        eventBuilder.add_attachment(data);
-        auto eventLocation = eventBuilder.Finish();
-        Sink::ApplicationDomain::Buffer::FinishEventBuffer(eventFbb, eventLocation);
-        memcpy((void *)rawDataPtr, rawData, attachmentSize);
-    }
 
-    flatbuffers::FlatBufferBuilder localFbb;
-    {
-        auto uid = localFbb.CreateString("testuid");
-        auto localBuilder = Sink::ApplicationDomain::Buffer::EventBuilder(localFbb);
-        localBuilder.add_uid(uid);
-        auto location = localBuilder.Finish();
-        Sink::ApplicationDomain::Buffer::FinishEventBuffer(localFbb, location);
+        auto msg = KMime::Message::Ptr::create();
+        msg->subject()->from7BitString("Some subject");
+        msg->setBody("This is the body now.");
+        msg->assemble();
+
+        const auto data = msg->encodedContent();
+
+        auto summary = eventFbb.CreateString("summary");
+        auto mimeMessage = eventFbb.CreateString(data.constData(), data.length());
+        Sink::ApplicationDomain::Buffer::MailBuilder eventBuilder(eventFbb);
+        eventBuilder.add_subject(summary);
+        eventBuilder.add_messageId(summary);
+        eventBuilder.add_mimeMessage(mimeMessage);
+        Sink::ApplicationDomain::Buffer::FinishMailBuffer(eventFbb, eventBuilder.Finish());
     }
 
     flatbuffers::FlatBufferBuilder entityFbb;
-    Sink::EntityBuffer::assembleEntityBuffer(entityFbb, 0, 0, eventFbb.GetBufferPointer(), eventFbb.GetSize(), localFbb.GetBufferPointer(), localFbb.GetSize());
+    Sink::EntityBuffer::assembleEntityBuffer(entityFbb, 0, 0, 0, 0, eventFbb.GetBufferPointer(), eventFbb.GetSize());
     bufferSize = entityFbb.GetSize();
 
     flatbuffers::FlatBufferBuilder fbb;
-    auto type = fbb.CreateString(Sink::ApplicationDomain::getTypeName<Sink::ApplicationDomain::Event>().toStdString().data());
+    auto type = fbb.CreateString(Sink::ApplicationDomain::getTypeName<Sink::ApplicationDomain::Mail>().toStdString().data());
     auto delta = fbb.CreateVector<uint8_t>(entityFbb.GetBufferPointer(), entityFbb.GetSize());
     Sink::Commands::CreateEntityBuilder builder(fbb);
     builder.add_domainType(type);
@@ -263,10 +262,7 @@ private slots:
 
     void runBenchmarks()
     {
-        writeInProcess(1000, mTimeStamp);
-        writeInProcess(2000, mTimeStamp);
         writeInProcess(5000, mTimeStamp);
-        writeInProcess(20000, mTimeStamp);
     }
 
     void ensureUsedMemoryRemainsStable()
diff --git a/tests/querytest.cpp b/tests/querytest.cpp
index 1584c48..ec6438d 100644
--- a/tests/querytest.cpp
+++ b/tests/querytest.cpp
@@ -14,6 +14,8 @@
 #include "testutils.h"
 #include "applicationdomaintype.h"
 
+#include <KMime/Message>
+
 using namespace Sink;
 using namespace Sink::ApplicationDomain;
 
@@ -1254,6 +1256,94 @@ private slots:
         VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1")));
     }
 
+    void testMailFulltextSubject()
+    {
+        // Setup
+        {
+            auto msg = KMime::Message::Ptr::create();
+            msg->subject()->from7BitString("Subject To Search");
+            msg->setBody("This is the searchable body.");
+            msg->assemble();
+            {
+                Mail mail("sink.dummy.instance1");
+                mail.setExtractedMessageId("test1");
+                mail.setExtractedSubject("Subject To Search");
+                mail.setFolder("folder1");
+                mail.setMimeMessage(msg->encodedContent());
+                VERIFYEXEC(Sink::Store::create<Mail>(mail));
+            }
+            {
+                Mail mail("sink.dummy.instance1");
+                mail.setExtractedMessageId("test2");
+                mail.setFolder("folder2");
+                mail.setExtractedSubject("Stuff");
+                VERIFYEXEC(Sink::Store::create<Mail>(mail));
+            }
+            VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1"));
+        }
+
+        // Test
+        {
+            Sink::Query query;
+            query.resourceFilter("sink.dummy.instance1");
+            query.filter<Mail::Subject>(QueryBase::Comparator(QString("Subject To Search"), QueryBase::Comparator::Fulltext));
+            auto model = Sink::Store::loadModel<Mail>(query);
+            QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool());
+            QCOMPARE(model->rowCount(), 1);
+        }
+        {
+            Sink::Query query;
+            query.resourceFilter("sink.dummy.instance1");
+            query.filter<Mail::Subject>(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext));
+            auto result = Sink::Store::read<Mail>(query);
+            QCOMPARE(result.size(), 1);
+        }
+        {
+            Sink::Query query;
+            query.resourceFilter("sink.dummy.instance1");
+            query.filter<Mail::Subject>(QueryBase::Comparator(QString("Search"), QueryBase::Comparator::Fulltext));
+            auto result = Sink::Store::read<Mail>(query);
+            QCOMPARE(result.size(), 1);
+        }
+        {
+            Sink::Query query;
+            query.resourceFilter("sink.dummy.instance1");
+            query.filter<Mail::Subject>(QueryBase::Comparator(QString("search"), QueryBase::Comparator::Fulltext));
+            auto result = Sink::Store::read<Mail>(query);
+            QCOMPARE(result.size(), 1);
+        }
+        {
+            Sink::Query query;
+            query.resourceFilter("sink.dummy.instance1");
+            query.filter<Mail::Subject>(QueryBase::Comparator(QString("sear*"), QueryBase::Comparator::Fulltext));
+            auto result = Sink::Store::read<Mail>(query);
+            QCOMPARE(result.size(), 1);
+        }
+        {
+            Sink::Query query;
+            query.resourceFilter("sink.dummy.instance1");
+            query.filter<Mail::MimeMessage>(QueryBase::Comparator(QString("searchable"), QueryBase::Comparator::Fulltext));
+            auto result = Sink::Store::read<Mail>(query);
+            QCOMPARE(result.size(), 1);
+        }
+        {
+            Sink::Query query;
+            query.resourceFilter("sink.dummy.instance1");
+            query.filter<Mail::Subject>(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext));
+            query.filter<Mail::Folder>("folder1");
+            auto result = Sink::Store::read<Mail>(query);
+            QCOMPARE(result.size(), 1);
+        }
+        {
+            Sink::Query query;
+            query.resourceFilter("sink.dummy.instance1");
+            query.filter<Mail::Subject>(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext));
+            query.filter<Mail::Folder>("folder2");
+            auto result = Sink::Store::read<Mail>(query);
+            QCOMPARE(result.size(), 0);
+        }
+    }
+
 };
 
 QTEST_MAIN(QueryTest)