From 6051c1247cde61bcc8e483eb4166e5a297c0ecc6 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 13 Oct 2016 18:38:35 +0200 Subject: Xapian based fulltext indexing This cuts into the sync performance by about 40%, but gives us fast fulltext searching for all local content. --- common/CMakeLists.txt | 3 + common/datastorequery.cpp | 4 + common/definitions.cpp | 2 +- common/domain/typeimplementations.cpp | 4 +- common/fulltextindex.cpp | 149 ++++++++++++++++++++++++++++++++++ common/fulltextindex.h | 39 +++++++++ common/indexer.cpp | 3 +- common/indexer.h | 6 +- common/mail/fulltextindexer.cpp | 64 +++++++++++++++ common/mail/fulltextindexer.h | 39 +++++++++ common/mail/threadindexer.cpp | 5 -- common/mail/threadindexer.h | 1 - common/mailpreprocessor.cpp | 40 ++++++++- common/mailpreprocessor.h | 1 - common/query.cpp | 3 + common/query.h | 3 +- common/storage/entitystore.cpp | 22 +++-- common/typeindex.cpp | 46 ++++++++--- common/typeindex.h | 11 ++- 19 files changed, 406 insertions(+), 39 deletions(-) create mode 100644 common/fulltextindex.cpp create mode 100644 common/fulltextindex.h create mode 100644 common/mail/fulltextindexer.cpp create mode 100644 common/mail/fulltextindexer.h (limited to 'common') diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index ec83f6f..76579dd 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -75,11 +75,13 @@ set(command_SRCS storage/entitystore.cpp indexer.cpp mail/threadindexer.cpp + mail/fulltextindexer.cpp notification.cpp commandprocessor.cpp inspector.cpp propertyparser.cpp utils.cpp + fulltextindex.cpp ${storage_SRCS}) add_library(${PROJECT_NAME} SHARED ${command_SRCS}) @@ -127,6 +129,7 @@ PRIVATE Qt5::Gui KF5::Mime KF5::Contacts + ${XAPIAN_LIBRARIES} ) install(TARGETS ${PROJECT_NAME} EXPORT SinkTargets ${KDE_INSTALL_TARGETS_DEFAULT_ARGS} ${LIBRARY_NAMELINK} ) diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp index 870daf8..3218d1a 100644 --- a/common/datastorequery.cpp +++ b/common/datastorequery.cpp @@ -160,6 +160,10 @@ public: for (const auto &filterProperty : propertyFilter.keys()) { const auto property = entity.getProperty(filterProperty); const auto comparator = propertyFilter.value(filterProperty); + //We can't deal with a fulltext filter + if (comparator.comparator == QueryBase::Comparator::Fulltext) { + continue; + } if (!comparator.matches(property)) { SinkTraceCtx(mDatastore->mLogCtx) << "Filtering entity due to property mismatch on filter: " << entity.identifier() << "Property: " << filterProperty << property << " Filter:" << comparator.value; return false; diff --git a/common/definitions.cpp b/common/definitions.cpp index b22137a..642b68c 100644 --- a/common/definitions.cpp +++ b/common/definitions.cpp @@ -90,5 +90,5 @@ QString Sink::resourceStorageLocation(const QByteArray &resourceInstanceIdentifi qint64 Sink::latestDatabaseVersion() { - return 1; + return 2; } diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp index 47a9cf7..29da7ea 100644 --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -27,6 +27,7 @@ #include "entitybuffer.h" #include "entity_generated.h" #include "mail/threadindexer.h" +#include "mail/fulltextindexer.h" #include "domainadaptor.h" #include "typeimplementations_p.h" @@ -45,7 +46,8 @@ typedef IndexConfig, SecondaryIndex, SecondaryIndex, - CustomSecondaryIndex + CustomSecondaryIndex, + CustomSecondaryIndex > MailIndexConfig; typedef IndexConfig + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +//xapian.h needs to be included first to build +#include +#include "fulltextindex.h" + +#include +#include + +#include "log.h" +#include "definitions.h" + +FulltextIndex::FulltextIndex(const QByteArray &resourceInstanceIdentifier, Sink::Storage::DataStore::AccessMode accessMode) + : mName("fulltext"), + mDbPath{QFile::encodeName(Sink::resourceStorageLocation(resourceInstanceIdentifier) + '/' + "fulltext")} +{ + try { + if (QDir{}.mkpath(mDbPath)) { + if (accessMode == Sink::Storage::DataStore::ReadWrite) { + mDb = new Xapian::WritableDatabase(mDbPath.toStdString(), Xapian::DB_CREATE_OR_OPEN); + } else { + mDb = new Xapian::Database(mDbPath.toStdString(), Xapian::DB_OPEN); + } + } else { + SinkError() << "Failed to open database" << mDbPath; + } + } catch (const Xapian::DatabaseError& e) { + SinkError() << "Failed to open database" << mDbPath << ":" << QString::fromStdString(e.get_msg()); + } +} + +FulltextIndex::~FulltextIndex() +{ + delete mDb; +} + +static std::string idTerm(const QByteArray &key) +{ + return "Q" + key.toStdString(); +} + +void FulltextIndex::add(const QByteArray &key, const QString &value) +{ + add(key, {{{}, value}}); +} + +void FulltextIndex::add(const QByteArray &key, const QList> &values) +{ + if (!mDb) { + return; + } + Xapian::TermGenerator generator; + Xapian::Document document; + generator.set_document(document); + + for (const auto &entry : values) { + if (!entry.second.isEmpty()) { + generator.index_text(entry.second.toStdString()); + } + } + document.add_value(0, key.toStdString()); + + const auto idterm = idTerm(key); + document.add_boolean_term(idterm); + + writableDatabase()->replace_document(idterm, document); +} + +void FulltextIndex::commitTransaction() +{ + if (mHasTransactionOpen) { + Q_ASSERT(mDb); + writableDatabase()->commit_transaction(); + mHasTransactionOpen = false; + } +} + +void FulltextIndex::abortTransaction() +{ + if (mHasTransactionOpen) { + Q_ASSERT(mDb); + writableDatabase()->cancel_transaction(); + mHasTransactionOpen = false; + } +} + +Xapian::WritableDatabase* FulltextIndex::writableDatabase() +{ + Q_ASSERT(dynamic_cast(mDb)); + auto db = static_cast(mDb); + if (!mHasTransactionOpen) { + db->begin_transaction(); + mHasTransactionOpen = true; + } + return db; +} + +void FulltextIndex::remove(const QByteArray &key) +{ + if (!mDb) { + return; + } + writableDatabase()->delete_document(idTerm(key)); +} + +QVector FulltextIndex::lookup(const QString &searchTerm) +{ + if (!mDb) { + return {}; + } + QVector results; + + try { + Xapian::QueryParser parser; + auto query = parser.parse_query(searchTerm.toStdString(), Xapian::QueryParser::FLAG_WILDCARD|Xapian::QueryParser::FLAG_PHRASE|Xapian::QueryParser::FLAG_BOOLEAN|Xapian::QueryParser::FLAG_LOVEHATE); + Xapian::Enquire enquire(*mDb); + enquire.set_query(query); + + auto limit = 1000; + Xapian::MSet mset = enquire.get_mset(0, limit); + Xapian::MSetIterator it = mset.begin(); + for (;it != mset.end(); it++) { + auto doc = it.get_document(); + const auto data = doc.get_value(0); + results << QByteArray{data.c_str(), int(data.length())}; + } + } + catch (const Xapian::Error &error) { + // Nothing to do, move along + } + return results; +} + diff --git a/common/fulltextindex.h b/common/fulltextindex.h new file mode 100644 index 0000000..e06f29d --- /dev/null +++ b/common/fulltextindex.h @@ -0,0 +1,39 @@ +#pragma once + +#include "sink_export.h" + +#include +#include +#include +#include +#include "storage.h" +#include "log.h" + +namespace Xapian { + class Database; + class WritableDatabase; +}; + +class SINK_EXPORT FulltextIndex +{ +public: + FulltextIndex(const QByteArray &resourceInstanceIdentifier, Sink::Storage::DataStore::AccessMode mode = Sink::Storage::DataStore::ReadOnly); + ~FulltextIndex(); + + void add(const QByteArray &key, const QString &value); + void add(const QByteArray &key, const QList> &values); + void remove(const QByteArray &key); + + void commitTransaction(); + void abortTransaction(); + + QVector lookup(const QString &key); + +private: + Xapian::WritableDatabase* writableDatabase(); + Q_DISABLE_COPY(FulltextIndex); + Xapian::Database *mDb{nullptr}; + QString mName; + QString mDbPath; + bool mHasTransactionOpen{false}; +}; diff --git a/common/indexer.cpp b/common/indexer.cpp index 1b223b3..c18170c 100644 --- a/common/indexer.cpp +++ b/common/indexer.cpp @@ -20,10 +20,11 @@ using namespace Sink; -void Indexer::setup(TypeIndex *index, Storage::DataStore::Transaction *transaction) +void Indexer::setup(TypeIndex *index, Storage::DataStore::Transaction *transaction, const QByteArray &resourceId) { mTypeIndex = index; mTransaction = transaction; + mResourceInstanceIdentifier = resourceId; } Storage::DataStore::Transaction &Indexer::transaction() diff --git a/common/indexer.h b/common/indexer.h index 26887fb..f0b32f5 100644 --- a/common/indexer.h +++ b/common/indexer.h @@ -33,16 +33,18 @@ public: virtual ~Indexer() = default; typedef QSharedPointer Ptr; virtual void add(const ApplicationDomain::ApplicationDomainType &entity) = 0; - virtual void modify(const ApplicationDomain::ApplicationDomainType &old, const ApplicationDomain::ApplicationDomainType &entity) = 0; virtual void remove(const ApplicationDomain::ApplicationDomainType &entity) = 0; + virtual void commitTransaction() {}; + virtual void abortTransaction() {}; protected: Storage::DataStore::Transaction &transaction(); TypeIndex &index(); + QByteArray mResourceInstanceIdentifier; private: friend class ::TypeIndex; - void setup(TypeIndex *, Storage::DataStore::Transaction *); + void setup(TypeIndex *, Storage::DataStore::Transaction *, const QByteArray &resourceId); Storage::DataStore::Transaction *mTransaction; TypeIndex *mTypeIndex; }; diff --git a/common/mail/fulltextindexer.cpp b/common/mail/fulltextindexer.cpp new file mode 100644 index 0000000..a980752 --- /dev/null +++ b/common/mail/fulltextindexer.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "fulltextindexer.h" + +#include "typeindex.h" +#include "fulltextindex.h" +#include "log.h" +#include "utils.h" + +using namespace Sink; +using namespace Sink::ApplicationDomain; + + +void FulltextIndexer::add(const ApplicationDomain::ApplicationDomainType &entity) +{ + if (!index) { + index.reset(new FulltextIndex{mResourceInstanceIdentifier, Storage::DataStore::ReadWrite}); + } + index->add(entity.identifier(), entity.getProperty("index").value>>()); +} + +void FulltextIndexer::remove(const ApplicationDomain::ApplicationDomainType &entity) +{ + if (!index) { + index.reset(new FulltextIndex{mResourceInstanceIdentifier, Storage::DataStore::ReadWrite}); + } + index->remove(entity.identifier()); +} + +void FulltextIndexer::commitTransaction() +{ + if (index) { + index->commitTransaction(); + } +} + +void FulltextIndexer::abortTransaction() +{ + if (index) { + index->abortTransaction(); + } +} + +QMap FulltextIndexer::databases() +{ + return {}; +} + diff --git a/common/mail/fulltextindexer.h b/common/mail/fulltextindexer.h new file mode 100644 index 0000000..ef41455 --- /dev/null +++ b/common/mail/fulltextindexer.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2015 Christian Mollekopf + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include "indexer.h" + +class FulltextIndex; +namespace Sink { + +class FulltextIndexer : public Indexer +{ +public: + typedef QSharedPointer Ptr; + virtual void add(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE; + virtual void remove(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE; + virtual void commitTransaction() Q_DECL_OVERRIDE; + virtual void abortTransaction() Q_DECL_OVERRIDE; + static QMap databases(); +private: + QSharedPointer index; +}; + +} diff --git a/common/mail/threadindexer.cpp b/common/mail/threadindexer.cpp index 2e6a6e7..fb47118 100644 --- a/common/mail/threadindexer.cpp +++ b/common/mail/threadindexer.cpp @@ -98,11 +98,6 @@ void ThreadIndexer::add(const ApplicationDomain::ApplicationDomainType &entity) updateThreadingIndex(entity.identifier(), entity, transaction()); } -void ThreadIndexer::modify(const ApplicationDomain::ApplicationDomainType &old, const ApplicationDomain::ApplicationDomainType &entity) -{ - -} - void ThreadIndexer::remove(const ApplicationDomain::ApplicationDomainType &entity) { auto messageId = entity.getProperty(Mail::MessageId::name); diff --git a/common/mail/threadindexer.h b/common/mail/threadindexer.h index 60d0863..b2e939a 100644 --- a/common/mail/threadindexer.h +++ b/common/mail/threadindexer.h @@ -27,7 +27,6 @@ class ThreadIndexer : public Indexer public: typedef QSharedPointer Ptr; virtual void add(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE; - virtual void modify(const ApplicationDomain::ApplicationDomainType &old, const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE; virtual void remove(const ApplicationDomain::ApplicationDomainType &entity) Q_DECL_OVERRIDE; static QMap databases(); private: diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp index 8f5a77d..58cb15b 100644 --- a/common/mailpreprocessor.cpp +++ b/common/mailpreprocessor.cpp @@ -21,9 +21,11 @@ #include #include +#include #include #include "pipeline.h" +#include "fulltextindex.h" #include "definitions.h" #include "applicationdomaintype.h" @@ -45,13 +47,34 @@ static QList getContactList(const KMime: return list; } +static QList> processPart(KMime::Content* content) +{ + if (KMime::Headers::ContentType* type = content->contentType(false)) { + if (type->isMultipart() && !type->isSubtype("encrypted")) { + QList> list; + for (const auto c : content->contents()) { + list << processPart(c); + } + return list; + } else if (type->isHTMLText()) { + // Only get HTML content, if no plain text content + QTextDocument doc; + doc.setHtml(content->decodedText()); + return {{{}, {doc.toPlainText()}}}; + } else if (type->isEmpty()) { + return {{{}, {content->decodedText()}}}; + } + } + return {}; +} + void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data) { if (data.isEmpty()) { return; } auto msg = KMime::Message::Ptr(new KMime::Message); - msg->setHead(KMime::CRLFtoLF(data)); + msg->setContent(KMime::CRLFtoLF(data)); msg->parse(); if (!msg) { return; @@ -103,6 +126,20 @@ void MailPropertyExtractor::updatedIndexedProperties(Sink::ApplicationDomain::Ma if (!parentMessageId.isEmpty()) { mail.setExtractedParentMessageId(parentMessageId); } + QList> contentToIndex; + contentToIndex.append({{}, msg->subject()->asUnicodeString()}); + if (KMime::Content* mainBody = msg->mainBodyPart("text/plain")) { + contentToIndex.append({{}, mainBody->decodedText()}); + } else { + contentToIndex << processPart(msg.data()); + } + contentToIndex.append({{}, msg->from(true)->asUnicodeString()}); + contentToIndex.append({{}, msg->to(true)->asUnicodeString()}); + contentToIndex.append({{}, msg->cc(true)->asUnicodeString()}); + contentToIndex.append({{}, msg->bcc(true)->asUnicodeString()}); + + //Prepare content for indexing; + mail.setProperty("index", QVariant::fromValue(contentToIndex)); } void MailPropertyExtractor::newEntity(Sink::ApplicationDomain::Mail &mail) @@ -114,4 +151,3 @@ void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail & { updatedIndexedProperties(newMail, newMail.getMimeMessage()); } - diff --git a/common/mailpreprocessor.h b/common/mailpreprocessor.h index d2e79ca..c0eacaf 100644 --- a/common/mailpreprocessor.h +++ b/common/mailpreprocessor.h @@ -29,4 +29,3 @@ public: protected: static void updatedIndexedProperties(Sink::ApplicationDomain::Mail &mail, const QByteArray &data); }; - diff --git a/common/query.cpp b/common/query.cpp index 3dc8f99..5f6d095 100644 --- a/common/query.cpp +++ b/common/query.cpp @@ -34,6 +34,8 @@ QDebug operator<<(QDebug dbg, const Sink::QueryBase::Comparator &c) dbg.nospace() << "contains " << c.value; } else if (c.comparator == Sink::Query::Comparator::In) { dbg.nospace() << "in " << c.value; + } else if (c.comparator == Sink::Query::Comparator::Fulltext) { + dbg.nospace() << "fulltext contains " << c.value; } else { dbg.nospace() << "unknown comparator: " << c.value; } @@ -169,6 +171,7 @@ bool QueryBase::Comparator::matches(const QVariant &v) const return false; } return value.value().contains(v.toByteArray()); + case Fulltext: case Invalid: default: break; diff --git a/common/query.h b/common/query.h index 5b37cdd..1e7b41d 100644 --- a/common/query.h +++ b/common/query.h @@ -35,7 +35,8 @@ public: Invalid, Equals, Contains, - In + In, + Fulltext }; Comparator(); diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 4ad3eaf..8fbc2ad 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -162,23 +162,27 @@ void EntityStore::startTransaction(Sink::Storage::DataStore::AccessMode accessMo { SinkTraceCtx(d->logCtx) << "Starting transaction: " << accessMode; Q_ASSERT(!d->transaction); - Sink::Storage::DataStore store(Sink::storageLocation(), dbLayout(d->resourceContext.instanceId()), accessMode); - d->transaction = store.createTransaction(accessMode); + d->transaction = Sink::Storage::DataStore(Sink::storageLocation(), dbLayout(d->resourceContext.instanceId()), accessMode).createTransaction(accessMode); } void EntityStore::commitTransaction() { SinkTraceCtx(d->logCtx) << "Committing transaction"; + + for (const auto &type : d->indexByType.keys()) { + d->typeIndex(type).commitTransaction(); + } + Q_ASSERT(d->transaction); d->transaction.commit(); - d->transaction = Storage::DataStore::Transaction(); + d->transaction = {}; } void EntityStore::abortTransaction() { SinkTraceCtx(d->logCtx) << "Aborting transaction"; d->transaction.abort(); - d->transaction = Storage::DataStore::Transaction(); + d->transaction = {}; } bool EntityStore::hasTransaction() const @@ -195,7 +199,7 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomain::ApplicationDoma SinkTraceCtx(d->logCtx) << "New entity " << entity; - d->typeIndex(type).add(entity.identifier(), entity, d->transaction); + d->typeIndex(type).add(entity.identifier(), entity, d->transaction, d->resourceContext.instanceId()); //The maxRevision may have changed meanwhile if the entity created sub-entities const qint64 newRevision = maxRevision() + 1; @@ -262,8 +266,8 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::Applic { SinkTraceCtx(d->logCtx) << "Modified entity: " << newEntity; - d->typeIndex(type).remove(current.identifier(), current, d->transaction); - d->typeIndex(type).add(newEntity.identifier(), newEntity, d->transaction); + d->typeIndex(type).remove(current.identifier(), current, d->transaction, d->resourceContext.instanceId()); + d->typeIndex(type).add(newEntity.identifier(), newEntity, d->transaction, d->resourceContext.instanceId()); const qint64 newRevision = DataStore::maxRevision(d->transaction) + 1; @@ -304,7 +308,7 @@ bool EntityStore::remove(const QByteArray &type, const Sink::ApplicationDomain:: return false; } - d->typeIndex(type).remove(current.identifier(), current, d->transaction); + d->typeIndex(type).remove(current.identifier(), current, d->transaction, d->resourceContext.instanceId()); SinkTraceCtx(d->logCtx) << "Removed entity " << current; @@ -422,7 +426,7 @@ QVector EntityStore::indexLookup(const QByteArray &type, const Query SinkTraceCtx(d->logCtx) << "Database is not existing: " << type; return QVector(); } - return d->typeIndex(type).query(query, appliedFilters, appliedSorting, d->getTransaction()); + return d->typeIndex(type).query(query, appliedFilters, appliedSorting, d->getTransaction(), d->resourceContext.instanceId()); } QVector EntityStore::indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value) diff --git a/common/typeindex.cpp b/common/typeindex.cpp index 0228ecb..f2c67a1 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -20,6 +20,7 @@ #include "log.h" #include "index.h" +#include "fulltextindex.h" #include #include @@ -158,7 +159,7 @@ void TypeIndex::addPropertyWithSorting( addPropertyWithSorting(property, sortProperty); } -void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction) +void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { for (const auto &property : mProperties) { const auto value = entity.getProperty(property); @@ -172,7 +173,7 @@ void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink:: indexer(add, identifier, value, sortValue, transaction); } for (const auto &indexer : mCustomIndexer) { - indexer->setup(this, &transaction); + indexer->setup(this, &transaction, resourceInstanceId); if (add) { indexer->add(entity); } else { @@ -182,14 +183,28 @@ void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink:: } -void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction) +void TypeIndex::commitTransaction() { - updateIndex(true, identifier, entity, transaction); + for (const auto &indexer : mCustomIndexer) { + indexer->commitTransaction(); + } } -void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction) +void TypeIndex::abortTransaction() { - updateIndex(false, identifier, entity, transaction); + for (const auto &indexer : mCustomIndexer) { + indexer->abortTransaction(); + } +} + +void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) +{ + updateIndex(true, identifier, entity, transaction, resourceInstanceId); +} + +void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) +{ + updateIndex(false, identifier, entity, transaction, resourceInstanceId); } static QVector indexLookup(Index &index, QueryBase::Comparator filter) @@ -211,13 +226,22 @@ static QVector indexLookup(Index &index, QueryBase::Comparator filte return keys; } -QVector TypeIndex::query(const Sink::QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction) +QVector TypeIndex::query(const Sink::QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { - QVector keys; + const auto baseFilters = query.getBaseFilters(); + for (auto it = baseFilters.constBegin(); it != baseFilters.constEnd(); it++) { + if (it.value().comparator == QueryBase::Comparator::Fulltext) { + FulltextIndex fulltextIndex{resourceInstanceId}; + const auto keys = fulltextIndex.lookup(it.value().value.toString()); + appliedFilters << it.key(); + return keys; + } + } + for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { if (query.hasFilter(it.key()) && query.sortProperty() == it.value()) { Index index(indexName(it.key(), it.value()), transaction); - keys << indexLookup(index, query.getFilter(it.key())); + const auto keys = indexLookup(index, query.getFilter(it.key())); appliedFilters << it.key(); appliedSorting = it.value(); SinkTraceCtx(mLogCtx) << "Index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys."; @@ -227,14 +251,14 @@ QVector TypeIndex::query(const Sink::QueryBase &query, QSet TypeIndex::lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) diff --git a/common/typeindex.h b/common/typeindex.h index 890c3db..b8b4d52 100644 --- a/common/typeindex.h +++ b/common/typeindex.h @@ -71,10 +71,10 @@ public: mCustomIndexer << CustomIndexer::Ptr::create(); } - void add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction); - void remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction); + void add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); + void remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); - QVector query(const Sink::QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction); + QVector query(const Sink::QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); QVector lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction); template @@ -104,10 +104,13 @@ public: template void unindex(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction); + void commitTransaction(); + void abortTransaction(); + private: friend class Sink::Storage::EntityStore; - void updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction); + void updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const; Sink::Log::Context mLogCtx; QByteArray mType; -- cgit v1.2.3