From 6051c1247cde61bcc8e483eb4166e5a297c0ecc6 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Thu, 13 Oct 2016 18:38:35 +0200 Subject: Xapian based fulltext indexing This cuts into the sync performance by about 40%, but gives us fast fulltext searching for all local content. --- tests/dummyresourcewritebenchmark.cpp | 42 ++++++++-------- tests/querytest.cpp | 90 +++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 23 deletions(-) (limited to 'tests') diff --git a/tests/dummyresourcewritebenchmark.cpp b/tests/dummyresourcewritebenchmark.cpp index 07f57f6..e0ec503 100644 --- a/tests/dummyresourcewritebenchmark.cpp +++ b/tests/dummyresourcewritebenchmark.cpp @@ -20,6 +20,7 @@ #include "hawd/formatter.h" #include "event_generated.h" +#include "mail_generated.h" #include "entity_generated.h" #include "metadata_generated.h" #include "createentity_generated.h" @@ -27,38 +28,36 @@ #include "getrssusage.h" #include "utils.h" +#include + static QByteArray createEntityBuffer(size_t attachmentSize, int &bufferSize) { - uint8_t rawData[attachmentSize]; flatbuffers::FlatBufferBuilder eventFbb; eventFbb.Clear(); { - uint8_t *rawDataPtr = Q_NULLPTR; - auto data = eventFbb.CreateUninitializedVector(attachmentSize, &rawDataPtr); - auto summary = eventFbb.CreateString("summary"); - Sink::ApplicationDomain::Buffer::EventBuilder eventBuilder(eventFbb); - eventBuilder.add_summary(summary); - eventBuilder.add_attachment(data); - auto eventLocation = eventBuilder.Finish(); - Sink::ApplicationDomain::Buffer::FinishEventBuffer(eventFbb, eventLocation); - memcpy((void *)rawDataPtr, rawData, attachmentSize); - } - flatbuffers::FlatBufferBuilder localFbb; - { - auto uid = localFbb.CreateString("testuid"); - auto localBuilder = Sink::ApplicationDomain::Buffer::EventBuilder(localFbb); - localBuilder.add_uid(uid); - auto location = localBuilder.Finish(); - Sink::ApplicationDomain::Buffer::FinishEventBuffer(localFbb, location); + auto msg = KMime::Message::Ptr::create(); + msg->subject()->from7BitString("Some subject"); + msg->setBody("This is the body now."); + msg->assemble(); + + const auto data = msg->encodedContent(); + + auto summary = eventFbb.CreateString("summary"); + auto mimeMessage = eventFbb.CreateString(data.constData(), data.length()); + Sink::ApplicationDomain::Buffer::MailBuilder eventBuilder(eventFbb); + eventBuilder.add_subject(summary); + eventBuilder.add_messageId(summary); + eventBuilder.add_mimeMessage(mimeMessage); + Sink::ApplicationDomain::Buffer::FinishMailBuffer(eventFbb, eventBuilder.Finish()); } flatbuffers::FlatBufferBuilder entityFbb; - Sink::EntityBuffer::assembleEntityBuffer(entityFbb, 0, 0, eventFbb.GetBufferPointer(), eventFbb.GetSize(), localFbb.GetBufferPointer(), localFbb.GetSize()); + Sink::EntityBuffer::assembleEntityBuffer(entityFbb, 0, 0, 0, 0, eventFbb.GetBufferPointer(), eventFbb.GetSize()); bufferSize = entityFbb.GetSize(); flatbuffers::FlatBufferBuilder fbb; - auto type = fbb.CreateString(Sink::ApplicationDomain::getTypeName().toStdString().data()); + auto type = fbb.CreateString(Sink::ApplicationDomain::getTypeName().toStdString().data()); auto delta = fbb.CreateVector(entityFbb.GetBufferPointer(), entityFbb.GetSize()); Sink::Commands::CreateEntityBuilder builder(fbb); builder.add_domainType(type); @@ -263,10 +262,7 @@ private slots: void runBenchmarks() { - writeInProcess(1000, mTimeStamp); - writeInProcess(2000, mTimeStamp); writeInProcess(5000, mTimeStamp); - writeInProcess(20000, mTimeStamp); } void ensureUsedMemoryRemainsStable() diff --git a/tests/querytest.cpp b/tests/querytest.cpp index 1584c48..ec6438d 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -14,6 +14,8 @@ #include "testutils.h" #include "applicationdomaintype.h" +#include + using namespace Sink; using namespace Sink::ApplicationDomain; @@ -1254,6 +1256,94 @@ private slots: VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } + void testMailFulltextSubject() + { + // Setup + { + auto msg = KMime::Message::Ptr::create(); + msg->subject()->from7BitString("Subject To Search"); + msg->setBody("This is the searchable body."); + msg->assemble(); + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedMessageId("test1"); + mail.setExtractedSubject("Subject To Search"); + mail.setFolder("folder1"); + mail.setMimeMessage(msg->encodedContent()); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedMessageId("test2"); + mail.setFolder("folder2"); + mail.setExtractedSubject("Stuff"); + VERIFYEXEC(Sink::Store::create(mail)); + } + VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); + } + + // Test + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QString("Subject To Search"), QueryBase::Comparator::Fulltext)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 1); + } + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); + auto result = Sink::Store::read(query); + QCOMPARE(result.size(), 1); + } + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QString("Search"), QueryBase::Comparator::Fulltext)); + auto result = Sink::Store::read(query); + QCOMPARE(result.size(), 1); + } + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QString("search"), QueryBase::Comparator::Fulltext)); + auto result = Sink::Store::read(query); + QCOMPARE(result.size(), 1); + } + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QString("sear*"), QueryBase::Comparator::Fulltext)); + auto result = Sink::Store::read(query); + QCOMPARE(result.size(), 1); + } + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QString("searchable"), QueryBase::Comparator::Fulltext)); + auto result = Sink::Store::read(query); + QCOMPARE(result.size(), 1); + } + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); + query.filter("folder1"); + auto result = Sink::Store::read(query); + QCOMPARE(result.size(), 1); + } + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); + query.filter("folder2"); + auto result = Sink::Store::read(query); + QCOMPARE(result.size(), 0); + } + } + }; QTEST_MAIN(QueryTest) -- cgit v1.2.3