From 335251d0420523ac8b9997c802f6850c427aaf01 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Sat, 11 Feb 2017 17:57:27 +0100 Subject: Fixed index removals --- common/index.cpp | 14 ++++++---- common/storage_lmdb.cpp | 6 ++++- common/typeindex.cpp | 70 ++++++++++++++++++++++++++++--------------------- common/typeindex.h | 5 ++-- 4 files changed, 57 insertions(+), 38 deletions(-) (limited to 'common') diff --git a/common/index.cpp b/common/index.cpp index f3d05f1..f09e265 100644 --- a/common/index.cpp +++ b/common/index.cpp @@ -18,22 +18,26 @@ Index::Index(const QByteArray &name, Sink::Storage::DataStore::Transaction &tran void Index::add(const QByteArray &key, const QByteArray &value) { - mDb.write(key, value); + mDb.write(key, value, [&] (const Sink::Storage::DataStore::Error &error) { + SinkWarning() << "Error while writing value" << error; + }); } void Index::remove(const QByteArray &key, const QByteArray &value) { - mDb.remove(key, value); + mDb.remove(key, value, [&] (const Sink::Storage::DataStore::Error &error) { + SinkWarning() << "Error while removing value: " << key << value << error << error.store; + }); } void Index::lookup(const QByteArray &key, const std::function &resultHandler, const std::function &errorHandler, bool matchSubStringKeys) { mDb.scan(key, - [this, resultHandler](const QByteArray &key, const QByteArray &value) -> bool { + [&](const QByteArray &key, const QByteArray &value) -> bool { resultHandler(value); return true; }, - [this, errorHandler](const Sink::Storage::DataStore::Error &error) { + [&](const Sink::Storage::DataStore::Error &error) { SinkWarning() << "Error while retrieving value" << error.message; errorHandler(Error(error.store, error.code, error.message)); }, @@ -44,6 +48,6 @@ QByteArray Index::lookup(const QByteArray &key) { QByteArray result; //We have to create a deep copy, otherwise the returned data may become invalid when the transaction ends. - lookup(key, [&result](const QByteArray &value) { result = QByteArray(value.constData(), value.size()); }, [this](const Index::Error &error) { SinkWarning() << "Error while retrieving value" << error.message; }); + lookup(key, [&](const QByteArray &value) { result = QByteArray(value.constData(), value.size()); }, [this](const Index::Error &error) { SinkWarning() << "Error while retrieving value" << error.message; }); return result; } diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index f534029..b389e58 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -195,7 +195,11 @@ void DataStore::NamedDatabase::remove(const QByteArray &k, const QByteArray &val } if (rc) { - Error error(d->name.toLatin1() + d->db, ErrorCodes::GenericError, QString("Error on mdb_del: %1 %2").arg(rc).arg(mdb_strerror(rc)).toLatin1()); + auto errorCode = ErrorCodes::GenericError; + if (rc == MDB_NOTFOUND) { + errorCode = ErrorCodes::NotFound; + } + Error error(d->name.toLatin1() + d->db, errorCode, QString("Error on mdb_del: %1 %2").arg(rc).arg(mdb_strerror(rc)).toLatin1()); errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); } } diff --git a/common/typeindex.cpp b/common/typeindex.cpp index a65c676..a0d6d43 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -72,9 +72,13 @@ QByteArray TypeIndex::indexName(const QByteArray &property, const QByteArray &so template <> void TypeIndex::addProperty(const QByteArray &property) { - auto indexer = [this, property](const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { + auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { // SinkTraceCtx(mLogCtx) << "Indexing " << mType + ".index." + property << value.toByteArray(); - Index(indexName(property), transaction).add(getByteArray(value), identifier); + if (add) { + Index(indexName(property), transaction).add(getByteArray(value), identifier); + } else { + Index(indexName(property), transaction).remove(getByteArray(value), identifier); + } }; mIndexer.insert(property, indexer); mProperties << property; @@ -83,9 +87,13 @@ void TypeIndex::addProperty(const QByteArray &property) template <> void TypeIndex::addProperty(const QByteArray &property) { - auto indexer = [this, property](const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { + auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { // SinkTraceCtx(mLogCtx) << "Indexing " << mType + ".index." + property << value.toByteArray(); - Index(indexName(property), transaction).add(getByteArray(value), identifier); + if (add) { + Index(indexName(property), transaction).add(getByteArray(value), identifier); + } else { + Index(indexName(property), transaction).remove(getByteArray(value), identifier); + } }; mIndexer.insert(property, indexer); mProperties << property; @@ -94,9 +102,13 @@ void TypeIndex::addProperty(const QByteArray &property) template <> void TypeIndex::addProperty(const QByteArray &property) { - auto indexer = [this, property](const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { + auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { //SinkTraceCtx(mLogCtx) << "Indexing " << mType + ".index." + property << getByteArray(value); - Index(indexName(property), transaction).add(getByteArray(value), identifier); + if (add) { + Index(indexName(property), transaction).add(getByteArray(value), identifier); + } else { + Index(indexName(property), transaction).remove(getByteArray(value), identifier); + } }; mIndexer.insert(property, indexer); mProperties << property; @@ -111,10 +123,15 @@ void TypeIndex::addProperty(const QByteArray &prop template <> void TypeIndex::addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty) { - auto indexer = [=](const QByteArray &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) { + auto indexer = [=](bool add, const QByteArray &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) { const auto date = sortValue.toDateTime(); const auto propertyValue = getByteArray(value); - Index(indexName(property, sortProperty), transaction).add(propertyValue + toSortableByteArray(date), identifier); + SinkWarning() << "Adding sorted value " << indexName(property, sortProperty); + if (add) { + Index(indexName(property, sortProperty), transaction).add(propertyValue + toSortableByteArray(date), identifier); + } else { + Index(indexName(property, sortProperty), transaction).remove(propertyValue + toSortableByteArray(date), identifier); + } }; mSortIndexer.insert(property + sortProperty, indexer); mSortedProperties.insert(property, sortProperty); @@ -126,45 +143,38 @@ void TypeIndex::addPropertyWithSorting( addPropertyWithSorting(property, sortProperty); } -void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction) +void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction) { - SinkTrace() << "add " << identifier; for (const auto &property : mProperties) { const auto value = entity.getProperty(property); auto indexer = mIndexer.value(property); - indexer(identifier, value, transaction); + indexer(add, identifier, value, transaction); } for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { const auto value = entity.getProperty(it.key()); const auto sortValue = entity.getProperty(it.value()); auto indexer = mSortIndexer.value(it.key() + it.value()); - indexer(identifier, value, sortValue, transaction); + indexer(add, identifier, value, sortValue, transaction); } for (const auto &indexer : mCustomIndexer) { indexer->setup(this, &transaction); - indexer->add(entity); + if (add) { + indexer->add(entity); + } else { + indexer->remove(entity); + } } + +} + +void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction) +{ + updateIndex(true, identifier, entity, transaction); } void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction) { - for (const auto &property : mProperties) { - const auto value = entity.getProperty(property); - Index(indexName(property), transaction).remove(getByteArray(value), identifier); - } - for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { - const auto propertyValue = entity.getProperty(it.key()); - const auto sortValue = entity.getProperty(it.value()); - if (sortValue.type() == QVariant::DateTime) { - Index(indexName(it.key(), it.value()), transaction).remove(propertyValue.toByteArray() + toSortableByteArray(sortValue.toDateTime()), identifier); - } else { - Index(indexName(it.key(), it.value()), transaction).remove(propertyValue.toByteArray() + sortValue.toByteArray(), identifier); - } - } - for (const auto &indexer : mCustomIndexer) { - indexer->setup(this, &transaction); - indexer->remove(entity); - } + updateIndex(false, identifier, entity, transaction); } static QVector indexLookup(Index &index, QueryBase::Comparator filter) diff --git a/common/typeindex.h b/common/typeindex.h index 7ff2029..1f216a7 100644 --- a/common/typeindex.h +++ b/common/typeindex.h @@ -98,6 +98,7 @@ public: private: friend class Sink::Storage::EntityStore; + void updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction); QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const; Sink::Log::Context mLogCtx; QByteArray mType; @@ -107,6 +108,6 @@ private: QMap mSecondaryProperties; QList mCustomIndexer; Sink::Storage::DataStore::Transaction *mTransaction; - QHash> mIndexer; - QHash> mSortIndexer; + QHash> mIndexer; + QHash> mSortIndexer; }; -- cgit v1.2.3