From c90ba4a98292a39eb0b3df12fd7e2dec0300e58d Mon Sep 17 00:00:00 2001 From: Minijackson Date: Tue, 26 Jun 2018 14:10:30 +0200 Subject: Move Key API into own files + some fixes --- common/CMakeLists.txt | 1 + common/changereplay.cpp | 1 + common/storage.h | 148 ------------------------------- common/storage/entitystore.cpp | 5 +- common/storage/entitystore.h | 1 + common/storage/key.cpp | 156 +++++++++++++++++++++++++++++++++ common/storage/key.h | 100 +++++++++++++++++++++ common/storage_common.cpp | 18 ---- sinksh/syntax_modules/sink_inspect.cpp | 2 + tests/pipelinetest.cpp | 9 +- tests/querytest.cpp | 38 -------- tests/storagetest.cpp | 1 + 12 files changed, 271 insertions(+), 209 deletions(-) create mode 100644 common/storage/key.cpp create mode 100644 common/storage/key.h diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 970990f..7c4630b 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -73,6 +73,7 @@ add_library(${PROJECT_NAME} SHARED specialpurposepreprocessor.cpp datastorequery.cpp storage/entitystore.cpp + storage/key.cpp indexer.cpp mail/threadindexer.cpp mail/fulltextindexer.cpp diff --git a/common/changereplay.cpp b/common/changereplay.cpp index e81f52c..e94ed80 100644 --- a/common/changereplay.cpp +++ b/common/changereplay.cpp @@ -23,6 +23,7 @@ #include "log.h" #include "definitions.h" #include "bufferutils.h" +#include "storage/key.h" #include diff --git a/common/storage.h b/common/storage.h index 3cc5adf..25d0fa6 100644 --- a/common/storage.h +++ b/common/storage.h @@ -40,151 +40,6 @@ struct SINK_EXPORT DbLayout { Databases tables; }; -class Identifier -{ -public: - // RFC 4122 Section 4.1.2 says 128 bits -> 16 bytes - static const constexpr size_t INTERNAL_REPR_SIZE = 16; - static const constexpr size_t DISPLAY_REPR_SIZE = 38; - - Identifier() : uid(QUuid::createUuid()) {}; - - QByteArray toInternalByteArray() const - { - return uid.toRfc4122(); - } - - static Identifier fromInternalByteArray(const QByteArray &bytes) - { - Q_ASSERT(bytes.size() == INTERNAL_REPR_SIZE); - return Identifier(QUuid::fromRfc4122(bytes)); - } - - QString toDisplayString() const - { - return uid.toString(); - } - - QByteArray toDisplayByteArray() const - { - return uid.toByteArray(); - } - - static Identifier fromDisplayByteArray(const QByteArray &bytes) - { - Q_ASSERT(bytes.size() == DISPLAY_REPR_SIZE); - return Identifier(QUuid::fromString(QString::fromUtf8(bytes))); - } - -private: - explicit Identifier(const QUuid &uid) : uid(uid) {} - QUuid uid; -}; - -class Revision -{ -public: - // qint64 has a 19 digit decimal representation - static const constexpr size_t INTERNAL_REPR_SIZE = 19; - static const constexpr size_t DISPLAY_REPR_SIZE = 19; - - Revision(qint64 rev) : rev(rev) {} - - QByteArray toInternalByteArray() const - { - return padNumber(rev); - } - - static Revision fromInternalByteArray(const QByteArray &bytes) - { - Q_ASSERT(bytes.size() == INTERNAL_REPR_SIZE); - return Revision(bytes.toLongLong()); - } - - QString toDisplayString() const - { - return QString::fromUtf8(toInternalByteArray()); - } - - QByteArray toDisplayByteArray() const - { - return toInternalByteArray(); - } - - static Revision fromDisplayByteArray(const QByteArray &bytes) - { - Q_ASSERT(bytes.size() == DISPLAY_REPR_SIZE); - return fromInternalByteArray(bytes); - } - - qint64 toQint64() const - { - return rev; - } - -private: - qint64 rev; -}; - -class Key -{ -public: - static const constexpr size_t INTERNAL_REPR_SIZE = Identifier::INTERNAL_REPR_SIZE + Revision::INTERNAL_REPR_SIZE; - static const constexpr size_t DISPLAY_REPR_SIZE = Identifier::DISPLAY_REPR_SIZE + Revision::DISPLAY_REPR_SIZE; - - Key(const Identifier &id, const Revision &rev) : id(id), rev(rev) {} - - QByteArray toInternalByteArray() const - { - return id.toInternalByteArray() + rev.toInternalByteArray(); - } - - static Key fromInternalByteArray(const QByteArray &bytes) - { - Q_ASSERT(bytes.size() == INTERNAL_REPR_SIZE); - auto idBytes = bytes.mid(0, Identifier::INTERNAL_REPR_SIZE); - auto revBytes = bytes.mid(Identifier::INTERNAL_REPR_SIZE); - return Key(Identifier::fromInternalByteArray(idBytes), Revision::fromInternalByteArray(revBytes)); - } - - QString toDisplayString() const - { - return id.toDisplayString() + rev.toDisplayString(); - } - - QByteArray toDisplayByteArray() const - { - return id.toDisplayByteArray() + rev.toDisplayByteArray(); - } - - static Key fromDisplayByteArray(const QByteArray &bytes) - { - Q_ASSERT(bytes.size() == DISPLAY_REPR_SIZE); - auto idBytes = bytes.mid(0, Identifier::DISPLAY_REPR_SIZE); - auto revBytes = bytes.mid(Identifier::DISPLAY_REPR_SIZE); - return Key(Identifier::fromDisplayByteArray(idBytes), Revision::fromDisplayByteArray(revBytes)); - } - - const Identifier &identifier() const - { - return id; - } - - const Revision &revision() const - { - return rev; - } - - void setRevision(const Revision &newRev) - { - rev = newRev; - } - -private: - Identifier id; - Revision rev; -}; - class SINK_EXPORT DataStore { public: @@ -407,6 +262,3 @@ private: } // namespace Sink SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::DataStore::Error &error); -SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::Identifier &); -SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::Revision &); -SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::Key &); diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index f74d3df..efafe8a 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -678,9 +678,10 @@ void EntityStore::readRevisions(const QByteArray &type, const QByteArray &uid, q DataStore::mainDatabase(d->transaction, type) .scan(uid, [&](const QByteArray &key, const QByteArray &value) -> bool { - const auto revision = DataStore::revisionFromKey(key); + const auto parsedKey = Key::fromDisplayByteArray(key); + const auto revision = parsedKey.revision().toQint64(); if (revision >= startingRevision) { - callback(DataStore::uidFromKey(key), revision, Sink::EntityBuffer(value.data(), value.size())); + callback(parsedKey.identifier().toDisplayByteArray(), revision, Sink::EntityBuffer(value.data(), value.size())); } return true; }, diff --git a/common/storage/entitystore.h b/common/storage/entitystore.h index 69de76c..1dcd285 100644 --- a/common/storage/entitystore.h +++ b/common/storage/entitystore.h @@ -25,6 +25,7 @@ #include "domaintypeadaptorfactoryinterface.h" #include "query.h" #include "storage.h" +#include "key.h" #include "resourcecontext.h" #include "metadata_generated.h" diff --git a/common/storage/key.cpp b/common/storage/key.cpp new file mode 100644 index 0000000..5d26722 --- /dev/null +++ b/common/storage/key.cpp @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2014 Christian Mollekopf + * Copyright (C) 2018 Rémi Nicole + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) version 3, or any + * later version accepted by the membership of KDE e.V. (or its + * successor approved by the membership of KDE e.V.), which shall + * act as a proxy defined in Section 6 of version 3 of the license. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + */ + +#include "key.h" +#include "utils.h" + +using Sink::Storage::Identifier; +using Sink::Storage::Key; +using Sink::Storage::Revision; + +QDebug &operator<<(QDebug &dbg, const Identifier &id) +{ + dbg << id.toDisplayString(); + return dbg; +} + +QDebug &operator<<(QDebug &dbg, const Revision &rev) +{ + dbg << rev.toDisplayString(); + return dbg; +} + +QDebug &operator<<(QDebug &dbg, const Key &key) +{ + dbg << key.toDisplayString(); + return dbg; +} + +// Identifier + +QByteArray Identifier::toInternalByteArray() const +{ + return uid.toRfc4122(); +} + +Identifier Identifier::fromInternalByteArray(const QByteArray &bytes) +{ + Q_ASSERT(bytes.size() == INTERNAL_REPR_SIZE); + return Identifier(QUuid::fromRfc4122(bytes)); +} + +QString Identifier::toDisplayString() const +{ + return uid.toString(); +} + +QByteArray Identifier::toDisplayByteArray() const +{ + return uid.toByteArray(); +} + +Identifier Identifier::fromDisplayByteArray(const QByteArray &bytes) +{ + Q_ASSERT(bytes.size() == DISPLAY_REPR_SIZE); + return Identifier(QUuid::fromString(QString::fromUtf8(bytes))); +} + +// Revision + +QByteArray Revision::toInternalByteArray() const +{ + return padNumber(rev); +} + +Revision Revision::fromInternalByteArray(const QByteArray &bytes) +{ + Q_ASSERT(bytes.size() == INTERNAL_REPR_SIZE); + return Revision(bytes.toLongLong()); +} + +QString Revision::toDisplayString() const +{ + return QString::fromUtf8(toInternalByteArray()); +} + +QByteArray Revision::toDisplayByteArray() const +{ + return toInternalByteArray(); +} + +Revision Revision::fromDisplayByteArray(const QByteArray &bytes) +{ + Q_ASSERT(bytes.size() == DISPLAY_REPR_SIZE); + return fromInternalByteArray(bytes); +} + +qint64 Revision::toQint64() const +{ + return rev; +} + +// Key + +QByteArray Key::toInternalByteArray() const +{ + return id.toInternalByteArray() + rev.toInternalByteArray(); +} + +Key Key::fromInternalByteArray(const QByteArray &bytes) +{ + Q_ASSERT(bytes.size() == INTERNAL_REPR_SIZE); + auto idBytes = bytes.mid(0, Identifier::INTERNAL_REPR_SIZE); + auto revBytes = bytes.mid(Identifier::INTERNAL_REPR_SIZE); + return Key(Identifier::fromInternalByteArray(idBytes), Revision::fromInternalByteArray(revBytes)); +} + +QString Key::toDisplayString() const +{ + return id.toDisplayString() + rev.toDisplayString(); +} + +QByteArray Key::toDisplayByteArray() const +{ + return id.toDisplayByteArray() + rev.toDisplayByteArray(); +} + +Key Key::fromDisplayByteArray(const QByteArray &bytes) +{ + Q_ASSERT(bytes.size() == DISPLAY_REPR_SIZE); + auto idBytes = bytes.mid(0, Identifier::DISPLAY_REPR_SIZE); + auto revBytes = bytes.mid(Identifier::DISPLAY_REPR_SIZE); + return Key(Identifier::fromDisplayByteArray(idBytes), Revision::fromDisplayByteArray(revBytes)); +} + +const Identifier &Key::identifier() const +{ + return id; +} + +const Revision &Key::revision() const +{ + return rev; +} + +void Key::setRevision(const Revision &newRev) +{ + rev = newRev; +} diff --git a/common/storage/key.h b/common/storage/key.h new file mode 100644 index 0000000..76dbd13 --- /dev/null +++ b/common/storage/key.h @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2014 Christian Mollekopf + * Copyright (C) 2018 Rémi Nicole + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) version 3, or any + * later version accepted by the membership of KDE e.V. (or its + * successor approved by the membership of KDE e.V.), which shall + * act as a proxy defined in Section 6 of version 3 of the license. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + */ + +#pragma once + +#include "sink_export.h" + +#include +#include +#include + +namespace Sink { +namespace Storage { + +class Identifier +{ +public: + // RFC 4122 Section 4.1.2 says 128 bits -> 16 bytes + static const constexpr size_t INTERNAL_REPR_SIZE = 16; + static const constexpr size_t DISPLAY_REPR_SIZE = 38; + + Identifier() : uid(QUuid::createUuid()){}; + + QByteArray toInternalByteArray() const; + static Identifier fromInternalByteArray(const QByteArray &bytes); + QString toDisplayString() const; + QByteArray toDisplayByteArray() const; + static Identifier fromDisplayByteArray(const QByteArray &bytes); + +private: + explicit Identifier(const QUuid &uid) : uid(uid) {} + QUuid uid; +}; + +class Revision +{ +public: + // qint64 has a 19 digit decimal representation + static const constexpr size_t INTERNAL_REPR_SIZE = 19; + static const constexpr size_t DISPLAY_REPR_SIZE = 19; + + Revision(qint64 rev) : rev(rev) {} + + QByteArray toInternalByteArray() const; + static Revision fromInternalByteArray(const QByteArray &bytes); + QString toDisplayString() const; + QByteArray toDisplayByteArray() const; + static Revision fromDisplayByteArray(const QByteArray &bytes); + qint64 toQint64() const; + +private: + qint64 rev; +}; + +class Key +{ +public: + static const constexpr size_t INTERNAL_REPR_SIZE = Identifier::INTERNAL_REPR_SIZE + Revision::INTERNAL_REPR_SIZE; + static const constexpr size_t DISPLAY_REPR_SIZE = Identifier::DISPLAY_REPR_SIZE + Revision::DISPLAY_REPR_SIZE; + + Key(const Identifier &id, const Revision &rev) : id(id), rev(rev) {} + + QByteArray toInternalByteArray() const; + static Key fromInternalByteArray(const QByteArray &bytes); + QString toDisplayString() const; + QByteArray toDisplayByteArray() const; + static Key fromDisplayByteArray(const QByteArray &bytes); + const Identifier &identifier() const; + const Revision &revision() const; + void setRevision(const Revision &newRev); + +private: + Identifier id; + Revision rev; +}; + +} // namespace Storage +} // namespace Sink + +SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::Identifier &); +SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::Revision &); +SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::Key &); diff --git a/common/storage_common.cpp b/common/storage_common.cpp index b36ffcb..c922d9d 100644 --- a/common/storage_common.cpp +++ b/common/storage_common.cpp @@ -32,24 +32,6 @@ QDebug& operator<<(QDebug &dbg, const Sink::Storage::DataStore::Error &error) return dbg; } -QDebug& operator<<(QDebug &dbg, const Sink::Storage::Identifier &id) -{ - dbg << id.toDisplayString(); - return dbg; -} - -QDebug& operator<<(QDebug &dbg, const Sink::Storage::Revision &rev) -{ - dbg << rev.toDisplayString(); - return dbg; -} - -QDebug& operator<<(QDebug &dbg, const Sink::Storage::Key &key) -{ - dbg << key.toDisplayString(); - return dbg; -} - namespace Sink { namespace Storage { diff --git a/sinksh/syntax_modules/sink_inspect.cpp b/sinksh/syntax_modules/sink_inspect.cpp index b496fe6..019ef31 100644 --- a/sinksh/syntax_modules/sink_inspect.cpp +++ b/sinksh/syntax_modules/sink_inspect.cpp @@ -34,6 +34,8 @@ #include "common/metadata_generated.h" #include "common/bufferutils.h" +#include "storage/key.h" + #include "sinksh_utils.h" #include "state.h" #include "syntaxtree.h" diff --git a/tests/pipelinetest.cpp b/tests/pipelinetest.cpp index 7431deb..5165d35 100644 --- a/tests/pipelinetest.cpp +++ b/tests/pipelinetest.cpp @@ -20,6 +20,7 @@ #include "domainadaptor.h" #include "definitions.h" #include "adaptorfactoryregistry.h" +#include "storage/key.h" static void removeFromDisk(const QString &name) { @@ -430,8 +431,8 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - const auto key = keys.first(); - const auto uid = Sink::Storage::DataStore::uidFromKey(key); + auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + const auto uid = key.identifier().toDisplayByteArray(); //Simulate local modification { @@ -453,8 +454,10 @@ private slots: pipeline.commit(); } + key.setRevision(3); + // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", Sink::Storage::DataStore::assembleKey(uid, 3)); + auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); diff --git a/tests/querytest.cpp b/tests/querytest.cpp index 41ce40c..7685086 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -1823,44 +1823,6 @@ private slots: } - void testOverlapLive() - { - eventsWithDates(); - - { - Sink::Query query; - query.resourceFilter("sink.dummy.instance1"); - query.setFlags(Query::LiveQuery); - query.filter(QueryBase::Comparator( - QVariantList{ QDateTime::fromString("2018-05-22T12:00:00Z", Qt::ISODate), - QDateTime::fromString("2018-05-30T13:00:00Z", Qt::ISODate) }, - QueryBase::Comparator::Overlap)); - auto model = Sink::Store::loadModel(query); - QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); - QCOMPARE(model->rowCount(), 5); - - Event event = Event::createEntity("sink.dummy.instance1"); - event.setExtractedStartTime(QDateTime::fromString("2018-05-23T12:00:00Z", Qt::ISODate)); - event.setExtractedEndTime(QDateTime::fromString("2018-05-23T13:00:00Z", Qt::ISODate)); - VERIFYEXEC(Sink::Store::create(event)); - - Event event2 = Event::createEntity("sink.dummy.instance1"); - event2.setExtractedStartTime(QDateTime::fromString("2018-05-33T12:00:00Z", Qt::ISODate)); - event2.setExtractedEndTime(QDateTime::fromString("2018-05-33T13:00:00Z", Qt::ISODate)); - VERIFYEXEC(Sink::Store::create(event2)); - - QTest::qWait(500); - QCOMPARE(model->rowCount(), 6); - - VERIFYEXEC(Sink::Store::remove(event)); - VERIFYEXEC(Sink::Store::remove(event2)); - - QTest::qWait(500); - QCOMPARE(model->rowCount(), 5); - } - - } - }; QTEST_MAIN(QueryTest) diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp index 40492f0..d3f180a 100644 --- a/tests/storagetest.cpp +++ b/tests/storagetest.cpp @@ -7,6 +7,7 @@ #include #include "common/storage.h" +#include "storage/key.h" /** * Test of the storage implementation to ensure it can do the low level operations as expected. -- cgit v1.2.3