From d8cd2d6585507a4e40881092a633ec1a80b14dd9 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Mon, 18 Jan 2016 15:17:30 +0100 Subject: Draft of inspection API --- common/CMakeLists.txt | 1 + common/clientapi.cpp | 17 ++++++++ common/clientapi.h | 25 ++++++++++++ common/commands.cpp | 2 + common/commands.h | 1 + common/commands/inspection.fbs | 12 ++++++ common/commands/notification.fbs | 5 ++- common/genericresource.cpp | 39 ++++++++++++++++++ common/genericresource.h | 1 + common/listener.cpp | 3 ++ common/notification.h | 40 +++++++++++++++++++ common/resource.h | 2 + common/resourceaccess.cpp | 22 ++++++++++ common/resourceaccess.h | 4 ++ docs/resource.md | 7 ++++ examples/dummyresource/resourcefactory.cpp | 20 ++++++++++ examples/dummyresource/resourcefactory.h | 1 + tests/CMakeLists.txt | 2 + tests/inspectiontest.cpp | 64 ++++++++++++++++++++++++++++++ 19 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 common/commands/inspection.fbs create mode 100644 common/notification.h create mode 100644 tests/inspectiontest.cpp diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index f07772a..7e142df 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -51,6 +51,7 @@ generate_flatbuffers( commands/synchronize commands/notification commands/revisionreplayed + commands/inspection domain/event domain/mail domain/folder diff --git a/common/clientapi.cpp b/common/clientapi.cpp index 5c0bcb8..deab962 100644 --- a/common/clientapi.cpp +++ b/common/clientapi.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -289,10 +290,26 @@ KAsync::Job > Store::fetch(const Akonadi2::Query }); } +template +KAsync::Job Resources::inspect(const Inspection &inspectionCommand) +{ + auto resource = inspectionCommand.resourceIdentifier; + + Trace() << "Sending inspection " << resource; + auto resourceAccess = QSharedPointer::create(resource); + resourceAccess->open(); + auto id = QUuid::createUuid().toByteArray(); + return resourceAccess->sendInspectionCommand(id, ApplicationDomain::getTypeName(), inspectionCommand.entityIdentifier, inspectionCommand.property, inspectionCommand.expectedValue) + .template then([resourceAccess]() { + //TODO wait for inspection notification + }); +} + #define REGISTER_TYPE(T) template KAsync::Job Store::remove(const T &domainObject); \ template KAsync::Job Store::create(const T &domainObject); \ template KAsync::Job Store::modify(const T &domainObject); \ template QSharedPointer Store::loadModel(Query query); \ + template KAsync::Job Resources::inspect(const Inspection &); \ template KAsync::Job Store::fetchOne(const Query &); \ template KAsync::Job > Store::fetchAll(const Query &); \ template KAsync::Job > Store::fetch(const Query &, int); \ diff --git a/common/clientapi.h b/common/clientapi.h index eff8e8d..d496715 100644 --- a/common/clientapi.h +++ b/common/clientapi.h @@ -111,6 +111,31 @@ public: static KAsync::Job > fetch(const Akonadi2::Query &query, int minimumAmount = 0); }; +namespace Resources { + struct Inspection { + static Inspection PropertyInspection(const Akonadi2::ApplicationDomain::Entity &entity, const QByteArray &property, const QVariant &expectedValue) + { + Inspection inspection; + inspection.resourceIdentifier = entity.resourceInstanceIdentifier(); + inspection.entityIdentifier = entity.identifier(); + inspection.property = property; + inspection.expectedValue = expectedValue; + return inspection; + } + + enum Type { + PropertyInspectionType + }; + + QByteArray resourceIdentifier; + QByteArray entityIdentifier; + QByteArray property; + QVariant expectedValue; + }; + template + KAsync::Job inspect(const Inspection &inspectionCommand); +} + } diff --git a/common/commands.cpp b/common/commands.cpp index 7a0ae23..16fd742 100644 --- a/common/commands.cpp +++ b/common/commands.cpp @@ -59,6 +59,8 @@ QByteArray name(int commandId) return "Ping"; case RevisionReplayedCommand: return "RevisionReplayed"; + case InspectionCommand: + return "Inspection"; case CustomCommand: return "Custom"; }; diff --git a/common/commands.h b/common/commands.h index c68ef90..bce278c 100644 --- a/common/commands.h +++ b/common/commands.h @@ -47,6 +47,7 @@ enum CommandIds { NotificationCommand, PingCommand, RevisionReplayedCommand, + InspectionCommand, CustomCommand = 0xffff }; diff --git a/common/commands/inspection.fbs b/common/commands/inspection.fbs new file mode 100644 index 0000000..aaae1ae --- /dev/null +++ b/common/commands/inspection.fbs @@ -0,0 +1,12 @@ +namespace Akonadi2.Commands; + +table Inspection { + id: string; + type: int; + entityId: string; + domainType: string; + property: string; + expectedValue: string; +} + +root_type Inspection; diff --git a/common/commands/notification.fbs b/common/commands/notification.fbs index 6684472..eb00986 100644 --- a/common/commands/notification.fbs +++ b/common/commands/notification.fbs @@ -1,9 +1,12 @@ namespace Akonadi2; -enum NotificationType : byte { Shutdown = 1, Status, Warning, Progress } +enum NotificationType : byte { Shutdown = 1, Status, Warning, Progress, Inspection } +enum NotificationCode : byte { Success = 0, Failure = 1, UserCode } table Notification { type: NotificationType = Status; + message: string; + code: int = 0; //Of type NotificationCode } root_type Notification; diff --git a/common/genericresource.cpp b/common/genericresource.cpp index 29acce4..90fc763 100644 --- a/common/genericresource.cpp +++ b/common/genericresource.cpp @@ -6,6 +6,7 @@ #include "createentity_generated.h" #include "modifyentity_generated.h" #include "deleteentity_generated.h" +#include "inspection_generated.h" #include "domainadaptor.h" #include "commands.h" #include "index.h" @@ -13,6 +14,7 @@ #include "definitions.h" #include +#include static int sBatchSize = 100; @@ -112,6 +114,7 @@ private: class CommandProcessor : public QObject { Q_OBJECT + typedef std::function(void const *, size_t)> InspectionFunction; public: CommandProcessor(Akonadi2::Pipeline *pipeline, QList commandQueues) : QObject(), @@ -135,6 +138,11 @@ public: mLowerBoundRevision = revision; } + void setInspectionCommand(const InspectionFunction &f) + { + mInspect = f; + } + signals: void error(int errorCode, const QString &errorMessage); @@ -176,6 +184,14 @@ private slots: return mPipeline->modifiedEntity(queuedCommand->command()->Data(), queuedCommand->command()->size()); case Akonadi2::Commands::CreateEntityCommand: return mPipeline->newEntity(queuedCommand->command()->Data(), queuedCommand->command()->size()); + case Akonadi2::Commands::InspectionCommand: + if (mInspect) { + return mInspect(queuedCommand->command()->Data(), queuedCommand->command()->size()).then([]() { + return -1; + }); + } else { + return KAsync::error(-1, "Missing inspection command."); + } default: return KAsync::error(-1, "Unhandled command"); } @@ -266,6 +282,7 @@ private: bool mProcessingLock; //The lowest revision we no longer need qint64 mLowerBoundRevision; + InspectionFunction mInspect; }; @@ -279,6 +296,22 @@ GenericResource::GenericResource(const QByteArray &resourceInstanceIdentifier, c mClientLowerBoundRevision(std::numeric_limits::max()) { mProcessor = new CommandProcessor(mPipeline.data(), QList() << &mUserQueue << &mSynchronizerQueue); + mProcessor->setInspectionCommand([this](void const *command, size_t size) { + flatbuffers::Verifier verifier((const uint8_t *)command, size); + if (Akonadi2::Commands::VerifyInspectionBuffer(verifier)) { + auto buffer = Akonadi2::Commands::GetInspection(command); + int inspectionType = buffer->type(); + QByteArray entityId = QByteArray::fromRawData(reinterpret_cast(buffer->entityId()->Data()), buffer->entityId()->size()); + QByteArray domainType = QByteArray::fromRawData(reinterpret_cast(buffer->domainType()->Data()), buffer->domainType()->size()); + QByteArray property = QByteArray::fromRawData(reinterpret_cast(buffer->property()->Data()), buffer->property()->size()); + QByteArray expectedValueString = QByteArray::fromRawData(reinterpret_cast(buffer->expectedValue()->Data()), buffer->expectedValue()->size()); + QDataStream s(expectedValueString); + QVariant expectedValue; + s >> expectedValue; + return inspect(inspectionType, domainType, entityId, property, expectedValue); + } + return KAsync::error(-1, "Invalid inspection command."); + }); QObject::connect(mProcessor, &CommandProcessor::error, [this](int errorCode, const QString &msg) { onProcessorError(errorCode, msg); }); QObject::connect(mPipeline.data(), &Pipeline::revisionUpdated, this, &Resource::revisionUpdated); mSourceChangeReplay = new ChangeReplay(resourceInstanceIdentifier, [this](const QByteArray &type, const QByteArray &key, const QByteArray &value) { @@ -301,6 +334,12 @@ GenericResource::~GenericResource() delete mSourceChangeReplay; } +KAsync::Job GenericResource::inspect(int inspectionType, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) +{ + Warning() << "Inspection not implemented"; + return KAsync::null(); +} + void GenericResource::enableChangeReplay(bool enable) { if (enable) { diff --git a/common/genericresource.h b/common/genericresource.h index f47c6f8..90b7c29 100644 --- a/common/genericresource.h +++ b/common/genericresource.h @@ -48,6 +48,7 @@ public: virtual KAsync::Job synchronizeWithSource(Akonadi2::Storage &mainStore, Akonadi2::Storage &synchronizationStore); virtual KAsync::Job processAllMessages() Q_DECL_OVERRIDE; virtual void setLowerBoundRevision(qint64 revision) Q_DECL_OVERRIDE; + virtual KAsync::Job inspect(int inspectionType, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue); int error() const; diff --git a/common/listener.cpp b/common/listener.cpp index 518aca9..5d55202 100644 --- a/common/listener.cpp +++ b/common/listener.cpp @@ -31,11 +31,13 @@ #include "common/synchronize_generated.h" #include "common/notification_generated.h" #include "common/revisionreplayed_generated.h" +#include "common/inspection_generated.h" #include #include #include #include +#include Listener::Listener(const QByteArray &resourceInstanceIdentifier, QObject *parent) : QObject(parent), @@ -241,6 +243,7 @@ void Listener::processCommand(int commandId, uint messageId, const QByteArray &c } break; } + case Akonadi2::Commands::InspectionCommand: case Akonadi2::Commands::FetchEntityCommand: case Akonadi2::Commands::DeleteEntityCommand: case Akonadi2::Commands::ModifyEntityCommand: diff --git a/common/notification.h b/common/notification.h new file mode 100644 index 0000000..8ffc24c --- /dev/null +++ b/common/notification.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 Christian Mollekopf + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) version 3, or any + * later version accepted by the membership of KDE e.V. (or its + * successor approved by the membership of KDE e.V.), which shall + * act as a proxy defined in Section 6 of version 3 of the license. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + */ +#pragma once + +#include +#include +#include "notification_generated.h" + +namespace Akonadi2 +{ + +/** + * A notification + */ +class AKONADI2COMMON_EXPORT ResourceNotification +{ +public: + int type; + QString message; + int code; +}; + +} diff --git a/common/resource.h b/common/resource.h index 4ed21b5..9a31d03 100644 --- a/common/resource.h +++ b/common/resource.h @@ -21,6 +21,7 @@ #include #include +#include "notification.h" namespace Akonadi2 { @@ -55,6 +56,7 @@ public: Q_SIGNALS: void revisionUpdated(qint64); + void notify(ResourceNotification); private: class Private; diff --git a/common/resourceaccess.cpp b/common/resourceaccess.cpp index 7be1259..65e9a8c 100644 --- a/common/resourceaccess.cpp +++ b/common/resourceaccess.cpp @@ -30,6 +30,7 @@ #include "common/modifyentity_generated.h" #include "common/deleteentity_generated.h" #include "common/revisionreplayed_generated.h" +#include "common/inspection_generated.h" #include "common/entitybuffer.h" #include "log.h" @@ -37,6 +38,8 @@ #include #include #include +#include +#include #undef Trace #define Trace() Akonadi2::Log::debugStream(Akonadi2::Log::DebugLevel::Trace, __LINE__, __FILE__, Q_FUNC_INFO, "ResourceAccess") @@ -338,6 +341,25 @@ KAsync::Job ResourceAccess::sendRevisionReplayedCommand(qint64 revision) return sendCommand(Akonadi2::Commands::RevisionReplayedCommand, fbb); } +KAsync::Job ResourceAccess::sendInspectionCommand(const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) +{ + flatbuffers::FlatBufferBuilder fbb; + auto id = fbb.CreateString(inspectionId.toStdString()); + auto domain = fbb.CreateString(domainType.toStdString()); + auto entity = fbb.CreateString(entityId.toStdString()); + auto prop = fbb.CreateString(property.toStdString()); + + QByteArray array; + QDataStream s(&array, QIODevice::WriteOnly); + s << expectedValue; + + auto expected = fbb.CreateString(array.toStdString()); + auto location = Akonadi2::Commands::CreateInspection (fbb, id, 0, entity, domain, prop, expected); + Akonadi2::Commands::FinishInspectionBuffer(fbb, location); + open(); + return sendCommand(Akonadi2::Commands::InspectionCommand, fbb); +} + void ResourceAccess::open() { if (d->socket && d->socket->isValid()) { diff --git a/common/resourceaccess.h b/common/resourceaccess.h index 7f61b30..fe3fa99 100644 --- a/common/resourceaccess.h +++ b/common/resourceaccess.h @@ -27,6 +27,7 @@ #include #include +#include "notification.h" namespace Akonadi2 { @@ -49,10 +50,12 @@ public: virtual KAsync::Job sendModifyCommand(const QByteArray &uid, qint64 revision, const QByteArray &resourceBufferType, const QByteArrayList &deletedProperties, const QByteArray &buffer) { return KAsync::null(); }; virtual KAsync::Job sendDeleteCommand(const QByteArray &uid, qint64 revision, const QByteArray &resourceBufferType) { return KAsync::null(); }; virtual KAsync::Job sendRevisionReplayedCommand(qint64 revision) {return KAsync::null(); }; + virtual KAsync::Job sendInspectionCommand(const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expecedValue) {return KAsync::null(); }; Q_SIGNALS: void ready(bool isReady); void revisionChanged(qint64 revision); + void notification(ResourceNotification revision); public Q_SLOTS: virtual void open() = 0; @@ -78,6 +81,7 @@ public: KAsync::Job sendModifyCommand(const QByteArray &uid, qint64 revision, const QByteArray &resourceBufferType, const QByteArrayList &deletedProperties, const QByteArray &buffer) Q_DECL_OVERRIDE; KAsync::Job sendDeleteCommand(const QByteArray &uid, qint64 revision, const QByteArray &resourceBufferType) Q_DECL_OVERRIDE; KAsync::Job sendRevisionReplayedCommand(qint64 revision) Q_DECL_OVERRIDE; + KAsync::Job sendInspectionCommand(const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expecedValue) Q_DECL_OVERRIDE; /** * Tries to connect to server, and returns a connected socket on success. */ diff --git a/docs/resource.md b/docs/resource.md index c8f58e9..0988535 100644 --- a/docs/resource.md +++ b/docs/resource.md @@ -141,3 +141,10 @@ The remoteid mapping has to be updated in two places: * New entities that are synchronized immediately get a localid assinged, that is then recorded together with the remoteid. This is required to be able to reference other entities directly in the command queue (i.e. for parent folders). * Entities created by clients get a remoteid assigned during change replay, so the entity can be recognized during the next sync. + +# Testing / Inspection +Resources new to be tested, which often requires inspections into the current state of the resource. This is difficult in an asynchronous system where the whole backend logic is encapsulated in a separate process without running tests in a vastly different setup from how it will be run in production. + +To alleviate this inspection commands are introduced. Inspection commands are special commands that the resource processes just like all other commands, and that have the sole purpose of inspecting the current resource state. Because the command is processed with the same mechanism as other commands we can rely on ordering of commands in a way that a prior command is guaranteed to be executed once the inspection command is processed. + +A typical inspection command could i.e. verify that a file has been created in the expected path after a create command. diff --git a/examples/dummyresource/resourcefactory.cpp b/examples/dummyresource/resourcefactory.cpp index a984097..27d5f17 100644 --- a/examples/dummyresource/resourcefactory.cpp +++ b/examples/dummyresource/resourcefactory.cpp @@ -134,6 +134,26 @@ void DummyResource::removeFromDisk(const QByteArray &instanceIdentifier) Akonadi2::Storage(Akonadi2::storageLocation(), instanceIdentifier + ".synchronization", Akonadi2::Storage::ReadWrite).removeFromDisk(); } +KAsync::Job DummyResource::inspect(int inspectionType, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) +{ + + Trace() << "Inspecting " << inspectionType << domainType << entityId << property << expectedValue; + if (property == "testInspection") { + Akonadi2::ResourceNotification n; + n.type = Akonadi2::NotificationType_Inspection; + if (expectedValue.toBool()) { + //Success + n.code = 0; + emit notify(n); + } else { + //Failure + n.code = 1; + emit notify(n); + } + } + return KAsync::null(); +} + DummyResourceFactory::DummyResourceFactory(QObject *parent) : Akonadi2::ResourceFactory(parent) { diff --git a/examples/dummyresource/resourcefactory.h b/examples/dummyresource/resourcefactory.h index 2ed4c5d..3f67187 100644 --- a/examples/dummyresource/resourcefactory.h +++ b/examples/dummyresource/resourcefactory.h @@ -40,6 +40,7 @@ public: KAsync::Job synchronizeWithSource(Akonadi2::Storage &mainStore, Akonadi2::Storage &synchronizationStore) Q_DECL_OVERRIDE; using GenericResource::synchronizeWithSource; static void removeFromDisk(const QByteArray &instanceIdentifier); + KAsync::Job inspect(int inspectionType, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) Q_DECL_OVERRIDE; private: KAsync::Job replay(Akonadi2::Storage &synchronizationStore, const QByteArray &type, const QByteArray &key, const QByteArray &value) Q_DECL_OVERRIDE; Akonadi2::ApplicationDomain::Event::Ptr createEvent(const QByteArray &rid, const QMap &data, Akonadi2::Storage::Transaction &); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1e0f6b5..38e5512 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -51,12 +51,14 @@ auto_tests ( databasepopulationandfacadequerybenchmark dummyresourcewritebenchmark modelinteractivitytest + inspectiontest ) target_link_libraries(dummyresourcetest akonadi2_resource_dummy) target_link_libraries(dummyresourcebenchmark akonadi2_resource_dummy) target_link_libraries(dummyresourcewritebenchmark akonadi2_resource_dummy) target_link_libraries(querytest akonadi2_resource_dummy) target_link_libraries(modelinteractivitytest akonadi2_resource_dummy) +target_link_libraries(inspectiontest akonadi2_resource_dummy) if (BUILD_MAILDIR) auto_tests ( diff --git a/tests/inspectiontest.cpp b/tests/inspectiontest.cpp new file mode 100644 index 0000000..e332844 --- /dev/null +++ b/tests/inspectiontest.cpp @@ -0,0 +1,64 @@ +#include + +#include + +#include "dummyresource/resourcefactory.h" +#include "clientapi.h" +#include "commands.h" +#include "entitybuffer.h" +#include "resourceconfig.h" +#include "modelresult.h" +#include "pipeline.h" +#include "log.h" + +/** + * Test of inspection system using the dummy resource. + * + * This test requires the dummy resource installed. + */ +class InspectionTest : public QObject +{ + Q_OBJECT +private Q_SLOTS: + void initTestCase() + { + Akonadi2::Log::setDebugOutputLevel(Akonadi2::Log::Trace); + auto factory = Akonadi2::ResourceFactory::load("org.kde.dummy"); + QVERIFY(factory); + DummyResource::removeFromDisk("org.kde.dummy.instance1"); + ResourceConfig::addResource("org.kde.dummy.instance1", "org.kde.dummy"); + } + + void cleanup() + { + Akonadi2::Store::shutdown(QByteArray("org.kde.dummy.instance1")).exec().waitForFinished(); + DummyResource::removeFromDisk("org.kde.dummy.instance1"); + auto factory = Akonadi2::ResourceFactory::load("org.kde.dummy"); + QVERIFY(factory); + Akonadi2::Store::start(QByteArray("org.kde.dummy.instance1")).exec().waitForFinished(); + } + + void init() + { + qDebug(); + qDebug() << "-----------------------------------------"; + qDebug(); + } + + void testMarkMailAsRead() + { + using namespace Akonadi2; + using namespace Akonadi2::ApplicationDomain; + + Mail mail(QByteArray("org.kde.dummy.instance1"), QByteArray("identifier"), 0, QSharedPointer::create()); + + auto inspectionCommand = Resources::Inspection::PropertyInspection(mail, "unread", true); + auto result = Resources::inspect(inspectionCommand).exec(); + result.waitForFinished(); + QVERIFY(!result.errorCode()); + Akonadi2::Store::flushMessageQueue(QByteArrayList() << QByteArray("org.kde.dummy.instance1")).exec().waitForFinished(); + } +}; + +QTEST_MAIN(InspectionTest) +#include "inspectiontest.moc" -- cgit v1.2.3