From f57dca821e2a33783ec0dd0af2f5c31daef4fa64 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Thu, 4 Dec 2014 19:23:09 +0100 Subject: kyoto! --- store/database.cpp | 220 ++++++++++++---------------------------- store/database.h | 5 +- store/test/CMakeLists.txt | 2 +- store/test/storagebenchmark.cpp | 21 ++-- 4 files changed, 81 insertions(+), 167 deletions(-) diff --git a/store/database.cpp b/store/database.cpp index 71ec47d..542667a 100644 --- a/store/database.cpp +++ b/store/database.cpp @@ -5,61 +5,46 @@ #include #include #include +#include #include #include #include -#include +#include class Database::Private { public: - Private(const QString &path); + Private(const QString &storageRoot, const QString &name); ~Private(); - MDB_dbi dbi; - MDB_env *env; - MDB_txn *transaction; - bool readTransaction; + kyotocabinet::TreeDB db; + bool dbOpen; + bool inTransaction; }; -Database::Private::Private(const QString &path) - : transaction(0), - readTransaction(false) +Database::Private::Private(const QString &storageRoot, const QString &name) + : inTransaction(false) { QDir dir; - dir.mkdir(path); + dir.mkdir(storageRoot); //create file - if (mdb_env_create(&env)) { + dbOpen = db.open((storageRoot + "/" + name + ".kch").toStdString(), kyotocabinet::BasicDB::OWRITER | kyotocabinet::BasicDB::OCREATE); + if (!dbOpen) { // TODO: handle error - } else { - int rc = mdb_env_open(env, path.toStdString().data(), 0, 0664); - - if (rc) { - std::cerr << "mdb_env_open: " << rc << " " << mdb_strerror(rc) << std::endl; - mdb_env_close(env); - env = 0; - } else { - const size_t dbSize = 10485760 * 100; //10MB * 100 - mdb_env_set_mapsize(env, dbSize); - } } } Database::Private::~Private() { - if (transaction) { - mdb_txn_abort(transaction); + if (dbOpen && inTransaction) { + db.end_transaction(false); } - - // it is still there and still unused, so we can shut it down - mdb_dbi_close(env, dbi); - mdb_env_close(env); } -Database::Database(const QString &path) - : d(new Private(path)) +Database::Database(const QString &storageRoot, const QString &name) + : d(new Private(storageRoot, name)) { } @@ -70,181 +55,110 @@ Database::~Database() bool Database::isInTransaction() const { - return d->transaction; + return d->inTransaction; } bool Database::startTransaction(TransactionType type) { - if (!d->env) { + if (!d->dbOpen) { return false; } - bool requestedRead = type == ReadOnly; - if (d->transaction && (!d->readTransaction || requestedRead)) { + if (d->inTransaction) { return true; } - if (d->transaction) { - // we are about to turn a read transaction into a writable one - abortTransaction(); - } - //TODO handle errors - int rc; - rc = mdb_txn_begin(d->env, NULL, requestedRead ? MDB_RDONLY : 0, &d->transaction); - if (!rc) { - rc = mdb_dbi_open(d->transaction, NULL, 0, &d->dbi); - } - - return !rc; + d->inTransaction = d->db.begin_transaction(); + return d->inTransaction; } bool Database::commitTransaction() { - if (!d->env) { + if (!d->dbOpen) { return false; } - if (!d->transaction) { + if (!d->inTransaction) { return false; } - int rc; - rc = mdb_txn_commit(d->transaction); - d->transaction = 0; - - if (rc) { - std::cerr << "mdb_txn_commit: " << rc << " " << mdb_strerror(rc) << std::endl; - } - - return !rc; + bool success = d->db.end_transaction(true); + d->inTransaction = false; + return success; } void Database::abortTransaction() { - if (!d->env || !d->transaction) { + if (!d->dbOpen || !d->inTransaction) { return; } - mdb_txn_abort(d->transaction); - d->transaction = 0; + d->db.end_transaction(false); + d->inTransaction = false; } -bool Database::write(const std::string &sKey, const std::string &sValue) +bool Database::write(const char *key, size_t keySize, const char *value, size_t valueSize) { - if (!d->env) { + if (!d->dbOpen) { return false; } - const bool implicitTransaction = !d->transaction || d->readTransaction; - if (implicitTransaction) { - // TODO: if this fails, still try the write below? - if (!startTransaction()) { - return false; - } - } - - int rc; - MDB_val key, data; - key.mv_size = sKey.size(); - key.mv_data = (void*)sKey.data(); - data.mv_size = sValue.size(); - data.mv_data = (void*)sValue.data(); - rc = mdb_put(d->transaction, d->dbi, &key, &data, 0); - - if (rc) { - std::cerr << "mdb_put: " << rc << " " << mdb_strerror(rc) << std::endl; - } + bool success = d->db.set(key, keySize, value, valueSize); + return success; +} - if (implicitTransaction) { - if (rc) { - abortTransaction(); - } else { - rc = commitTransaction(); - } +bool Database::write(const std::string &sKey, const std::string &sValue) +{ + if (!d->dbOpen) { + return false; } - return !rc; + bool success = d->db.set(sKey, sValue); + return success; } void Database::read(const std::string &sKey, const std::function &resultHandler) { - read(sKey, - [&](void *ptr, int size) { - const std::string resultValue(static_cast(ptr), size); - resultHandler(resultValue); - }); -// std::cout << "key: " << resultKey << " data: " << resultValue << std::endl; -} - -void Database::read(const std::string &sKey, const std::function &resultHandler) -{ - if (!d->env) { + if (!d->dbOpen) { return; } - int rc; - MDB_val key; - MDB_val data; - MDB_cursor *cursor; - - key.mv_size = sKey.size(); - key.mv_data = (void*)sKey.data(); - - /* -TODO: do we need a write transaction before a read transaction? only relevant for implicitTransactions in any case - { - //A write transaction is at least required the first time - rc = mdb_txn_begin(env, nullptr, 0, &txn); - //Open the database - //With this we could open multiple named databases if we wanted to - rc = mdb_dbi_open(txn, nullptr, 0, &dbi); - mdb_txn_abort(txn); - } - */ - const bool implicitTransaction = !d->transaction; - if (implicitTransaction) { - // TODO: if this fails, still try the write below? - if (!startTransaction(ReadOnly)) { - return; - } + std::string value; + if (d->db.get(sKey, &value)) { + resultHandler(value); } +} - rc = mdb_cursor_open(d->transaction, d->dbi, &cursor); - if (rc) { - std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl; +void Database::read(const std::string &sKey, const std::function &resultHandler) +{ + if (!d->dbOpen) { return; } - if (sKey.empty()) { - std::cout << "Iterating over all values of store!" << std::endl; - while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { - resultHandler(key.mv_data, data.mv_size); - } - - //We never find the last value - if (rc == MDB_NOTFOUND) { - rc = 0; - } - } else { - if ((rc = mdb_cursor_get(cursor, &key, &data, MDB_SET)) == 0) { - resultHandler(data.mv_data, data.mv_size); - } else { - std::cout << "couldn't find value " << sKey << " " << std::endl; - } - } + size_t valueSize; + char *valueBuffer = d->db.get(sKey.data(), sKey.size(), &valueSize); + resultHandler(valueBuffer, valueSize); + delete[] valueBuffer; +} - if (rc) { - std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl; +qint64 Database::diskUsage() const +{ + if (!d->dbOpen) { + return 0; } - mdb_cursor_close(cursor); + QFileInfo info(QString::fromStdString(d->db.path())); + return info.size(); +} - /** - we don't abort the transaction since we need it for reading the values - if (implicitTransaction) { - abortTransaction(); +void Database::removeFromDisk() const +{ + if (!d->dbOpen) { + return; } - */ -} + QFileInfo info(QString::fromStdString(d->db.path())); + QDir dir = info.dir(); + dir.remove(info.fileName()); +} diff --git a/store/database.h b/store/database.h index 2bf0556..e752ff5 100644 --- a/store/database.h +++ b/store/database.h @@ -7,17 +7,20 @@ class Database { public: enum TransactionType { ReadOnly, ReadWrite }; - Database(const QString &path); + Database(const QString &storageRoot, const QString &name); ~Database(); bool isInTransaction() const; bool startTransaction(TransactionType type = ReadWrite); bool commitTransaction(); void abortTransaction(); + bool write(const char *key, size_t keySize, const char *value, size_t valueSize); bool write(const std::string &sKey, const std::string &sValue); //Perhaps prefer iterators (assuming we need to be able to match multiple values void read(const std::string &sKey, const std::function &); void read(const std::string &sKey, const std::function &); + qint64 diskUsage() const; + void removeFromDisk() const; private: class Private; Private * const d; diff --git a/store/test/CMakeLists.txt b/store/test/CMakeLists.txt index 4743cfb..1b9dc9e 100644 --- a/store/test/CMakeLists.txt +++ b/store/test/CMakeLists.txt @@ -12,7 +12,7 @@ macro(manual_tests) foreach(_testname ${ARGN}) add_executable(${_testname} ${_testname}.cpp ${store_SRCS}) qt5_use_modules(${_testname} Core Test) - target_link_libraries(${_testname} lmdb) + target_link_libraries(${_testname} kyotocabinet) endforeach(_testname) endmacro(auto_tests) diff --git a/store/test/storagebenchmark.cpp b/store/test/storagebenchmark.cpp index 3be90e9..d42dea8 100644 --- a/store/test/storagebenchmark.cpp +++ b/store/test/storagebenchmark.cpp @@ -46,21 +46,16 @@ class StorageBenchmark : public QObject private: //This should point to a directory on disk and not a ramdisk (since we're measuring performance) QString testDataPath; - QString dbPath; + QString dbName; QString filePath; const int count = 50000; private Q_SLOTS: void initTestCase() { - testDataPath = "./"; - dbPath = testDataPath + "testdb"; + testDataPath = "./testdb"; + dbName = "test"; filePath = testDataPath + "buffer.fb"; - - QDir dir(testDataPath); - dir.remove("testdb/data.mdb"); - dir.remove("testdb/lock.mdb"); - dir.remove(filePath); } void testWriteRead_data() @@ -79,7 +74,7 @@ private Q_SLOTS: Database *db = 0; if (useDb) { - db = new Database(dbPath); + db = new Database(testDataPath, dbName); } std::ofstream myfile; @@ -118,7 +113,7 @@ private Q_SLOTS: { for (int i = 0; i < count; i++) { if (db) { - db->read(keyPrefix + std::to_string(i), [](void *ptr, int size){}); + db->read(keyPrefix + std::to_string(i), [](std::string value){}); } } } @@ -149,9 +144,11 @@ private Q_SLOTS: void testSizes() { - QFileInfo dbInfo(dbPath, "data.mdb"); + Database db(testDataPath, dbName); + qDebug() << "Database size [kb]: " << db.diskUsage()/1024; + db.removeFromDisk(); + QFileInfo fileInfo(filePath); - qDebug() << "Database size [kb]: " << dbInfo.size()/1024; qDebug() << "File size [kb]: " << fileInfo.size()/1024; } }; -- cgit v1.2.3 From 6081493b35855090a965aa4f81d07253458179b7 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Thu, 4 Dec 2014 21:04:36 +0100 Subject: build --- dummyresource/facade.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dummyresource/facade.cpp b/dummyresource/facade.cpp index 162f76f..012bcf0 100644 --- a/dummyresource/facade.cpp +++ b/dummyresource/facade.cpp @@ -69,14 +69,14 @@ public: DummyEvent const *buffer; //Keep query alive so values remain valid - QSharedPointer db; + QSharedPointer db; }; void DummyResourceFacade::load(const Akonadi2::Query &query, const std::function &resultCallback) { qDebug() << "load called"; //TODO only read values matching the query - auto db = QSharedPointer::create("dummyresource"); + auto db = QSharedPointer::create("dummyresource"); db->read("", [resultCallback, db](void *data, int size) { //TODO read second buffer as well auto eventBuffer = GetDummyEvent(data); -- cgit v1.2.3 From cec2b706698aa25daeb16bae8e8586aec72be747 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Thu, 4 Dec 2014 21:13:08 +0100 Subject: add a standard storage location --- client/clientapi.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/clientapi.h b/client/clientapi.h index 44fc98b..59efe32 100644 --- a/client/clientapi.h +++ b/client/clientapi.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -277,6 +278,11 @@ private: */ class Store { public: + static QString storageLocation() + { + return QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + "/akonadi2"; + } + /** * Asynchronusly load a dataset */ -- cgit v1.2.3 From 2b4e5743cca6a59e6e1a32b03863bf5ec4a4c30f Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Thu, 4 Dec 2014 21:13:17 +0100 Subject: use the standard storage location --- dummyresource/facade.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dummyresource/facade.cpp b/dummyresource/facade.cpp index 012bcf0..7e9881b 100644 --- a/dummyresource/facade.cpp +++ b/dummyresource/facade.cpp @@ -11,7 +11,7 @@ using namespace flatbuffers; DummyResourceFacade::DummyResourceFacade() : Akonadi2::StoreFacade(), mResourceAccess(new ResourceAccess("dummyresource")), - mDatabase(new Database("dummyresource")) + mDatabase(new Database(Akonadi2::Store::storageLocation(), "dummyresource")) { // connect(mResourceAccess.data(), &ResourceAccess::ready, this, onReadyChanged); } @@ -76,7 +76,7 @@ void DummyResourceFacade::load(const Akonadi2::Query &query, const std::function { qDebug() << "load called"; //TODO only read values matching the query - auto db = QSharedPointer::create("dummyresource"); + auto db = QSharedPointer::create(Akonadi2::Store::storageLocation(), "dummyresource"); db->read("", [resultCallback, db](void *data, int size) { //TODO read second buffer as well auto eventBuffer = GetDummyEvent(data); -- cgit v1.2.3 From 767312e2063f4e58af3de0f27aba52de49e14295 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 5 Dec 2014 09:17:46 +0100 Subject: major reorg that puts Storage (previously Database) into common there is now a top-level tests dir, and a compile time switch for lmdb vs kyotocabinet --- CMakeLists.txt | 6 +- common/CMakeLists.txt | 12 +- common/storage.h | 28 +++++ common/storage_kyoto.cpp | 164 +++++++++++++++++++++++++ common/storage_lmdb.cpp | 266 ++++++++++++++++++++++++++++++++++++++++ dummyresource/facade.cpp | 10 +- dummyresource/facade.h | 4 +- store/CMakeLists.txt | 1 - store/database.cpp | 164 ------------------------- store/database.h | 28 ----- store/test/CMakeLists.txt | 21 ---- store/test/calendar.fbs | 12 -- store/test/storagebenchmark.cpp | 157 ------------------------ tests/CMakeLists.txt | 16 +++ tests/calendar.fbs | 12 ++ tests/storagebenchmark.cpp | 157 ++++++++++++++++++++++++ 16 files changed, 664 insertions(+), 394 deletions(-) create mode 100644 common/storage.h create mode 100644 common/storage_kyoto.cpp create mode 100644 common/storage_lmdb.cpp delete mode 100644 store/CMakeLists.txt delete mode 100644 store/database.cpp delete mode 100644 store/database.h delete mode 100644 store/test/CMakeLists.txt delete mode 100644 store/test/calendar.fbs delete mode 100644 store/test/storagebenchmark.cpp create mode 100644 tests/CMakeLists.txt create mode 100644 tests/calendar.fbs create mode 100644 tests/storagebenchmark.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 392ca39..e8b6f23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,10 +47,10 @@ add_subdirectory(client) # the resource add_subdirectory(resource) -# the store -add_subdirectory(store) - # a simple dummy resource implementation add_subdirectory(dummyresource) +# some tests +add_subdirectory(tests) + feature_summary(WHAT ALL FATAL_ON_MISSING_REQUIRED_PACKAGES) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index d409828..9b3f777 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -2,10 +2,20 @@ project(akonadinextcommon) generate_flatbuffers(commands/handshake commands/revisionupdate) +if (STORAGE_KYOTO) + set(storage_SRCS storage_kyoto.cpp) + set(storage_LIBS kyotocabinet) +else (STORAGE_KYOTO) + set(storage_SRCS storage_lmdb.cpp) + set(storage_LIBS lmdb) +endif (STORAGE_KYOTO) + set(command_SRCS commands.cpp console.cpp - ${CMAKE_CURRENT_BINARY_DIR}/commands/handshake_generated.h) + ${storage_SRCS}) add_library(${PROJECT_NAME} ${command_SRCS}) +SET_TARGET_PROPERTIES(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) qt5_use_modules(${PROJECT_NAME} Widgets) +target_link_libraries(${PROJECT_NAME} ${storage_LIBS}) diff --git a/common/storage.h b/common/storage.h new file mode 100644 index 0000000..f7dbd89 --- /dev/null +++ b/common/storage.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +class Storage { +public: + enum TransactionType { ReadOnly, ReadWrite }; + + Storage(const QString &storageRoot, const QString &name); + ~Storage(); + bool isInTransaction() const; + bool startTransaction(TransactionType type = ReadWrite); + bool commitTransaction(); + void abortTransaction(); + bool write(const char *key, size_t keySize, const char *value, size_t valueSize); + bool write(const std::string &sKey, const std::string &sValue); + //Perhaps prefer iterators (assuming we need to be able to match multiple values + void read(const std::string &sKey, const std::function &); + void read(const std::string &sKey, const std::function &); + + qint64 diskUsage() const; + void removeFromDisk() const; +private: + class Private; + Private * const d; +}; + diff --git a/common/storage_kyoto.cpp b/common/storage_kyoto.cpp new file mode 100644 index 0000000..05942c2 --- /dev/null +++ b/common/storage_kyoto.cpp @@ -0,0 +1,164 @@ +#include "storage.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +class Storage::Private +{ +public: + Private(const QString &storageRoot, const QString &name); + ~Private(); + + kyotocabinet::TreeDB db; + bool dbOpen; + bool inTransaction; +}; + +Storage::Private::Private(const QString &storageRoot, const QString &name) + : inTransaction(false) +{ + QDir dir; + dir.mkdir(storageRoot); + + //create file + dbOpen = db.open((storageRoot + "/" + name + ".kch").toStdString(), kyotocabinet::BasicDB::OWRITER | kyotocabinet::BasicDB::OCREATE); + if (!dbOpen) { + // TODO: handle error + } +} + +Storage::Private::~Private() +{ + if (dbOpen && inTransaction) { + db.end_transaction(false); + } +} + +Storage::Storage(const QString &storageRoot, const QString &name) + : d(new Private(storageRoot, name)) +{ +} + +Storage::~Storage() +{ + delete d; +} + +bool Storage::isInTransaction() const +{ + return d->inTransaction; +} + +bool Storage::startTransaction(TransactionType type) +{ + if (!d->dbOpen) { + return false; + } + + if (d->inTransaction) { + return true; + } + + //TODO handle errors + d->inTransaction = d->db.begin_transaction(); + return d->inTransaction; +} + +bool Storage::commitTransaction() +{ + if (!d->dbOpen) { + return false; + } + + if (!d->inTransaction) { + return false; + } + + bool success = d->db.end_transaction(true); + d->inTransaction = false; + return success; +} + +void Storage::abortTransaction() +{ + if (!d->dbOpen || !d->inTransaction) { + return; + } + + d->db.end_transaction(false); + d->inTransaction = false; +} + +bool Storage::write(const char *key, size_t keySize, const char *value, size_t valueSize) +{ + if (!d->dbOpen) { + return false; + } + + bool success = d->db.set(key, keySize, value, valueSize); + return success; +} + +bool Storage::write(const std::string &sKey, const std::string &sValue) +{ + if (!d->dbOpen) { + return false; + } + + bool success = d->db.set(sKey, sValue); + return success; +} + +void Storage::read(const std::string &sKey, const std::function &resultHandler) +{ + if (!d->dbOpen) { + return; + } + + std::string value; + if (d->db.get(sKey, &value)) { + resultHandler(value); + } +} + +void Storage::read(const std::string &sKey, const std::function &resultHandler) +{ + if (!d->dbOpen) { + return; + } + + size_t valueSize; + char *valueBuffer = d->db.get(sKey.data(), sKey.size(), &valueSize); + resultHandler(valueBuffer, valueSize); + delete[] valueBuffer; +} + +qint64 Storage::diskUsage() const +{ + if (!d->dbOpen) { + return 0; + } + + QFileInfo info(QString::fromStdString(d->db.path())); + return info.size(); +} + +void Storage::removeFromDisk() const +{ + if (!d->dbOpen) { + return; + } + + QFileInfo info(QString::fromStdString(d->db.path())); + QDir dir = info.dir(); + dir.remove(info.fileName()); +} diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp new file mode 100644 index 0000000..6c25448 --- /dev/null +++ b/common/storage_lmdb.cpp @@ -0,0 +1,266 @@ +#include "storage.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include + +class Storage::Private +{ +public: + Private(const QString &p); + ~Private(); + + QString path; + MDB_dbi dbi; + MDB_env *env; + MDB_txn *transaction; + bool readTransaction; + bool firstOpen; +}; + +Storage::Private::Private(const QString &p) + : path(p), + transaction(0), + readTransaction(false), + firstOpen(true) +{ + QDir dir; + dir.mkdir(path); + + //create file + if (mdb_env_create(&env)) { + // TODO: handle error + } else { + int rc = mdb_env_open(env, path.toStdString().data(), 0, 0664); + + if (rc) { + std::cerr << "mdb_env_open: " << rc << " " << mdb_strerror(rc) << std::endl; + mdb_env_close(env); + env = 0; + } else { + const size_t dbSize = 10485760 * 100; //10MB * 100 + mdb_env_set_mapsize(env, dbSize); + } + } +} + +Storage::Private::~Private() +{ + if (transaction) { + mdb_txn_abort(transaction); + } + + // it is still there and still unused, so we can shut it down + mdb_dbi_close(env, dbi); + mdb_env_close(env); +} + +Storage::Storage(const QString &storageRoot, const QString &name) + : d(new Private(storageRoot + '/' + name)) +{ +} + +Storage::~Storage() +{ + delete d; +} + +bool Storage::isInTransaction() const +{ + return d->transaction; +} + +bool Storage::startTransaction(TransactionType type) +{ + if (!d->env) { + return false; + } + + bool requestedRead = type == ReadOnly; + if (d->transaction && (!d->readTransaction || requestedRead)) { + return true; + } + + if (d->transaction) { + // we are about to turn a read transaction into a writable one + abortTransaction(); + } + + if (d->firstOpen && requestedRead) { + //A write transaction is at least required the first time + mdb_txn_begin(d->env, nullptr, 0, &d->transaction); + //Open the database + //With this we could open multiple named databases if we wanted to + mdb_dbi_open(d->transaction, nullptr, 0, &d->dbi); + mdb_txn_abort(d->transaction); + } + + int rc; + rc = mdb_txn_begin(d->env, NULL, requestedRead ? MDB_RDONLY : 0, &d->transaction); + if (!rc) { + rc = mdb_dbi_open(d->transaction, NULL, 0, &d->dbi); + } + + d->firstOpen = false; + return !rc; +} + +bool Storage::commitTransaction() +{ + if (!d->env) { + return false; + } + + if (!d->transaction) { + return false; + } + + int rc; + rc = mdb_txn_commit(d->transaction); + d->transaction = 0; + + if (rc) { + std::cerr << "mdb_txn_commit: " << rc << " " << mdb_strerror(rc) << std::endl; + } + + return !rc; +} + +void Storage::abortTransaction() +{ + if (!d->env || !d->transaction) { + return; + } + + mdb_txn_abort(d->transaction); + d->transaction = 0; +} + +bool Storage::write(const std::string &sKey, const std::string &sValue) +{ + if (!d->env) { + return false; + } + + const bool implicitTransaction = !d->transaction || d->readTransaction; + if (implicitTransaction) { + // TODO: if this fails, still try the write below? + if (!startTransaction()) { + return false; + } + } + + int rc; + MDB_val key, data; + key.mv_size = sKey.size(); + key.mv_data = (void*)sKey.data(); + data.mv_size = sValue.size(); + data.mv_data = (void*)sValue.data(); + rc = mdb_put(d->transaction, d->dbi, &key, &data, 0); + + if (rc) { + std::cerr << "mdb_put: " << rc << " " << mdb_strerror(rc) << std::endl; + } + + if (implicitTransaction) { + if (rc) { + abortTransaction(); + } else { + rc = commitTransaction(); + } + } + + return !rc; +} + +void Storage::read(const std::string &sKey, const std::function &resultHandler) +{ + read(sKey, + [&](void *ptr, int size) { + const std::string resultValue(static_cast(ptr), size); + resultHandler(resultValue); + }); +// std::cout << "key: " << resultKey << " data: " << resultValue << std::endl; +} + +void Storage::read(const std::string &sKey, const std::function &resultHandler) +{ + if (!d->env) { + return; + } + + int rc; + MDB_val key; + MDB_val data; + MDB_cursor *cursor; + + key.mv_size = sKey.size(); + key.mv_data = (void*)sKey.data(); + + const bool implicitTransaction = !d->transaction; + if (implicitTransaction) { + // TODO: if this fails, still try the write below? + if (!startTransaction(ReadOnly)) { + return; + } + } + + rc = mdb_cursor_open(d->transaction, d->dbi, &cursor); + if (rc) { + std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl; + return; + } + + if (sKey.empty()) { + std::cout << "Iterating over all values of store!" << std::endl; + rc = mdb_cursor_get(cursor, &key, &data, MDB_FIRST); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + resultHandler(key.mv_data, data.mv_size); + } + + //We never find the last value + if (rc == MDB_NOTFOUND) { + rc = 0; + } + } else { + if ((rc = mdb_cursor_get(cursor, &key, &data, MDB_SET)) == 0) { + resultHandler(data.mv_data, data.mv_size); + } else { + std::cout << "couldn't find value " << sKey << " " << std::endl; + } + } + + if (rc) { + std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl; + } + + mdb_cursor_close(cursor); + + /** + we don't abort the transaction since we need it for reading the values + if (implicitTransaction) { + abortTransaction(); + } + */ +} + +qint64 Storage::diskUsage() const +{ + QFileInfo info(d->path, "data.mdb"); + return info.size(); +} + +void Storage::removeFromDisk() const +{ + QDir dir(d->path); + dir.remove("data.mdb"); + dir.remove("lock.mdb"); +} + diff --git a/dummyresource/facade.cpp b/dummyresource/facade.cpp index 7e9881b..9758c1b 100644 --- a/dummyresource/facade.cpp +++ b/dummyresource/facade.cpp @@ -11,7 +11,7 @@ using namespace flatbuffers; DummyResourceFacade::DummyResourceFacade() : Akonadi2::StoreFacade(), mResourceAccess(new ResourceAccess("dummyresource")), - mDatabase(new Database(Akonadi2::Store::storageLocation(), "dummyresource")) + mStorage(new Storage(Akonadi2::Store::storageLocation(), "dummyresource")) { // connect(mResourceAccess.data(), &ResourceAccess::ready, this, onReadyChanged); } @@ -69,20 +69,20 @@ public: DummyEvent const *buffer; //Keep query alive so values remain valid - QSharedPointer db; + QSharedPointer storage; }; void DummyResourceFacade::load(const Akonadi2::Query &query, const std::function &resultCallback) { qDebug() << "load called"; //TODO only read values matching the query - auto db = QSharedPointer::create(Akonadi2::Store::storageLocation(), "dummyresource"); - db->read("", [resultCallback, db](void *data, int size) { + auto storage = QSharedPointer::create(Akonadi2::Store::storageLocation(), "dummyresource"); + storage->read("", [resultCallback, storage](void *data, int size) { //TODO read second buffer as well auto eventBuffer = GetDummyEvent(data); auto event = QSharedPointer::create("dummyresource", "key", 0); event->buffer = eventBuffer; - event->db = db; + event->storage = storage; resultCallback(event); }); } diff --git a/dummyresource/facade.h b/dummyresource/facade.h index 7a516de..310ef77 100644 --- a/dummyresource/facade.h +++ b/dummyresource/facade.h @@ -1,7 +1,7 @@ #pragma once #include "client/clientapi.h" -#include "store/database.h" +#include "common/storage.h" class ResourceAccess; @@ -18,5 +18,5 @@ public: private: QSharedPointer mResourceAccess; - QSharedPointer mDatabase; + QSharedPointer mStorage; }; diff --git a/store/CMakeLists.txt b/store/CMakeLists.txt deleted file mode 100644 index 552439e..0000000 --- a/store/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(test) diff --git a/store/database.cpp b/store/database.cpp deleted file mode 100644 index 542667a..0000000 --- a/store/database.cpp +++ /dev/null @@ -1,164 +0,0 @@ -#include "database.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -class Database::Private -{ -public: - Private(const QString &storageRoot, const QString &name); - ~Private(); - - kyotocabinet::TreeDB db; - bool dbOpen; - bool inTransaction; -}; - -Database::Private::Private(const QString &storageRoot, const QString &name) - : inTransaction(false) -{ - QDir dir; - dir.mkdir(storageRoot); - - //create file - dbOpen = db.open((storageRoot + "/" + name + ".kch").toStdString(), kyotocabinet::BasicDB::OWRITER | kyotocabinet::BasicDB::OCREATE); - if (!dbOpen) { - // TODO: handle error - } -} - -Database::Private::~Private() -{ - if (dbOpen && inTransaction) { - db.end_transaction(false); - } -} - -Database::Database(const QString &storageRoot, const QString &name) - : d(new Private(storageRoot, name)) -{ -} - -Database::~Database() -{ - delete d; -} - -bool Database::isInTransaction() const -{ - return d->inTransaction; -} - -bool Database::startTransaction(TransactionType type) -{ - if (!d->dbOpen) { - return false; - } - - if (d->inTransaction) { - return true; - } - - //TODO handle errors - d->inTransaction = d->db.begin_transaction(); - return d->inTransaction; -} - -bool Database::commitTransaction() -{ - if (!d->dbOpen) { - return false; - } - - if (!d->inTransaction) { - return false; - } - - bool success = d->db.end_transaction(true); - d->inTransaction = false; - return success; -} - -void Database::abortTransaction() -{ - if (!d->dbOpen || !d->inTransaction) { - return; - } - - d->db.end_transaction(false); - d->inTransaction = false; -} - -bool Database::write(const char *key, size_t keySize, const char *value, size_t valueSize) -{ - if (!d->dbOpen) { - return false; - } - - bool success = d->db.set(key, keySize, value, valueSize); - return success; -} - -bool Database::write(const std::string &sKey, const std::string &sValue) -{ - if (!d->dbOpen) { - return false; - } - - bool success = d->db.set(sKey, sValue); - return success; -} - -void Database::read(const std::string &sKey, const std::function &resultHandler) -{ - if (!d->dbOpen) { - return; - } - - std::string value; - if (d->db.get(sKey, &value)) { - resultHandler(value); - } -} - -void Database::read(const std::string &sKey, const std::function &resultHandler) -{ - if (!d->dbOpen) { - return; - } - - size_t valueSize; - char *valueBuffer = d->db.get(sKey.data(), sKey.size(), &valueSize); - resultHandler(valueBuffer, valueSize); - delete[] valueBuffer; -} - -qint64 Database::diskUsage() const -{ - if (!d->dbOpen) { - return 0; - } - - QFileInfo info(QString::fromStdString(d->db.path())); - return info.size(); -} - -void Database::removeFromDisk() const -{ - if (!d->dbOpen) { - return; - } - - QFileInfo info(QString::fromStdString(d->db.path())); - QDir dir = info.dir(); - dir.remove(info.fileName()); -} diff --git a/store/database.h b/store/database.h deleted file mode 100644 index e752ff5..0000000 --- a/store/database.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include -#include - -class Database { -public: - enum TransactionType { ReadOnly, ReadWrite }; - - Database(const QString &storageRoot, const QString &name); - ~Database(); - bool isInTransaction() const; - bool startTransaction(TransactionType type = ReadWrite); - bool commitTransaction(); - void abortTransaction(); - bool write(const char *key, size_t keySize, const char *value, size_t valueSize); - bool write(const std::string &sKey, const std::string &sValue); - //Perhaps prefer iterators (assuming we need to be able to match multiple values - void read(const std::string &sKey, const std::function &); - void read(const std::string &sKey, const std::function &); - - qint64 diskUsage() const; - void removeFromDisk() const; -private: - class Private; - Private * const d; -}; - diff --git a/store/test/CMakeLists.txt b/store/test/CMakeLists.txt deleted file mode 100644 index 1b9dc9e..0000000 --- a/store/test/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -set(CMAKE_AUTOMOC ON) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -set(store_path "../") -set(store_SRCS - ${store_path}/database.cpp -) - -generate_flatbuffers(calendar) - -macro(manual_tests) - foreach(_testname ${ARGN}) - add_executable(${_testname} ${_testname}.cpp ${store_SRCS}) - qt5_use_modules(${_testname} Core Test) - target_link_libraries(${_testname} kyotocabinet) - endforeach(_testname) -endmacro(auto_tests) - -manual_tests ( - storagebenchmark -) diff --git a/store/test/calendar.fbs b/store/test/calendar.fbs deleted file mode 100644 index 203ee43..0000000 --- a/store/test/calendar.fbs +++ /dev/null @@ -1,12 +0,0 @@ -// example IDL file - -namespace Calendar; - -table Event { - summary:string; - description:string; - attachment:[byte]; -} - -root_type Event; -file_identifier "AKFB"; diff --git a/store/test/storagebenchmark.cpp b/store/test/storagebenchmark.cpp deleted file mode 100644 index d42dea8..0000000 --- a/store/test/storagebenchmark.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include - -#include "calendar_generated.h" - -#include -#include - -#include -#include -#include -#include - -#include "store/database.h" - -using namespace Calendar; -using namespace flatbuffers; - -static std::string createEvent() -{ - FlatBufferBuilder fbb; - { - auto summary = fbb.CreateString("summary"); - - const int attachmentSize = 1024*2; // 2KB - int8_t rawData[attachmentSize]; - auto data = fbb.CreateVector(rawData, attachmentSize); - - Calendar::EventBuilder eventBuilder(fbb); - eventBuilder.add_summary(summary); - eventBuilder.add_attachment(data); - auto eventLocation = eventBuilder.Finish(); - FinishEventBuffer(fbb, eventLocation); - } - return std::string(reinterpret_cast(fbb.GetBufferPointer()), fbb.GetSize()); -} - -// static void readEvent(const std::string &data) -// { -// auto readEvent = GetEvent(data.c_str()); -// std::cout << readEvent->summary()->c_str() << std::endl; -// } - -class StorageBenchmark : public QObject -{ - Q_OBJECT -private: - //This should point to a directory on disk and not a ramdisk (since we're measuring performance) - QString testDataPath; - QString dbName; - QString filePath; - const int count = 50000; - -private Q_SLOTS: - void initTestCase() - { - testDataPath = "./testdb"; - dbName = "test"; - filePath = testDataPath + "buffer.fb"; - } - - void testWriteRead_data() - { - QTest::addColumn("useDb"); - QTest::addColumn("count"); - - QTest::newRow("db, 50k") << true << count; - QTest::newRow("file, 50k") << false << count; - } - - void testWriteRead() - { - QFETCH(bool, useDb); - QFETCH(int, count); - - Database *db = 0; - if (useDb) { - db = new Database(testDataPath, dbName); - } - - std::ofstream myfile; - myfile.open(filePath.toStdString()); - const char *keyPrefix = "key"; - - QTime time; - - time.start(); - { - auto event = createEvent(); - for (int i = 0; i < count; i++) { - if (db) { - if (i % 10000 == 0) { - if (i > 0) { - db->commitTransaction(); - } - db->startTransaction(); - } - - db->write(keyPrefix + std::to_string(i), event); - } else { - myfile << event; - } - } - - if (db) { - db->commitTransaction(); - } else { - myfile.close(); - } - } - const int writeDuration = time.restart(); - qDebug() << "Writing took[ms]: " << writeDuration; - - { - for (int i = 0; i < count; i++) { - if (db) { - db->read(keyPrefix + std::to_string(i), [](std::string value){}); - } - } - } - const int readDuration = time.restart(); - - if (db) { - qDebug() << "Reading took[ms]: " << readDuration; - } else { - qDebug() << "File reading is not implemented."; - } - - delete db; - } - - void testBufferCreation() - { - QTime time; - - time.start(); - { - for (int i = 0; i < count; i++) { - auto event = createEvent(); - } - } - const int bufferDuration = time.elapsed(); - qDebug() << "Creating buffers took[ms]: " << bufferDuration; - } - - void testSizes() - { - Database db(testDataPath, dbName); - qDebug() << "Database size [kb]: " << db.diskUsage()/1024; - db.removeFromDisk(); - - QFileInfo fileInfo(filePath); - qDebug() << "File size [kb]: " << fileInfo.size()/1024; - } -}; - -QTEST_MAIN(StorageBenchmark) -#include "storagebenchmark.moc" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..23776a1 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,16 @@ +set(CMAKE_AUTOMOC ON) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +generate_flatbuffers(calendar) + +macro(manual_tests) + foreach(_testname ${ARGN}) + add_executable(${_testname} ${_testname}.cpp) + qt5_use_modules(${_testname} Core Test) + target_link_libraries(${_testname} akonadinextcommon) + endforeach(_testname) +endmacro(manual_tests) + +manual_tests ( + storagebenchmark +) diff --git a/tests/calendar.fbs b/tests/calendar.fbs new file mode 100644 index 0000000..203ee43 --- /dev/null +++ b/tests/calendar.fbs @@ -0,0 +1,12 @@ +// example IDL file + +namespace Calendar; + +table Event { + summary:string; + description:string; + attachment:[byte]; +} + +root_type Event; +file_identifier "AKFB"; diff --git a/tests/storagebenchmark.cpp b/tests/storagebenchmark.cpp new file mode 100644 index 0000000..15da9da --- /dev/null +++ b/tests/storagebenchmark.cpp @@ -0,0 +1,157 @@ +#include + +#include "calendar_generated.h" + +#include +#include + +#include +#include +#include +#include + +#include "common/storage.h" + +using namespace Calendar; +using namespace flatbuffers; + +static std::string createEvent() +{ + FlatBufferBuilder fbb; + { + auto summary = fbb.CreateString("summary"); + + const int attachmentSize = 1024*2; // 2KB + int8_t rawData[attachmentSize]; + auto data = fbb.CreateVector(rawData, attachmentSize); + + Calendar::EventBuilder eventBuilder(fbb); + eventBuilder.add_summary(summary); + eventBuilder.add_attachment(data); + auto eventLocation = eventBuilder.Finish(); + FinishEventBuffer(fbb, eventLocation); + } + return std::string(reinterpret_cast(fbb.GetBufferPointer()), fbb.GetSize()); +} + +// static void readEvent(const std::string &data) +// { +// auto readEvent = GetEvent(data.c_str()); +// std::cout << readEvent->summary()->c_str() << std::endl; +// } + +class StorageBenchmark : public QObject +{ + Q_OBJECT +private: + //This should point to a directory on disk and not a ramdisk (since we're measuring performance) + QString testDataPath; + QString dbName; + QString filePath; + const int count = 50000; + +private Q_SLOTS: + void initTestCase() + { + testDataPath = "./testdb"; + dbName = "test"; + filePath = testDataPath + "buffer.fb"; + } + + void testWriteRead_data() + { + QTest::addColumn("useDb"); + QTest::addColumn("count"); + + QTest::newRow("db, 50k") << true << count; + QTest::newRow("file, 50k") << false << count; + } + + void testWriteRead() + { + QFETCH(bool, useDb); + QFETCH(int, count); + + Storage *store = 0; + if (useDb) { + store = new Storage(testDataPath, dbName); + } + + std::ofstream myfile; + myfile.open(filePath.toStdString()); + const char *keyPrefix = "key"; + + QTime time; + + time.start(); + { + auto event = createEvent(); + for (int i = 0; i < count; i++) { + if (store) { + if (i % 10000 == 0) { + if (i > 0) { + store->commitTransaction(); + } + store->startTransaction(); + } + + store->write(keyPrefix + std::to_string(i), event); + } else { + myfile << event; + } + } + + if (store) { + store->commitTransaction(); + } else { + myfile.close(); + } + } + const int writeDuration = time.restart(); + qDebug() << "Writing took[ms]: " << writeDuration; + + { + for (int i = 0; i < count; i++) { + if (store) { + store->read(keyPrefix + std::to_string(i), [](std::string value){}); + } + } + } + const int readDuration = time.restart(); + + if (store) { + qDebug() << "Reading took[ms]: " << readDuration; + } else { + qDebug() << "File reading is not implemented."; + } + + delete store; + } + + void testBufferCreation() + { + QTime time; + + time.start(); + { + for (int i = 0; i < count; i++) { + auto event = createEvent(); + } + } + const int bufferDuration = time.elapsed(); + qDebug() << "Creating buffers took[ms]: " << bufferDuration; + } + + void testSizes() + { + Storage store(testDataPath, dbName); + qDebug() << "Database size [kb]: " << store.diskUsage()/1024; + store.removeFromDisk(); + + QFileInfo fileInfo(filePath); + qDebug() << "File size [kb]: " << fileInfo.size()/1024; + } +}; + +QTEST_MAIN(StorageBenchmark) +#include "storagebenchmark.moc" -- cgit v1.2.3 From 88085fd52f886692c9cbb534166e68b791d5abce Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 5 Dec 2014 09:29:37 +0100 Subject: port over 707344 by cmollekopf: small cleanup --- tests/storagebenchmark.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/storagebenchmark.cpp b/tests/storagebenchmark.cpp index 15da9da..f9fea7c 100644 --- a/tests/storagebenchmark.cpp +++ b/tests/storagebenchmark.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include @@ -58,6 +57,12 @@ private Q_SLOTS: filePath = testDataPath + "buffer.fb"; } + void cleanupTestCase() + { + Storage store(testDataPath, dbName); + store.removeFromDisk(); + } + void testWriteRead_data() { QTest::addColumn("useDb"); @@ -146,7 +151,6 @@ private Q_SLOTS: { Storage store(testDataPath, dbName); qDebug() << "Database size [kb]: " << store.diskUsage()/1024; - store.removeFromDisk(); QFileInfo fileInfo(filePath); qDebug() << "File size [kb]: " << fileInfo.size()/1024; -- cgit v1.2.3 From ee41f8d17bdc667fbbbc83deeff766faf048cf5e Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 5 Dec 2014 00:57:03 +0100 Subject: A storagetest including concurrency read test. Conflicts: tests/CMakeLists.txt --- store/test/storagetest.cpp | 111 +++++++++++++++++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 3 +- 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 store/test/storagetest.cpp diff --git a/store/test/storagetest.cpp b/store/test/storagetest.cpp new file mode 100644 index 0000000..1b105af --- /dev/null +++ b/store/test/storagetest.cpp @@ -0,0 +1,111 @@ +#include + +#include + +#include +#include +#include + +#include "store/database.h" + +class StorageTest : public QObject +{ + Q_OBJECT +private: + //This should point to a directory on disk and not a ramdisk (since we're measuring performance) + QString testDataPath; + QString dbName; + const char *keyPrefix = "key"; + + void populate(int count) + { + Database db(testDataPath, dbName); + for (int i = 0; i < count; i++) { + //This should perhaps become an implementation detail of the db? + if (i % 10000 == 0) { + if (i > 0) { + db.commitTransaction(); + } + db.startTransaction(); + } + db.write(keyPrefix + std::to_string(i), keyPrefix + std::to_string(i)); + } + db.commitTransaction(); + } + + bool verify(Database &db, int i) + { + bool error = false; + const auto reference = keyPrefix + std::to_string(i); + db.read(keyPrefix + std::to_string(i), [&error, &reference](const std::string &value) { + if (value != reference) { + qDebug() << "Mismatch while reading"; + error = true; + } + }); + return !error; + } + +private Q_SLOTS: + void initTestCase() + { + testDataPath = "./testdb"; + dbName = "test"; + } + + void cleanupTestCase() + { + Database db(testDataPath, dbName); + db.removeFromDisk(); + } + + + void testRead() + { + const int count = 100; + + populate(count); + + //ensure we can read everything back correctly + { + Database db(testDataPath, dbName); + for (int i = 0; i < count; i++) { + QVERIFY(verify(db, i)); + } + } + + Database db(testDataPath, dbName); + db.removeFromDisk(); + } + + void testConcurrentRead() + { + const int count = 10000; + + populate(count); + + //Try to concurrently read + QList > futures; + const int concurrencyLevel = 4; + for (int num = 0; num < concurrencyLevel; num++) { + futures << QtConcurrent::run([this, count](){ + Database db(testDataPath, dbName); + for (int i = 0; i < count; i++) { + if (!verify(db, i)) { + qWarning() << "invalid value"; + break; + } + } + }); + } + for(auto future : futures) { + future.waitForFinished(); + } + + Database db(testDataPath, dbName); + db.removeFromDisk(); + } +}; + +QTEST_MAIN(StorageTest) +#include "storagetest.moc" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 23776a1..1629acb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -6,11 +6,12 @@ generate_flatbuffers(calendar) macro(manual_tests) foreach(_testname ${ARGN}) add_executable(${_testname} ${_testname}.cpp) - qt5_use_modules(${_testname} Core Test) + qt5_use_modules(${_testname} Core Test Concurrent) target_link_libraries(${_testname} akonadinextcommon) endforeach(_testname) endmacro(manual_tests) manual_tests ( storagebenchmark + storagetest ) -- cgit v1.2.3 From 0c1400c7f0cf2f545a6cd7347314c1158fbfa36f Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 5 Dec 2014 09:33:35 +0100 Subject: mv storagetest.cpp to the right location --- store/test/storagetest.cpp | 111 --------------------------------------------- tests/storagetest.cpp | 111 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 111 deletions(-) delete mode 100644 store/test/storagetest.cpp create mode 100644 tests/storagetest.cpp diff --git a/store/test/storagetest.cpp b/store/test/storagetest.cpp deleted file mode 100644 index 1b105af..0000000 --- a/store/test/storagetest.cpp +++ /dev/null @@ -1,111 +0,0 @@ -#include - -#include - -#include -#include -#include - -#include "store/database.h" - -class StorageTest : public QObject -{ - Q_OBJECT -private: - //This should point to a directory on disk and not a ramdisk (since we're measuring performance) - QString testDataPath; - QString dbName; - const char *keyPrefix = "key"; - - void populate(int count) - { - Database db(testDataPath, dbName); - for (int i = 0; i < count; i++) { - //This should perhaps become an implementation detail of the db? - if (i % 10000 == 0) { - if (i > 0) { - db.commitTransaction(); - } - db.startTransaction(); - } - db.write(keyPrefix + std::to_string(i), keyPrefix + std::to_string(i)); - } - db.commitTransaction(); - } - - bool verify(Database &db, int i) - { - bool error = false; - const auto reference = keyPrefix + std::to_string(i); - db.read(keyPrefix + std::to_string(i), [&error, &reference](const std::string &value) { - if (value != reference) { - qDebug() << "Mismatch while reading"; - error = true; - } - }); - return !error; - } - -private Q_SLOTS: - void initTestCase() - { - testDataPath = "./testdb"; - dbName = "test"; - } - - void cleanupTestCase() - { - Database db(testDataPath, dbName); - db.removeFromDisk(); - } - - - void testRead() - { - const int count = 100; - - populate(count); - - //ensure we can read everything back correctly - { - Database db(testDataPath, dbName); - for (int i = 0; i < count; i++) { - QVERIFY(verify(db, i)); - } - } - - Database db(testDataPath, dbName); - db.removeFromDisk(); - } - - void testConcurrentRead() - { - const int count = 10000; - - populate(count); - - //Try to concurrently read - QList > futures; - const int concurrencyLevel = 4; - for (int num = 0; num < concurrencyLevel; num++) { - futures << QtConcurrent::run([this, count](){ - Database db(testDataPath, dbName); - for (int i = 0; i < count; i++) { - if (!verify(db, i)) { - qWarning() << "invalid value"; - break; - } - } - }); - } - for(auto future : futures) { - future.waitForFinished(); - } - - Database db(testDataPath, dbName); - db.removeFromDisk(); - } -}; - -QTEST_MAIN(StorageTest) -#include "storagetest.moc" diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp new file mode 100644 index 0000000..1b105af --- /dev/null +++ b/tests/storagetest.cpp @@ -0,0 +1,111 @@ +#include + +#include + +#include +#include +#include + +#include "store/database.h" + +class StorageTest : public QObject +{ + Q_OBJECT +private: + //This should point to a directory on disk and not a ramdisk (since we're measuring performance) + QString testDataPath; + QString dbName; + const char *keyPrefix = "key"; + + void populate(int count) + { + Database db(testDataPath, dbName); + for (int i = 0; i < count; i++) { + //This should perhaps become an implementation detail of the db? + if (i % 10000 == 0) { + if (i > 0) { + db.commitTransaction(); + } + db.startTransaction(); + } + db.write(keyPrefix + std::to_string(i), keyPrefix + std::to_string(i)); + } + db.commitTransaction(); + } + + bool verify(Database &db, int i) + { + bool error = false; + const auto reference = keyPrefix + std::to_string(i); + db.read(keyPrefix + std::to_string(i), [&error, &reference](const std::string &value) { + if (value != reference) { + qDebug() << "Mismatch while reading"; + error = true; + } + }); + return !error; + } + +private Q_SLOTS: + void initTestCase() + { + testDataPath = "./testdb"; + dbName = "test"; + } + + void cleanupTestCase() + { + Database db(testDataPath, dbName); + db.removeFromDisk(); + } + + + void testRead() + { + const int count = 100; + + populate(count); + + //ensure we can read everything back correctly + { + Database db(testDataPath, dbName); + for (int i = 0; i < count; i++) { + QVERIFY(verify(db, i)); + } + } + + Database db(testDataPath, dbName); + db.removeFromDisk(); + } + + void testConcurrentRead() + { + const int count = 10000; + + populate(count); + + //Try to concurrently read + QList > futures; + const int concurrencyLevel = 4; + for (int num = 0; num < concurrencyLevel; num++) { + futures << QtConcurrent::run([this, count](){ + Database db(testDataPath, dbName); + for (int i = 0; i < count; i++) { + if (!verify(db, i)) { + qWarning() << "invalid value"; + break; + } + } + }); + } + for(auto future : futures) { + future.waitForFinished(); + } + + Database db(testDataPath, dbName); + db.removeFromDisk(); + } +}; + +QTEST_MAIN(StorageTest) +#include "storagetest.moc" -- cgit v1.2.3 From 351a66b5fb1c8659bff8ea20d60f5a6d2d3263ad Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 5 Dec 2014 09:46:53 +0100 Subject: make read return a bool on success not happy with this API, but we need to discuss the whole read thing anyways --- common/storage.h | 4 ++-- common/storage_kyoto.cpp | 16 +++++++++++----- common/storage_lmdb.cpp | 18 ++++++++++-------- tests/storagetest.cpp | 23 ++++++++++++----------- 4 files changed, 35 insertions(+), 26 deletions(-) diff --git a/common/storage.h b/common/storage.h index f7dbd89..0b548fb 100644 --- a/common/storage.h +++ b/common/storage.h @@ -16,8 +16,8 @@ public: bool write(const char *key, size_t keySize, const char *value, size_t valueSize); bool write(const std::string &sKey, const std::string &sValue); //Perhaps prefer iterators (assuming we need to be able to match multiple values - void read(const std::string &sKey, const std::function &); - void read(const std::string &sKey, const std::function &); + bool read(const std::string &sKey, const std::function &); + bool read(const std::string &sKey, const std::function &); qint64 diskUsage() const; void removeFromDisk() const; diff --git a/common/storage_kyoto.cpp b/common/storage_kyoto.cpp index 05942c2..40bd3e6 100644 --- a/common/storage_kyoto.cpp +++ b/common/storage_kyoto.cpp @@ -118,28 +118,34 @@ bool Storage::write(const std::string &sKey, const std::string &sValue) return success; } -void Storage::read(const std::string &sKey, const std::function &resultHandler) +bool Storage::read(const std::string &sKey, const std::function &resultHandler) { if (!d->dbOpen) { - return; + return false; } std::string value; if (d->db.get(sKey, &value)) { resultHandler(value); + return true; } + + return false; } -void Storage::read(const std::string &sKey, const std::function &resultHandler) +bool Storage::read(const std::string &sKey, const std::function &resultHandler) { if (!d->dbOpen) { - return; + return false; } size_t valueSize; char *valueBuffer = d->db.get(sKey.data(), sKey.size(), &valueSize); - resultHandler(valueBuffer, valueSize); + if (valueBuffer) { + resultHandler(valueBuffer, valueSize); + } delete[] valueBuffer; + return valueBuffer != nullptr; } qint64 Storage::diskUsage() const diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index 6c25448..18a5aa6 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -180,9 +180,9 @@ bool Storage::write(const std::string &sKey, const std::string &sValue) return !rc; } -void Storage::read(const std::string &sKey, const std::function &resultHandler) +bool Storage::read(const std::string &sKey, const std::function &resultHandler) { - read(sKey, + return read(sKey, [&](void *ptr, int size) { const std::string resultValue(static_cast(ptr), size); resultHandler(resultValue); @@ -190,10 +190,10 @@ void Storage::read(const std::string &sKey, const std::function &resultHandler) +bool Storage::read(const std::string &sKey, const std::function &resultHandler) { if (!d->env) { - return; + return false; } int rc; @@ -208,14 +208,14 @@ void Storage::read(const std::string &sKey, const std::functiontransaction, d->dbi, &cursor); if (rc) { std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl; - return; + return false; } if (sKey.empty()) { @@ -237,18 +237,20 @@ void Storage::read(const std::string &sKey, const std::function #include -#include "store/database.h" +#include "common/storage.h" class StorageTest : public QObject { @@ -19,31 +19,32 @@ private: void populate(int count) { - Database db(testDataPath, dbName); + Storage storage(testDataPath, dbName); for (int i = 0; i < count; i++) { //This should perhaps become an implementation detail of the db? if (i % 10000 == 0) { if (i > 0) { - db.commitTransaction(); + storage.commitTransaction(); } - db.startTransaction(); + storage.startTransaction(); } - db.write(keyPrefix + std::to_string(i), keyPrefix + std::to_string(i)); + storage.write(keyPrefix + std::to_string(i), keyPrefix + std::to_string(i)); } - db.commitTransaction(); + storage.commitTransaction(); } - bool verify(Database &db, int i) + bool verify(Storage &storage, int i) { - bool error = false; + bool success = true; + bool keyMatch = true; const auto reference = keyPrefix + std::to_string(i); - db.read(keyPrefix + std::to_string(i), [&error, &reference](const std::string &value) { + success = storage.read(keyPrefix + std::to_string(i), [&error, &reference](const std::string &value) { if (value != reference) { qDebug() << "Mismatch while reading"; - error = true; + keyMatch = false; } }); - return !error; + return succes && keyMatch; } private Q_SLOTS: -- cgit v1.2.3