From 767312e2063f4e58af3de0f27aba52de49e14295 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 5 Dec 2014 09:17:46 +0100 Subject: major reorg that puts Storage (previously Database) into common there is now a top-level tests dir, and a compile time switch for lmdb vs kyotocabinet --- common/CMakeLists.txt | 12 ++- common/storage.h | 28 +++++ common/storage_kyoto.cpp | 164 +++++++++++++++++++++++++++++ common/storage_lmdb.cpp | 266 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 469 insertions(+), 1 deletion(-) create mode 100644 common/storage.h create mode 100644 common/storage_kyoto.cpp create mode 100644 common/storage_lmdb.cpp (limited to 'common') diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index d409828..9b3f777 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -2,10 +2,20 @@ project(akonadinextcommon) generate_flatbuffers(commands/handshake commands/revisionupdate) +if (STORAGE_KYOTO) + set(storage_SRCS storage_kyoto.cpp) + set(storage_LIBS kyotocabinet) +else (STORAGE_KYOTO) + set(storage_SRCS storage_lmdb.cpp) + set(storage_LIBS lmdb) +endif (STORAGE_KYOTO) + set(command_SRCS commands.cpp console.cpp - ${CMAKE_CURRENT_BINARY_DIR}/commands/handshake_generated.h) + ${storage_SRCS}) add_library(${PROJECT_NAME} ${command_SRCS}) +SET_TARGET_PROPERTIES(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) qt5_use_modules(${PROJECT_NAME} Widgets) +target_link_libraries(${PROJECT_NAME} ${storage_LIBS}) diff --git a/common/storage.h b/common/storage.h new file mode 100644 index 0000000..f7dbd89 --- /dev/null +++ b/common/storage.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +class Storage { +public: + enum TransactionType { ReadOnly, ReadWrite }; + + Storage(const QString &storageRoot, const QString &name); + ~Storage(); + bool isInTransaction() const; + bool startTransaction(TransactionType type = ReadWrite); + bool commitTransaction(); + void abortTransaction(); + bool write(const char *key, size_t keySize, const char *value, size_t valueSize); + bool write(const std::string &sKey, const std::string &sValue); + //Perhaps prefer iterators (assuming we need to be able to match multiple values + void read(const std::string &sKey, const std::function &); + void read(const std::string &sKey, const std::function &); + + qint64 diskUsage() const; + void removeFromDisk() const; +private: + class Private; + Private * const d; +}; + diff --git a/common/storage_kyoto.cpp b/common/storage_kyoto.cpp new file mode 100644 index 0000000..05942c2 --- /dev/null +++ b/common/storage_kyoto.cpp @@ -0,0 +1,164 @@ +#include "storage.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +class Storage::Private +{ +public: + Private(const QString &storageRoot, const QString &name); + ~Private(); + + kyotocabinet::TreeDB db; + bool dbOpen; + bool inTransaction; +}; + +Storage::Private::Private(const QString &storageRoot, const QString &name) + : inTransaction(false) +{ + QDir dir; + dir.mkdir(storageRoot); + + //create file + dbOpen = db.open((storageRoot + "/" + name + ".kch").toStdString(), kyotocabinet::BasicDB::OWRITER | kyotocabinet::BasicDB::OCREATE); + if (!dbOpen) { + // TODO: handle error + } +} + +Storage::Private::~Private() +{ + if (dbOpen && inTransaction) { + db.end_transaction(false); + } +} + +Storage::Storage(const QString &storageRoot, const QString &name) + : d(new Private(storageRoot, name)) +{ +} + +Storage::~Storage() +{ + delete d; +} + +bool Storage::isInTransaction() const +{ + return d->inTransaction; +} + +bool Storage::startTransaction(TransactionType type) +{ + if (!d->dbOpen) { + return false; + } + + if (d->inTransaction) { + return true; + } + + //TODO handle errors + d->inTransaction = d->db.begin_transaction(); + return d->inTransaction; +} + +bool Storage::commitTransaction() +{ + if (!d->dbOpen) { + return false; + } + + if (!d->inTransaction) { + return false; + } + + bool success = d->db.end_transaction(true); + d->inTransaction = false; + return success; +} + +void Storage::abortTransaction() +{ + if (!d->dbOpen || !d->inTransaction) { + return; + } + + d->db.end_transaction(false); + d->inTransaction = false; +} + +bool Storage::write(const char *key, size_t keySize, const char *value, size_t valueSize) +{ + if (!d->dbOpen) { + return false; + } + + bool success = d->db.set(key, keySize, value, valueSize); + return success; +} + +bool Storage::write(const std::string &sKey, const std::string &sValue) +{ + if (!d->dbOpen) { + return false; + } + + bool success = d->db.set(sKey, sValue); + return success; +} + +void Storage::read(const std::string &sKey, const std::function &resultHandler) +{ + if (!d->dbOpen) { + return; + } + + std::string value; + if (d->db.get(sKey, &value)) { + resultHandler(value); + } +} + +void Storage::read(const std::string &sKey, const std::function &resultHandler) +{ + if (!d->dbOpen) { + return; + } + + size_t valueSize; + char *valueBuffer = d->db.get(sKey.data(), sKey.size(), &valueSize); + resultHandler(valueBuffer, valueSize); + delete[] valueBuffer; +} + +qint64 Storage::diskUsage() const +{ + if (!d->dbOpen) { + return 0; + } + + QFileInfo info(QString::fromStdString(d->db.path())); + return info.size(); +} + +void Storage::removeFromDisk() const +{ + if (!d->dbOpen) { + return; + } + + QFileInfo info(QString::fromStdString(d->db.path())); + QDir dir = info.dir(); + dir.remove(info.fileName()); +} diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp new file mode 100644 index 0000000..6c25448 --- /dev/null +++ b/common/storage_lmdb.cpp @@ -0,0 +1,266 @@ +#include "storage.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include + +class Storage::Private +{ +public: + Private(const QString &p); + ~Private(); + + QString path; + MDB_dbi dbi; + MDB_env *env; + MDB_txn *transaction; + bool readTransaction; + bool firstOpen; +}; + +Storage::Private::Private(const QString &p) + : path(p), + transaction(0), + readTransaction(false), + firstOpen(true) +{ + QDir dir; + dir.mkdir(path); + + //create file + if (mdb_env_create(&env)) { + // TODO: handle error + } else { + int rc = mdb_env_open(env, path.toStdString().data(), 0, 0664); + + if (rc) { + std::cerr << "mdb_env_open: " << rc << " " << mdb_strerror(rc) << std::endl; + mdb_env_close(env); + env = 0; + } else { + const size_t dbSize = 10485760 * 100; //10MB * 100 + mdb_env_set_mapsize(env, dbSize); + } + } +} + +Storage::Private::~Private() +{ + if (transaction) { + mdb_txn_abort(transaction); + } + + // it is still there and still unused, so we can shut it down + mdb_dbi_close(env, dbi); + mdb_env_close(env); +} + +Storage::Storage(const QString &storageRoot, const QString &name) + : d(new Private(storageRoot + '/' + name)) +{ +} + +Storage::~Storage() +{ + delete d; +} + +bool Storage::isInTransaction() const +{ + return d->transaction; +} + +bool Storage::startTransaction(TransactionType type) +{ + if (!d->env) { + return false; + } + + bool requestedRead = type == ReadOnly; + if (d->transaction && (!d->readTransaction || requestedRead)) { + return true; + } + + if (d->transaction) { + // we are about to turn a read transaction into a writable one + abortTransaction(); + } + + if (d->firstOpen && requestedRead) { + //A write transaction is at least required the first time + mdb_txn_begin(d->env, nullptr, 0, &d->transaction); + //Open the database + //With this we could open multiple named databases if we wanted to + mdb_dbi_open(d->transaction, nullptr, 0, &d->dbi); + mdb_txn_abort(d->transaction); + } + + int rc; + rc = mdb_txn_begin(d->env, NULL, requestedRead ? MDB_RDONLY : 0, &d->transaction); + if (!rc) { + rc = mdb_dbi_open(d->transaction, NULL, 0, &d->dbi); + } + + d->firstOpen = false; + return !rc; +} + +bool Storage::commitTransaction() +{ + if (!d->env) { + return false; + } + + if (!d->transaction) { + return false; + } + + int rc; + rc = mdb_txn_commit(d->transaction); + d->transaction = 0; + + if (rc) { + std::cerr << "mdb_txn_commit: " << rc << " " << mdb_strerror(rc) << std::endl; + } + + return !rc; +} + +void Storage::abortTransaction() +{ + if (!d->env || !d->transaction) { + return; + } + + mdb_txn_abort(d->transaction); + d->transaction = 0; +} + +bool Storage::write(const std::string &sKey, const std::string &sValue) +{ + if (!d->env) { + return false; + } + + const bool implicitTransaction = !d->transaction || d->readTransaction; + if (implicitTransaction) { + // TODO: if this fails, still try the write below? + if (!startTransaction()) { + return false; + } + } + + int rc; + MDB_val key, data; + key.mv_size = sKey.size(); + key.mv_data = (void*)sKey.data(); + data.mv_size = sValue.size(); + data.mv_data = (void*)sValue.data(); + rc = mdb_put(d->transaction, d->dbi, &key, &data, 0); + + if (rc) { + std::cerr << "mdb_put: " << rc << " " << mdb_strerror(rc) << std::endl; + } + + if (implicitTransaction) { + if (rc) { + abortTransaction(); + } else { + rc = commitTransaction(); + } + } + + return !rc; +} + +void Storage::read(const std::string &sKey, const std::function &resultHandler) +{ + read(sKey, + [&](void *ptr, int size) { + const std::string resultValue(static_cast(ptr), size); + resultHandler(resultValue); + }); +// std::cout << "key: " << resultKey << " data: " << resultValue << std::endl; +} + +void Storage::read(const std::string &sKey, const std::function &resultHandler) +{ + if (!d->env) { + return; + } + + int rc; + MDB_val key; + MDB_val data; + MDB_cursor *cursor; + + key.mv_size = sKey.size(); + key.mv_data = (void*)sKey.data(); + + const bool implicitTransaction = !d->transaction; + if (implicitTransaction) { + // TODO: if this fails, still try the write below? + if (!startTransaction(ReadOnly)) { + return; + } + } + + rc = mdb_cursor_open(d->transaction, d->dbi, &cursor); + if (rc) { + std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl; + return; + } + + if (sKey.empty()) { + std::cout << "Iterating over all values of store!" << std::endl; + rc = mdb_cursor_get(cursor, &key, &data, MDB_FIRST); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + resultHandler(key.mv_data, data.mv_size); + } + + //We never find the last value + if (rc == MDB_NOTFOUND) { + rc = 0; + } + } else { + if ((rc = mdb_cursor_get(cursor, &key, &data, MDB_SET)) == 0) { + resultHandler(data.mv_data, data.mv_size); + } else { + std::cout << "couldn't find value " << sKey << " " << std::endl; + } + } + + if (rc) { + std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl; + } + + mdb_cursor_close(cursor); + + /** + we don't abort the transaction since we need it for reading the values + if (implicitTransaction) { + abortTransaction(); + } + */ +} + +qint64 Storage::diskUsage() const +{ + QFileInfo info(d->path, "data.mdb"); + return info.size(); +} + +void Storage::removeFromDisk() const +{ + QDir dir(d->path); + dir.remove("data.mdb"); + dir.remove("lock.mdb"); +} + -- cgit v1.2.3 From 351a66b5fb1c8659bff8ea20d60f5a6d2d3263ad Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 5 Dec 2014 09:46:53 +0100 Subject: make read return a bool on success not happy with this API, but we need to discuss the whole read thing anyways --- common/storage.h | 4 ++-- common/storage_kyoto.cpp | 16 +++++++++++----- common/storage_lmdb.cpp | 18 ++++++++++-------- 3 files changed, 23 insertions(+), 15 deletions(-) (limited to 'common') diff --git a/common/storage.h b/common/storage.h index f7dbd89..0b548fb 100644 --- a/common/storage.h +++ b/common/storage.h @@ -16,8 +16,8 @@ public: bool write(const char *key, size_t keySize, const char *value, size_t valueSize); bool write(const std::string &sKey, const std::string &sValue); //Perhaps prefer iterators (assuming we need to be able to match multiple values - void read(const std::string &sKey, const std::function &); - void read(const std::string &sKey, const std::function &); + bool read(const std::string &sKey, const std::function &); + bool read(const std::string &sKey, const std::function &); qint64 diskUsage() const; void removeFromDisk() const; diff --git a/common/storage_kyoto.cpp b/common/storage_kyoto.cpp index 05942c2..40bd3e6 100644 --- a/common/storage_kyoto.cpp +++ b/common/storage_kyoto.cpp @@ -118,28 +118,34 @@ bool Storage::write(const std::string &sKey, const std::string &sValue) return success; } -void Storage::read(const std::string &sKey, const std::function &resultHandler) +bool Storage::read(const std::string &sKey, const std::function &resultHandler) { if (!d->dbOpen) { - return; + return false; } std::string value; if (d->db.get(sKey, &value)) { resultHandler(value); + return true; } + + return false; } -void Storage::read(const std::string &sKey, const std::function &resultHandler) +bool Storage::read(const std::string &sKey, const std::function &resultHandler) { if (!d->dbOpen) { - return; + return false; } size_t valueSize; char *valueBuffer = d->db.get(sKey.data(), sKey.size(), &valueSize); - resultHandler(valueBuffer, valueSize); + if (valueBuffer) { + resultHandler(valueBuffer, valueSize); + } delete[] valueBuffer; + return valueBuffer != nullptr; } qint64 Storage::diskUsage() const diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index 6c25448..18a5aa6 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -180,9 +180,9 @@ bool Storage::write(const std::string &sKey, const std::string &sValue) return !rc; } -void Storage::read(const std::string &sKey, const std::function &resultHandler) +bool Storage::read(const std::string &sKey, const std::function &resultHandler) { - read(sKey, + return read(sKey, [&](void *ptr, int size) { const std::string resultValue(static_cast(ptr), size); resultHandler(resultValue); @@ -190,10 +190,10 @@ void Storage::read(const std::string &sKey, const std::function &resultHandler) +bool Storage::read(const std::string &sKey, const std::function &resultHandler) { if (!d->env) { - return; + return false; } int rc; @@ -208,14 +208,14 @@ void Storage::read(const std::string &sKey, const std::functiontransaction, d->dbi, &cursor); if (rc) { std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl; - return; + return false; } if (sKey.empty()) { @@ -237,18 +237,20 @@ void Storage::read(const std::string &sKey, const std::function