From 9ee8378d393778ac67314be7ea8d5bcbaeee9ee0 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Sun, 7 Dec 2014 10:08:07 +0100 Subject: try out unqlite --- common/storage_unqlite.cpp | 221 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 common/storage_unqlite.cpp (limited to 'common/storage_unqlite.cpp') diff --git a/common/storage_unqlite.cpp b/common/storage_unqlite.cpp new file mode 100644 index 0000000..d0eaa38 --- /dev/null +++ b/common/storage_unqlite.cpp @@ -0,0 +1,221 @@ +#include "storage.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +class Storage::Private +{ +public: + Private(const QString &storageRoot, const QString &name, AccessMode m); + ~Private(); + + QString name; + kyotocabinet::TreeDB db; + AccessMode mode; + bool dbOpen; + bool inTransaction; +}; + +Storage::Private::Private(const QString &storageRoot, const QString &n, AccessMode m) + : name(n), + mode(m), + dbOpen(false), + inTransaction(false) +{ + QDir dir; + dir.mkdir(storageRoot); + + //create file + uint32_t openMode = kyotocabinet::BasicDB::OCREATE | + (mode == ReadOnly ? kyotocabinet::BasicDB::OREADER + : kyotocabinet::BasicDB::OWRITER); + dbOpen = db.open((storageRoot + "/" + name + ".kch").toStdString(), openMode); + if (!dbOpen) { + std::cerr << "Could not open database: " << db.error().codename(db.error().code()) << " " << db.error().message() << std::endl; + // TODO: handle error + } +} + +Storage::Private::~Private() +{ + if (dbOpen && inTransaction) { + db.end_transaction(false); + } +} + +Storage::Storage(const QString &storageRoot, const QString &name, AccessMode mode) + : d(new Private(storageRoot, name, mode)) +{ +} + +Storage::~Storage() +{ + delete d; +} + +bool Storage::isInTransaction() const +{ + return d->inTransaction; +} + +bool Storage::startTransaction(AccessMode type) +{ + if (!d->dbOpen) { + return false; + } + + if (type == ReadWrite && d->mode != ReadWrite) { + return false; + } + + if (d->inTransaction) { + return true; + } + + //TODO handle errors + d->inTransaction = d->db.begin_transaction(); + return d->inTransaction; +} + +bool Storage::commitTransaction() +{ + if (!d->dbOpen) { + return false; + } + + if (!d->inTransaction) { + return false; + } + + bool success = d->db.end_transaction(true); + d->inTransaction = false; + return success; +} + +void Storage::abortTransaction() +{ + if (!d->dbOpen || !d->inTransaction) { + return; + } + + d->db.end_transaction(false); + d->inTransaction = false; +} + +bool Storage::write(const char *key, size_t keySize, const char *value, size_t valueSize) +{ + if (!d->dbOpen) { + return false; + } + + bool success = d->db.set(key, keySize, value, valueSize); + return success; +} + +bool Storage::write(const std::string &sKey, const std::string &sValue) +{ + if (!d->dbOpen) { + return false; + } + + bool success = d->db.set(sKey, sValue); + return success; +} + +void Storage::read(const std::string &sKey, + const std::function &resultHandler, + const std::function &errorHandler) +{ + if (!d->dbOpen) { + Error error(d->name.toStdString(), -1, "Not open"); + errorHandler(error); + return; + } + + std::string value; + if (sKey.empty()) { + kyotocabinet::DB::Cursor *cursor = d->db.cursor(); + cursor->jump(); + + std::string key, value; + while (cursor->get_value(&value, true) && resultHandler(value)) {} + + delete cursor; + return; + } else { + if (d->db.get(sKey, &value)) { + resultHandler(value); + return; + } + } + + Error error(d->name.toStdString(), d->db.error().code(), d->db.error().message()); + errorHandler(error); +} + +void Storage::read(const std::string &sKey, + const std::function &resultHandler, + const std::function &errorHandler) +{ + if (!d->dbOpen) { + Error error(d->name.toStdString(), -1, "Not open"); + errorHandler(error); + return; + } + + size_t valueSize; + char *valueBuffer; + if (sKey.empty()) { + kyotocabinet::DB::Cursor *cursor = d->db.cursor(); + cursor->jump(); + + while ((valueBuffer = cursor->get_value(&valueSize, true))) { + bool ok = resultHandler(valueBuffer, valueSize); + delete[] valueBuffer; + if (!ok) { + break; + } + } + + delete cursor; + } else { + valueBuffer = d->db.get(sKey.data(), sKey.size(), &valueSize); + if (valueBuffer) { + resultHandler(valueBuffer, valueSize); + } else { + Error error(d->name.toStdString(), d->db.error().code(), d->db.error().message()); + errorHandler(error); + } + delete[] valueBuffer; + } +} + +qint64 Storage::diskUsage() const +{ + if (!d->dbOpen) { + return 0; + } + + QFileInfo info(QString::fromStdString(d->db.path())); + return info.size(); +} + +void Storage::removeFromDisk() const +{ + if (!d->dbOpen) { + return; + } + + QFileInfo info(QString::fromStdString(d->db.path())); + QDir dir = info.dir(); + dir.remove(info.fileName()); +} -- cgit v1.2.3 From a6ed70495f9f3ecb21c26860dda16aadcdc91c3a Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Sun, 14 Dec 2014 11:59:39 +0100 Subject: and now unqlite storage works --- common/storage_unqlite.cpp | 252 ++++++++++++++++++++++++++++----------------- 1 file changed, 159 insertions(+), 93 deletions(-) (limited to 'common/storage_unqlite.cpp') diff --git a/common/storage_unqlite.cpp b/common/storage_unqlite.cpp index d0eaa38..5c5ef09 100644 --- a/common/storage_unqlite.cpp +++ b/common/storage_unqlite.cpp @@ -5,51 +5,93 @@ #include #include #include -#include #include #include #include -#include +#include "unqlite/unqlite.h" class Storage::Private { public: - Private(const QString &storageRoot, const QString &name, AccessMode m); + Private(const QString &s, const QString &name, AccessMode m); ~Private(); + void reportDbError(const char *functionName); + void reportDbError(const char *functionName, int errorCode, + const std::function &errorHandler); + + QString storageRoot; QString name; - kyotocabinet::TreeDB db; AccessMode mode; - bool dbOpen; + + unqlite *db; bool inTransaction; }; -Storage::Private::Private(const QString &storageRoot, const QString &n, AccessMode m) - : name(n), +Storage::Private::Private(const QString &s, const QString &n, AccessMode m) + : storageRoot(s), + name(n), mode(m), - dbOpen(false), + db(0), inTransaction(false) { + const QString fullPath(storageRoot + '/' + name); QDir dir; dir.mkdir(storageRoot); //create file - uint32_t openMode = kyotocabinet::BasicDB::OCREATE | - (mode == ReadOnly ? kyotocabinet::BasicDB::OREADER - : kyotocabinet::BasicDB::OWRITER); - dbOpen = db.open((storageRoot + "/" + name + ".kch").toStdString(), openMode); - if (!dbOpen) { - std::cerr << "Could not open database: " << db.error().codename(db.error().code()) << " " << db.error().message() << std::endl; - // TODO: handle error + int openFlags = UNQLITE_OPEN_CREATE; + if (mode == ReadOnly) { + openFlags |= UNQLITE_OPEN_READONLY | UNQLITE_OPEN_MMAP; + } else { + openFlags |= UNQLITE_OPEN_READWRITE; + } + + int rc = unqlite_open(&db, fullPath.toStdString().data(), openFlags); + + if (rc != UNQLITE_OK) { + reportDbError("unqlite_open"); } } Storage::Private::~Private() { - if (dbOpen && inTransaction) { - db.end_transaction(false); + unqlite_close(db); +} + +void Storage::Private::reportDbError(const char *functionName) +{ + std::cerr << "ERROR: " << functionName; + if (db) { + const char *errorMessage; + int length; + /* Something goes wrong, extract database error log */ + unqlite_config(db, UNQLITE_CONFIG_ERR_LOG, &errorMessage, &length); + if (length > 0) { + std::cerr << ": " << errorMessage; + } + } + std::cerr << std::endl; +} + +void Storage::Private::reportDbError(const char *functionName, int errorCode, + const std::function &errorHandler) +{ + if (db) { + const char *errorMessage; + int length; + /* Something goes wrong, extract database error log */ + unqlite_config(db, UNQLITE_CONFIG_ERR_LOG, &errorMessage, &length); + if (length > 0) { + Error error(name.toStdString(), errorCode, errorMessage); + errorHandler(error); + return; + } } + + Error error(name.toStdString(), errorCode, functionName); + errorHandler(error); } Storage::Storage(const QString &storageRoot, const QString &name, AccessMode mode) @@ -59,6 +101,10 @@ Storage::Storage(const QString &storageRoot, const QString &name, AccessMode mod Storage::~Storage() { + if (d->inTransaction) { + abortTransaction(); + } + delete d; } @@ -69,11 +115,7 @@ bool Storage::isInTransaction() const bool Storage::startTransaction(AccessMode type) { - if (!d->dbOpen) { - return false; - } - - if (type == ReadWrite && d->mode != ReadWrite) { + if (!d->db) { return false; } @@ -81,141 +123,165 @@ bool Storage::startTransaction(AccessMode type) return true; } - //TODO handle errors - d->inTransaction = d->db.begin_transaction(); + d->inTransaction = unqlite_begin(d->db) == UNQLITE_OK; + + if (!d->inTransaction) { + d->reportDbError("unqlite_begin"); + } + return d->inTransaction; } bool Storage::commitTransaction() { - if (!d->dbOpen) { + if (!d->db) { return false; } if (!d->inTransaction) { - return false; + return true; } - bool success = d->db.end_transaction(true); + int rc = unqlite_commit(d->db); d->inTransaction = false; - return success; + + if (rc != UNQLITE_OK) { + d->reportDbError("unqlite_commit"); + } + + return rc == UNQLITE_OK; } void Storage::abortTransaction() { - if (!d->dbOpen || !d->inTransaction) { + if (!d->db || !d->inTransaction) { return; } - d->db.end_transaction(false); + unqlite_rollback(d->db); d->inTransaction = false; } bool Storage::write(const char *key, size_t keySize, const char *value, size_t valueSize) { - if (!d->dbOpen) { - return false; - } - - bool success = d->db.set(key, keySize, value, valueSize); - return success; + return write(std::string(key, keySize), std::string(value, valueSize)); } bool Storage::write(const std::string &sKey, const std::string &sValue) { - if (!d->dbOpen) { + if (!d->db) { return false; } - bool success = d->db.set(sKey, sValue); - return success; + int rc = unqlite_kv_store(d->db, sKey.data(), -1, sValue.data(), sValue.size()); + + if (rc != UNQLITE_OK) { + d->reportDbError("unqlite_kv_store"); + } + + return !rc; } void Storage::read(const std::string &sKey, const std::function &resultHandler, const std::function &errorHandler) { - if (!d->dbOpen) { - Error error(d->name.toStdString(), -1, "Not open"); - errorHandler(error); - return; - } + read(sKey, + [&](void *ptr, int size) -> bool { + if (ptr) { + const std::string resultValue(static_cast(ptr), size); + return resultHandler(resultValue); + } - std::string value; - if (sKey.empty()) { - kyotocabinet::DB::Cursor *cursor = d->db.cursor(); - cursor->jump(); + return true; + }, errorHandler); +} - std::string key, value; - while (cursor->get_value(&value, true) && resultHandler(value)) {} +void Storage::read(const std::string &sKey, + const std::function &resultHandler, + const std::function &errorHandler) +{ + scan(sKey.data(), sKey.size(), [resultHandler](void *keyPtr, int keySize, void *valuePtr, int valueSize) { + return resultHandler(valuePtr, valueSize); + }, errorHandler); +} - delete cursor; - return; - } else { - if (d->db.get(sKey, &value)) { - resultHandler(value); - return; +void fetchCursorData(unqlite_kv_cursor *cursor, + void **keyBuffer, int *keyBufferLength, void **dataBuffer, unqlite_int64 *dataBufferLength, + const std::function &resultHandler) +{ + int keyLength = 0; + unqlite_int64 dataLength = 0; + // now fetch the data sizes + if (unqlite_kv_cursor_key(cursor, nullptr, &keyLength) == UNQLITE_OK && + unqlite_kv_cursor_data(cursor, nullptr, &dataLength) == UNQLITE_OK) { + if (keyLength > *keyBufferLength) { + *keyBuffer = realloc(*keyBuffer, keyLength); + *keyBufferLength = keyLength; } - } - Error error(d->name.toStdString(), d->db.error().code(), d->db.error().message()); - errorHandler(error); + if (dataLength > *dataBufferLength) { + *dataBuffer = realloc(*dataBuffer, dataLength); + *dataBufferLength = dataLength; + } + + if (unqlite_kv_cursor_key(cursor, *keyBuffer, &keyLength) == UNQLITE_OK && + unqlite_kv_cursor_data(cursor, *dataBuffer, &dataLength) == UNQLITE_OK) { + resultHandler(*keyBuffer, keyLength, *dataBuffer, dataLength); + } + } } -void Storage::read(const std::string &sKey, - const std::function &resultHandler, +void Storage::scan(const char *keyData, uint keySize, + const std::function &resultHandler, const std::function &errorHandler) { - if (!d->dbOpen) { + if (!d->db) { Error error(d->name.toStdString(), -1, "Not open"); errorHandler(error); return; } - size_t valueSize; - char *valueBuffer; - if (sKey.empty()) { - kyotocabinet::DB::Cursor *cursor = d->db.cursor(); - cursor->jump(); + unqlite_kv_cursor *cursor; - while ((valueBuffer = cursor->get_value(&valueSize, true))) { - bool ok = resultHandler(valueBuffer, valueSize); - delete[] valueBuffer; - if (!ok) { - break; - } - } + int rc = unqlite_kv_cursor_init(d->db, &cursor); + if (rc != UNQLITE_OK) { + d->reportDbError("unqlite_kv_cursor_init", rc, errorHandler); + return; + } - delete cursor; + void *keyBuffer = nullptr; + int keyBufferLength = 0; + void *dataBuffer = nullptr; + //FIXME: 64bit ints, but feeding int lenghts to the callbacks. can result in truncation + unqlite_int64 dataBufferLength = 0; + if (!keyData || keySize == 0) { + for (unqlite_kv_cursor_first_entry(cursor); unqlite_kv_cursor_valid_entry(cursor); unqlite_kv_cursor_next_entry(cursor)) { + fetchCursorData(cursor, &keyBuffer, &keyBufferLength, &dataBuffer, &dataBufferLength, resultHandler); + } } else { - valueBuffer = d->db.get(sKey.data(), sKey.size(), &valueSize); - if (valueBuffer) { - resultHandler(valueBuffer, valueSize); + rc = unqlite_kv_cursor_seek(cursor, keyData, keySize, UNQLITE_CURSOR_MATCH_EXACT); + if (rc == UNQLITE_OK) { + fetchCursorData(cursor, &keyBuffer, &keyBufferLength, &dataBuffer, &dataBufferLength, resultHandler); } else { - Error error(d->name.toStdString(), d->db.error().code(), d->db.error().message()); - errorHandler(error); + std::cout << "couldn't find value " << std::string(keyData, keySize) << std::endl; } - delete[] valueBuffer; + } + + free(keyBuffer); + free(dataBuffer); + unqlite_kv_cursor_release(d->db, cursor); } qint64 Storage::diskUsage() const { - if (!d->dbOpen) { - return 0; - } - - QFileInfo info(QString::fromStdString(d->db.path())); + QFileInfo info(d->storageRoot + '/' + d->name); return info.size(); } void Storage::removeFromDisk() const { - if (!d->dbOpen) { - return; - } - - QFileInfo info(QString::fromStdString(d->db.path())); - QDir dir = info.dir(); - dir.remove(info.fileName()); + QFile::remove(d->storageRoot + '/' + d->name); } + -- cgit v1.2.3