major reorg that puts Storage (previously Database) into common

there is now a top-level tests dir, and a compile time switch for
lmdb vs kyotocabinet

11 files changed, 325 insertions, 55 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 392ca39..e8b6f23 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -47,10 +47,10 @@ add_subdirectory(client)
 # the resource
 add_subdirectory(resource)
 
-# the store
-add_subdirectory(store)
-
 # a simple dummy resource implementation
 add_subdirectory(dummyresource)
 
+# some tests
+add_subdirectory(tests)
+
 feature_summary(WHAT ALL FATAL_ON_MISSING_REQUIRED_PACKAGES)
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
index d409828..9b3f777 100644
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -2,10 +2,20 @@ project(akonadinextcommon)
 generate_flatbuffers(commands/handshake
                      commands/revisionupdate)
 
+if (STORAGE_KYOTO)
+    set(storage_SRCS storage_kyoto.cpp)
+    set(storage_LIBS kyotocabinet)
+else (STORAGE_KYOTO)
+    set(storage_SRCS storage_lmdb.cpp)
+    set(storage_LIBS lmdb)
+endif (STORAGE_KYOTO)
+
 set(command_SRCS
    commands.cpp
    console.cpp
-   ${CMAKE_CURRENT_BINARY_DIR}/commands/handshake_generated.h)
+   ${storage_SRCS})
 
 add_library(${PROJECT_NAME} ${command_SRCS})
+SET_TARGET_PROPERTIES(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
 qt5_use_modules(${PROJECT_NAME} Widgets)
+target_link_libraries(${PROJECT_NAME} ${storage_LIBS})
diff --git a/store/database.h b/common/storage.h
index e752ff5..f7dbd89 100644
--- a/store/database.h
+++ b/common/storage.h
@@ -3,12 +3,12 @@
 #include <string>
 #include <QString>
 
-class Database {
+class Storage {
 public:
     enum TransactionType { ReadOnly, ReadWrite };
 
-    Database(const QString &storageRoot, const QString &name);
-    ~Database();
+    Storage(const QString &storageRoot, const QString &name);
+    ~Storage();
     bool isInTransaction() const;
     bool startTransaction(TransactionType type = ReadWrite);
     bool commitTransaction();
diff --git a/store/database.cpp b/common/storage_kyoto.cpp
index 542667a..05942c2 100644
--- a/store/database.cpp
+++ b/common/storage_kyoto.cpp
@@ -1,4 +1,4 @@
-#include "database.h"
+#include "storage.h"
 
 #include <iostream>
 
@@ -12,7 +12,7 @@
 
 #include <kchashdb.h>
 
-class Database::Private
+class Storage::Private
 {
 public:
     Private(const QString &storageRoot, const QString &name);
@@ -23,7 +23,7 @@ public:
     bool inTransaction;
 };
 
-Database::Private::Private(const QString &storageRoot, const QString &name)
+Storage::Private::Private(const QString &storageRoot, const QString &name)
     : inTransaction(false)
 {
     QDir dir;
@@ -36,29 +36,29 @@ Database::Private::Private(const QString &storageRoot, const QString &name)
     }
 }
 
-Database::Private::~Private()
+Storage::Private::~Private()
 {
     if (dbOpen && inTransaction) {
         db.end_transaction(false);
     }
 }
 
-Database::Database(const QString &storageRoot, const QString &name)
+Storage::Storage(const QString &storageRoot, const QString &name)
     : d(new Private(storageRoot, name))
 {
 }
 
-Database::~Database()
+Storage::~Storage()
 {
     delete d;
 }
 
-bool Database::isInTransaction() const
+bool Storage::isInTransaction() const
 {
     return d->inTransaction;
 }
 
-bool Database::startTransaction(TransactionType type)
+bool Storage::startTransaction(TransactionType type)
 {
     if (!d->dbOpen) {
         return false;
@@ -73,7 +73,7 @@ bool Database::startTransaction(TransactionType type)
     return d->inTransaction;
 }
 
-bool Database::commitTransaction()
+bool Storage::commitTransaction()
 {
     if (!d->dbOpen) {
         return false;
@@ -88,7 +88,7 @@ bool Database::commitTransaction()
     return success;
 }
 
-void Database::abortTransaction()
+void Storage::abortTransaction()
 {
     if (!d->dbOpen || !d->inTransaction) {
         return;
@@ -98,7 +98,7 @@ void Database::abortTransaction()
     d->inTransaction = false;
 }
 
-bool Database::write(const char *key, size_t keySize, const char *value, size_t valueSize)
+bool Storage::write(const char *key, size_t keySize, const char *value, size_t valueSize)
 {
     if (!d->dbOpen) {
         return false;
@@ -108,7 +108,7 @@ bool Database::write(const char *key, size_t keySize, const char *value, size_t
     return success; 
 }
 
-bool Database::write(const std::string &sKey, const std::string &sValue)
+bool Storage::write(const std::string &sKey, const std::string &sValue)
 {
     if (!d->dbOpen) {
         return false;
@@ -118,7 +118,7 @@ bool Database::write(const std::string &sKey, const std::string &sValue)
     return success; 
 }
 
-void Database::read(const std::string &sKey, const std::function<void(const std::string &value)> &resultHandler)
+void Storage::read(const std::string &sKey, const std::function<void(const std::string &value)> &resultHandler)
 {
     if (!d->dbOpen) {
         return;
@@ -130,7 +130,7 @@ void Database::read(const std::string &sKey, const std::function<void(const std:
     }
 }
 
-void Database::read(const std::string &sKey, const std::function<void(void *ptr, int size)> &resultHandler)
+void Storage::read(const std::string &sKey, const std::function<void(void *ptr, int size)> &resultHandler)
 {
     if (!d->dbOpen) {
         return;
@@ -142,7 +142,7 @@ void Database::read(const std::string &sKey, const std::function<void(void *ptr,
     delete[] valueBuffer;
 }
 
-qint64 Database::diskUsage() const
+qint64 Storage::diskUsage() const
 {
     if (!d->dbOpen) {
         return 0;
@@ -152,7 +152,7 @@ qint64 Database::diskUsage() const
     return info.size();
 }
 
-void Database::removeFromDisk() const
+void Storage::removeFromDisk() const
 {
     if (!d->dbOpen) {
         return;
diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp
new file mode 100644
index 0000000..6c25448
--- /dev/null
+++ b/common/storage_lmdb.cpp
@@ -0,0 +1,266 @@
+#include "storage.h"
+
+#include <iostream>
+
+#include <QAtomicInt>
+#include <QDebug>
+#include <QDir>
+#include <QReadWriteLock>
+#include <QString>
+#include <QTime>
+
+#include <lmdb.h>
+
+class Storage::Private
+{
+public:
+    Private(const QString &p);
+    ~Private();
+
+    QString path;
+    MDB_dbi dbi;
+    MDB_env *env;
+    MDB_txn *transaction;
+    bool readTransaction;
+    bool firstOpen;
+};
+
+Storage::Private::Private(const QString &p)
+    : path(p),
+      transaction(0),
+      readTransaction(false),
+      firstOpen(true)
+{
+    QDir dir;
+    dir.mkdir(path);
+
+    //create file
+    if (mdb_env_create(&env)) {
+        // TODO: handle error
+    } else {
+        int rc = mdb_env_open(env, path.toStdString().data(), 0, 0664);
+
+        if (rc) {
+            std::cerr << "mdb_env_open: " << rc << " " << mdb_strerror(rc) << std::endl;
+            mdb_env_close(env);
+            env = 0;
+        } else {
+            const size_t dbSize = 10485760 * 100; //10MB * 100
+            mdb_env_set_mapsize(env, dbSize);
+        }
+    }
+}
+
+Storage::Private::~Private()
+{
+    if (transaction) {
+        mdb_txn_abort(transaction);
+    }
+
+    // it is still there and still unused, so we can shut it down
+    mdb_dbi_close(env, dbi);
+    mdb_env_close(env);
+}
+
+Storage::Storage(const QString &storageRoot, const QString &name)
+    : d(new Private(storageRoot + '/' + name))
+{
+}
+
+Storage::~Storage()
+{
+    delete d;
+}
+
+bool Storage::isInTransaction() const
+{
+    return d->transaction;
+}
+
+bool Storage::startTransaction(TransactionType type)
+{
+    if (!d->env) {
+        return false;
+    }
+
+    bool requestedRead = type == ReadOnly;
+    if (d->transaction && (!d->readTransaction || requestedRead)) {
+        return true;
+    }
+
+    if (d->transaction) {
+        // we are about to turn a read transaction into a writable one
+        abortTransaction();
+    }
+
+    if (d->firstOpen && requestedRead) {
+        //A write transaction is at least required the first time
+        mdb_txn_begin(d->env, nullptr, 0, &d->transaction);
+        //Open the database
+        //With this we could open multiple named databases if we wanted to
+        mdb_dbi_open(d->transaction, nullptr, 0, &d->dbi);
+        mdb_txn_abort(d->transaction);
+    }
+
+    int rc;
+    rc = mdb_txn_begin(d->env, NULL, requestedRead ? MDB_RDONLY : 0, &d->transaction);
+    if (!rc) {
+        rc = mdb_dbi_open(d->transaction, NULL, 0, &d->dbi);
+    }
+
+    d->firstOpen = false;
+    return !rc;
+}
+
+bool Storage::commitTransaction()
+{
+    if (!d->env) {
+        return false;
+    }
+
+    if (!d->transaction) {
+        return false;
+    }
+
+    int rc;
+    rc = mdb_txn_commit(d->transaction);
+    d->transaction = 0;
+
+    if (rc) {
+        std::cerr << "mdb_txn_commit: " << rc << " " << mdb_strerror(rc) << std::endl;
+    }
+
+    return !rc;
+}
+
+void Storage::abortTransaction()
+{
+    if (!d->env || !d->transaction) {
+        return;
+    }
+
+    mdb_txn_abort(d->transaction);
+    d->transaction = 0;
+}
+
+bool Storage::write(const std::string &sKey, const std::string &sValue)
+{
+    if (!d->env) {
+        return false;
+    }
+
+    const bool implicitTransaction = !d->transaction || d->readTransaction;
+    if (implicitTransaction) {
+        // TODO: if this fails, still try the write below?
+        if (!startTransaction()) {
+            return false;
+        }
+    }
+
+    int rc;
+    MDB_val key, data;
+    key.mv_size = sKey.size();
+    key.mv_data = (void*)sKey.data();
+    data.mv_size = sValue.size();
+    data.mv_data = (void*)sValue.data();
+    rc = mdb_put(d->transaction, d->dbi, &key, &data, 0);
+
+    if (rc) {
+        std::cerr << "mdb_put: " << rc << " " << mdb_strerror(rc) << std::endl;
+    }
+
+    if (implicitTransaction) {
+        if (rc) {
+            abortTransaction();
+        } else {
+            rc = commitTransaction();
+        }
+    }
+
+    return !rc;
+}
+
+void Storage::read(const std::string &sKey, const std::function<void(const std::string &value)> &resultHandler)
+{
+    read(sKey,
+         [&](void *ptr, int size) {
+            const std::string resultValue(static_cast<char*>(ptr), size);
+            resultHandler(resultValue);
+         });
+// std::cout << "key: " << resultKey << " data: " << resultValue << std::endl;
+}
+
+void Storage::read(const std::string &sKey, const std::function<void(void *ptr, int size)> &resultHandler)
+{
+    if (!d->env) {
+        return;
+    }
+
+    int rc;
+    MDB_val key;
+    MDB_val data;
+    MDB_cursor *cursor;
+
+    key.mv_size = sKey.size();
+    key.mv_data = (void*)sKey.data();
+
+    const bool implicitTransaction = !d->transaction;
+    if (implicitTransaction) {
+        // TODO: if this fails, still try the write below?
+        if (!startTransaction(ReadOnly)) {
+            return;
+        }
+    }
+
+    rc = mdb_cursor_open(d->transaction, d->dbi, &cursor);
+    if (rc) {
+        std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl;
+        return;
+    }
+
+    if (sKey.empty()) {
+        std::cout << "Iterating over all values of store!" << std::endl;
+        rc = mdb_cursor_get(cursor, &key, &data, MDB_FIRST);
+        while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
+            resultHandler(key.mv_data, data.mv_size);
+        }
+
+        //We never find the last value
+        if (rc == MDB_NOTFOUND) {
+            rc = 0;
+        }
+    } else {
+        if ((rc = mdb_cursor_get(cursor, &key, &data, MDB_SET)) == 0) {
+            resultHandler(data.mv_data, data.mv_size);
+        } else {
+            std::cout << "couldn't find value " << sKey << " " << std::endl;
+        }
+    }
+
+    if (rc) {
+        std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl;
+    }
+
+    mdb_cursor_close(cursor);
+
+    /**
+      we don't abort the transaction since we need it for reading the values
+    if (implicitTransaction) {
+        abortTransaction();
+    }
+    */
+}
+
+qint64 Storage::diskUsage() const
+{
+    QFileInfo info(d->path, "data.mdb");
+    return info.size();
+}
+
+void Storage::removeFromDisk() const
+{
+    QDir dir(d->path);
+    dir.remove("data.mdb");
+    dir.remove("lock.mdb");
+}
+
diff --git a/dummyresource/facade.cpp b/dummyresource/facade.cpp
index 7e9881b..9758c1b 100644
--- a/dummyresource/facade.cpp
+++ b/dummyresource/facade.cpp
@@ -11,7 +11,7 @@ using namespace flatbuffers;
 DummyResourceFacade::DummyResourceFacade()
     : Akonadi2::StoreFacade<Akonadi2::Domain::Event>(),
     mResourceAccess(new ResourceAccess("dummyresource")),
-    mDatabase(new Database(Akonadi2::Store::storageLocation(), "dummyresource"))
+    mStorage(new Storage(Akonadi2::Store::storageLocation(), "dummyresource"))
 {
     // connect(mResourceAccess.data(), &ResourceAccess::ready, this, onReadyChanged);
 }
@@ -69,20 +69,20 @@ public:
     DummyEvent const *buffer;
 
     //Keep query alive so values remain valid
-    QSharedPointer<Database> db;
+    QSharedPointer<Storage> storage;
 };
 
 void DummyResourceFacade::load(const Akonadi2::Query &query, const std::function<void(const Akonadi2::Domain::Event::Ptr &)> &resultCallback)
 {
     qDebug() << "load called";
     //TODO only read values matching the query
-    auto db = QSharedPointer<Database>::create(Akonadi2::Store::storageLocation(), "dummyresource");
-    db->read("", [resultCallback, db](void *data, int size) {
+    auto storage = QSharedPointer<Storage>::create(Akonadi2::Store::storageLocation(), "dummyresource");
+    storage->read("", [resultCallback, storage](void *data, int size) {
         //TODO read second buffer as well
         auto eventBuffer = GetDummyEvent(data);
         auto event = QSharedPointer<DummyEventAdaptor>::create("dummyresource", "key", 0);
         event->buffer = eventBuffer;
-        event->db = db;
+        event->storage = storage;
         resultCallback(event);
     });
 }
diff --git a/dummyresource/facade.h b/dummyresource/facade.h
index 7a516de..310ef77 100644
--- a/dummyresource/facade.h
+++ b/dummyresource/facade.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include "client/clientapi.h"
-#include "store/database.h"
+#include "common/storage.h"
 
 class ResourceAccess;
 
@@ -18,5 +18,5 @@ public:
 
 private:
     QSharedPointer<ResourceAccess> mResourceAccess;
-    QSharedPointer<Database> mDatabase;
+    QSharedPointer<Storage> mStorage;
 };
diff --git a/store/CMakeLists.txt b/store/CMakeLists.txt
deleted file mode 100644
index 552439e..0000000
--- a/store/CMakeLists.txt
+++ /dev/null
@@ -1 +0,0 @@
-add_subdirectory(test)
diff --git a/store/test/CMakeLists.txt b/tests/CMakeLists.txt
index 1b9dc9e..23776a1 100644
--- a/store/test/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,20 +1,15 @@
 set(CMAKE_AUTOMOC ON)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-set(store_path "../")
-set(store_SRCS
-    ${store_path}/database.cpp
-)
-
 generate_flatbuffers(calendar)
 
 macro(manual_tests)
     foreach(_testname ${ARGN})
-        add_executable(${_testname} ${_testname}.cpp ${store_SRCS})
+        add_executable(${_testname} ${_testname}.cpp)
         qt5_use_modules(${_testname} Core Test)
-        target_link_libraries(${_testname} kyotocabinet)
+        target_link_libraries(${_testname} akonadinextcommon)
     endforeach(_testname)
-endmacro(auto_tests)
+endmacro(manual_tests)
 
 manual_tests (
     storagebenchmark
diff --git a/store/test/calendar.fbs b/tests/calendar.fbs
index 203ee43..203ee43 100644
--- a/store/test/calendar.fbs
+++ b/tests/calendar.fbs
diff --git a/store/test/storagebenchmark.cpp b/tests/storagebenchmark.cpp
index d42dea8..15da9da 100644
--- a/store/test/storagebenchmark.cpp
+++ b/tests/storagebenchmark.cpp
@@ -10,7 +10,7 @@
 #include <QString>
 #include <QTime>
 
-#include "store/database.h"
+#include "common/storage.h"
 
 using namespace Calendar;
 using namespace flatbuffers;
@@ -72,9 +72,9 @@ private Q_SLOTS:
         QFETCH(bool, useDb);
         QFETCH(int, count);
 
-        Database *db = 0;
+        Storage *store = 0;
         if (useDb) {
-            db = new Database(testDataPath, dbName);
+            store = new Storage(testDataPath, dbName);
         }
 
         std::ofstream myfile;
@@ -87,22 +87,22 @@ private Q_SLOTS:
         {
             auto event = createEvent();
             for (int i = 0; i < count; i++) {
-                if (db) {
+                if (store) {
                     if (i % 10000 == 0) {
                         if (i > 0) {
-                            db->commitTransaction();
+                            store->commitTransaction();
                         }
-                        db->startTransaction();
+                        store->startTransaction();
                     }
 
-                    db->write(keyPrefix + std::to_string(i), event);
+                    store->write(keyPrefix + std::to_string(i), event);
                 } else {
                     myfile << event;
                 }
             }
 
-            if (db) {
-                db->commitTransaction();
+            if (store) {
+                store->commitTransaction();
             } else {
                 myfile.close();
             }
@@ -112,20 +112,20 @@ private Q_SLOTS:
 
         {
             for (int i = 0; i < count; i++) {
-                if (db) {
-                    db->read(keyPrefix + std::to_string(i), [](std::string value){});
+                if (store) {
+                    store->read(keyPrefix + std::to_string(i), [](std::string value){});
                 }
             }
         }
         const int readDuration = time.restart();
 
-        if (db) {
+        if (store) {
             qDebug() << "Reading took[ms]: " << readDuration;
         } else {
             qDebug() << "File reading is not implemented.";
         }
 
-        delete db;
+        delete store;
     }
 
     void testBufferCreation()
@@ -144,9 +144,9 @@ private Q_SLOTS:
 
     void testSizes()
     {
-        Database db(testDataPath, dbName);
-        qDebug() << "Database size [kb]: " << db.diskUsage()/1024;
-        db.removeFromDisk();
+        Storage store(testDataPath, dbName);
+        qDebug() << "Database size [kb]: " << store.diskUsage()/1024;
+        store.removeFromDisk();
 
         QFileInfo fileInfo(filePath);
         qDebug() << "File size [kb]: " << fileInfo.size()/1024;