Merge branch 'kyoto'

Conflicts:
	common/storage.h
	common/storage_lmdb.cpp
	dummyresource/facade.cpp
	store/test/CMakeLists.txt
	tests/storagebenchmark.cpp

12 files changed, 352 insertions, 58 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 392ca39..e8b6f23 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -47,10 +47,10 @@ add_subdirectory(client)
 # the resource
 add_subdirectory(resource)
 
-# the store
-add_subdirectory(store)
-
 # a simple dummy resource implementation
 add_subdirectory(dummyresource)
 
+# some tests
+add_subdirectory(tests)
+
 feature_summary(WHAT ALL FATAL_ON_MISSING_REQUIRED_PACKAGES)
diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
index d409828..9b3f777 100644
--- a/common/CMakeLists.txt
+++ b/common/CMakeLists.txt
@@ -2,10 +2,20 @@ project(akonadinextcommon)
 generate_flatbuffers(commands/handshake
                      commands/revisionupdate)
 
+if (STORAGE_KYOTO)
+    set(storage_SRCS storage_kyoto.cpp)
+    set(storage_LIBS kyotocabinet)
+else (STORAGE_KYOTO)
+    set(storage_SRCS storage_lmdb.cpp)
+    set(storage_LIBS lmdb)
+endif (STORAGE_KYOTO)
+
 set(command_SRCS
    commands.cpp
    console.cpp
-   ${CMAKE_CURRENT_BINARY_DIR}/commands/handshake_generated.h)
+   ${storage_SRCS})
 
 add_library(${PROJECT_NAME} ${command_SRCS})
+SET_TARGET_PROPERTIES(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
 qt5_use_modules(${PROJECT_NAME} Widgets)
+target_link_libraries(${PROJECT_NAME} ${storage_LIBS})
diff --git a/store/database.h b/common/storage.h
index 1cede39..0b548fb 100644
--- a/store/database.h
+++ b/common/storage.h
@@ -3,12 +3,12 @@
 #include <string>
 #include <QString>
 
-class Database {
+class Storage {
 public:
     enum TransactionType { ReadOnly, ReadWrite };
 
-    Database(const QString &storageRoot, const QString &name);
-    ~Database();
+    Storage(const QString &storageRoot, const QString &name);
+    ~Storage();
     bool isInTransaction() const;
     bool startTransaction(TransactionType type = ReadWrite);
     bool commitTransaction();
diff --git a/common/storage_kyoto.cpp b/common/storage_kyoto.cpp
new file mode 100644
index 0000000..40bd3e6
--- /dev/null
+++ b/common/storage_kyoto.cpp
@@ -0,0 +1,170 @@
+#include "storage.h"
+
+#include <iostream>
+
+#include <QAtomicInt>
+#include <QDebug>
+#include <QDir>
+#include <QFileInfo>
+#include <QReadWriteLock>
+#include <QString>
+#include <QTime>
+
+#include <kchashdb.h>
+
+class Storage::Private
+{
+public:
+    Private(const QString &storageRoot, const QString &name);
+    ~Private();
+
+    kyotocabinet::TreeDB db;
+    bool dbOpen;
+    bool inTransaction;
+};
+
+Storage::Private::Private(const QString &storageRoot, const QString &name)
+    : inTransaction(false)
+{
+    QDir dir;
+    dir.mkdir(storageRoot);
+
+    //create file
+    dbOpen = db.open((storageRoot + "/" + name + ".kch").toStdString(), kyotocabinet::BasicDB::OWRITER | kyotocabinet::BasicDB::OCREATE);
+    if (!dbOpen) {
+        // TODO: handle error
+    }
+}
+
+Storage::Private::~Private()
+{
+    if (dbOpen && inTransaction) {
+        db.end_transaction(false);
+    }
+}
+
+Storage::Storage(const QString &storageRoot, const QString &name)
+    : d(new Private(storageRoot, name))
+{
+}
+
+Storage::~Storage()
+{
+    delete d;
+}
+
+bool Storage::isInTransaction() const
+{
+    return d->inTransaction;
+}
+
+bool Storage::startTransaction(TransactionType type)
+{
+    if (!d->dbOpen) {
+        return false;
+    }
+
+    if (d->inTransaction) {
+        return true;
+    }
+
+    //TODO handle errors
+    d->inTransaction = d->db.begin_transaction();
+    return d->inTransaction;
+}
+
+bool Storage::commitTransaction()
+{
+    if (!d->dbOpen) {
+        return false;
+    }
+
+    if (!d->inTransaction) {
+        return false;
+    }
+
+    bool success = d->db.end_transaction(true);
+    d->inTransaction = false;
+    return success;
+}
+
+void Storage::abortTransaction()
+{
+    if (!d->dbOpen || !d->inTransaction) {
+        return;
+    }
+
+    d->db.end_transaction(false);
+    d->inTransaction = false;
+}
+
+bool Storage::write(const char *key, size_t keySize, const char *value, size_t valueSize)
+{
+    if (!d->dbOpen) {
+        return false;
+    }
+
+    bool success = d->db.set(key, keySize, value, valueSize);
+    return success; 
+}
+
+bool Storage::write(const std::string &sKey, const std::string &sValue)
+{
+    if (!d->dbOpen) {
+        return false;
+    }
+
+    bool success = d->db.set(sKey, sValue);
+    return success; 
+}
+
+bool Storage::read(const std::string &sKey, const std::function<void(const std::string &value)> &resultHandler)
+{
+    if (!d->dbOpen) {
+        return false;
+    }
+
+    std::string value;
+    if (d->db.get(sKey, &value)) {
+        resultHandler(value);
+        return true;
+    }
+
+    return false;
+}
+
+bool Storage::read(const std::string &sKey, const std::function<void(void *ptr, int size)> &resultHandler)
+{
+    if (!d->dbOpen) {
+        return false;
+    }
+
+    size_t valueSize;
+    char *valueBuffer = d->db.get(sKey.data(), sKey.size(), &valueSize);
+    if (valueBuffer) {
+        resultHandler(valueBuffer, valueSize);
+    }
+    delete[] valueBuffer;
+    return valueBuffer != nullptr;
+}
+
+qint64 Storage::diskUsage() const
+{
+    if (!d->dbOpen) {
+        return 0;
+    }
+
+    QFileInfo info(QString::fromStdString(d->db.path()));
+    return info.size();
+}
+
+void Storage::removeFromDisk() const
+{
+    if (!d->dbOpen) {
+        return;
+    }
+
+   QFileInfo info(QString::fromStdString(d->db.path()));
+   QDir dir = info.dir();
+   dir.remove(info.fileName());
+}
diff --git a/store/database.cpp b/common/storage_lmdb.cpp
index c4dfbd6..283205f 100644
--- a/store/database.cpp
+++ b/common/storage_lmdb.cpp
@@ -1,4 +1,4 @@
-#include "database.h"
+#include "storage.h"
 
 #include <iostream>
 
@@ -11,7 +11,7 @@
 
 #include <lmdb.h>
 
-class Database::Private
+class Storage::Private
 {
 public:
     Private(const QString &s, const QString &name);
@@ -19,6 +19,7 @@ public:
 
     QString storageRoot;
     QString name;
+
     MDB_dbi dbi;
     MDB_env *env;
     MDB_txn *transaction;
@@ -26,7 +27,7 @@ public:
     bool firstOpen;
 };
 
-Database::Private::Private(const QString &s, const QString &n)
+Storage::Private::Private(const QString &s, const QString &n)
     : transaction(0),
       readTransaction(false),
       firstOpen(true),
@@ -53,7 +54,7 @@ Database::Private::Private(const QString &s, const QString &n)
     }
 }
 
-Database::Private::~Private()
+Storage::Private::~Private()
 {
     if (transaction) {
         mdb_txn_abort(transaction);
@@ -64,22 +65,22 @@ Database::Private::~Private()
     mdb_env_close(env);
 }
 
-Database::Database(const QString &storageRoot, const QString &name)
+Storage::Storage(const QString &storageRoot, const QString &name)
     : d(new Private(storageRoot, name))
 {
 }
 
-Database::~Database()
+Storage::~Storage()
 {
     delete d;
 }
 
-bool Database::isInTransaction() const
+bool Storage::isInTransaction() const
 {
     return d->transaction;
 }
 
-bool Database::startTransaction(TransactionType type)
+bool Storage::startTransaction(TransactionType type)
 {
     if (!d->env) {
         return false;
@@ -114,7 +115,7 @@ bool Database::startTransaction(TransactionType type)
     return !rc;
 }
 
-bool Database::commitTransaction()
+bool Storage::commitTransaction()
 {
     if (!d->env) {
         return false;
@@ -135,7 +136,7 @@ bool Database::commitTransaction()
     return !rc;
 }
 
-void Database::abortTransaction()
+void Storage::abortTransaction()
 {
     if (!d->env || !d->transaction) {
         return;
@@ -145,12 +146,12 @@ void Database::abortTransaction()
     d->transaction = 0;
 }
 
-bool Database::write(const char *key, size_t keySize, const char *value, size_t valueSize)
+bool Storage::write(const char *key, size_t keySize, const char *value, size_t valueSize)
 {
     write(std::string(key, keySize), std::string(value, valueSize));
 }
 
-bool Database::write(const std::string &sKey, const std::string &sValue)
+bool Storage::write(const std::string &sKey, const std::string &sValue)
 {
     if (!d->env) {
         return false;
@@ -187,7 +188,7 @@ bool Database::write(const std::string &sKey, const std::string &sValue)
     return !rc;
 }
 
-bool Database::read(const std::string &sKey, const std::function<void(const std::string &value)> &resultHandler)
+bool Storage::read(const std::string &sKey, const std::function<void(const std::string &value)> &resultHandler)
 {
     return read(sKey,
          [&](void *ptr, int size) {
@@ -197,7 +198,7 @@ bool Database::read(const std::string &sKey, const std::function<void(const std:
 // std::cout << "key: " << resultKey << " data: " << resultValue << std::endl;
 }
 
-bool Database::read(const std::string &sKey, const std::function<void(void *ptr, int size)> &resultHandler)
+bool Storage::read(const std::string &sKey, const std::function<void(void *ptr, int size)> &resultHandler)
 {
     if (!d->env) {
         return false;
@@ -244,13 +245,13 @@ bool Database::read(const std::string &sKey, const std::function<void(void *ptr,
         }
     }
 
+    mdb_cursor_close(cursor);
+
     if (rc) {
         std::cerr << "mdb_cursor_get: " << rc << " " << mdb_strerror(rc) << std::endl;
         return false;
     }
 
-    mdb_cursor_close(cursor);
-
     /**
       we don't abort the transaction since we need it for reading the values
     if (implicitTransaction) {
@@ -260,13 +261,20 @@ bool Database::read(const std::string &sKey, const std::function<void(void *ptr,
     return true;
 }
 
-qint64 Database::diskUsage() const
+qint64 Storage::diskUsage() const
 {
     QFileInfo info(d->storageRoot + "/data.mdb");
     return info.size();
 }
 
-void Database::removeFromDisk() const
+void Storage::removeFromDisk() const
+{
+    QDir dir(d->path);
+    dir.remove("data.mdb");
+    dir.remove("lock.mdb");
+}
+
+void Storage::removeFromDisk() const
 {
     QDir dir(d->storageRoot);
     dir.remove("data.mdb");
diff --git a/dummyresource/facade.cpp b/dummyresource/facade.cpp
index 7e9881b..9758c1b 100644
--- a/dummyresource/facade.cpp
+++ b/dummyresource/facade.cpp
@@ -11,7 +11,7 @@ using namespace flatbuffers;
 DummyResourceFacade::DummyResourceFacade()
     : Akonadi2::StoreFacade<Akonadi2::Domain::Event>(),
     mResourceAccess(new ResourceAccess("dummyresource")),
-    mDatabase(new Database(Akonadi2::Store::storageLocation(), "dummyresource"))
+    mStorage(new Storage(Akonadi2::Store::storageLocation(), "dummyresource"))
 {
     // connect(mResourceAccess.data(), &ResourceAccess::ready, this, onReadyChanged);
 }
@@ -69,20 +69,20 @@ public:
     DummyEvent const *buffer;
 
     //Keep query alive so values remain valid
-    QSharedPointer<Database> db;
+    QSharedPointer<Storage> storage;
 };
 
 void DummyResourceFacade::load(const Akonadi2::Query &query, const std::function<void(const Akonadi2::Domain::Event::Ptr &)> &resultCallback)
 {
     qDebug() << "load called";
     //TODO only read values matching the query
-    auto db = QSharedPointer<Database>::create(Akonadi2::Store::storageLocation(), "dummyresource");
-    db->read("", [resultCallback, db](void *data, int size) {
+    auto storage = QSharedPointer<Storage>::create(Akonadi2::Store::storageLocation(), "dummyresource");
+    storage->read("", [resultCallback, storage](void *data, int size) {
         //TODO read second buffer as well
         auto eventBuffer = GetDummyEvent(data);
         auto event = QSharedPointer<DummyEventAdaptor>::create("dummyresource", "key", 0);
         event->buffer = eventBuffer;
-        event->db = db;
+        event->storage = storage;
         resultCallback(event);
     });
 }
diff --git a/dummyresource/facade.h b/dummyresource/facade.h
index 7a516de..310ef77 100644
--- a/dummyresource/facade.h
+++ b/dummyresource/facade.h
@@ -1,7 +1,7 @@
 #pragma once
 
 #include "client/clientapi.h"
-#include "store/database.h"
+#include "common/storage.h"
 
 class ResourceAccess;
 
@@ -18,5 +18,5 @@ public:
 
 private:
     QSharedPointer<ResourceAccess> mResourceAccess;
-    QSharedPointer<Database> mDatabase;
+    QSharedPointer<Storage> mStorage;
 };
diff --git a/store/CMakeLists.txt b/store/CMakeLists.txt
deleted file mode 100644
index 552439e..0000000
--- a/store/CMakeLists.txt
+++ /dev/null
@@ -1 +0,0 @@
-add_subdirectory(test)
diff --git a/store/test/CMakeLists.txt b/tests/CMakeLists.txt
index c5c4fcb..1629acb 100644
--- a/store/test/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,20 +1,15 @@
 set(CMAKE_AUTOMOC ON)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-set(store_path "../")
-set(store_SRCS
-    ${store_path}/database.cpp
-)
-
 generate_flatbuffers(calendar)
 
 macro(manual_tests)
     foreach(_testname ${ARGN})
-        add_executable(${_testname} ${_testname}.cpp ${store_SRCS})
+        add_executable(${_testname} ${_testname}.cpp)
         qt5_use_modules(${_testname} Core Test Concurrent)
-        target_link_libraries(${_testname} lmdb)
+        target_link_libraries(${_testname} akonadinextcommon)
     endforeach(_testname)
-endmacro(auto_tests)
+endmacro(manual_tests)
 
 manual_tests (
     storagebenchmark
diff --git a/store/test/calendar.fbs b/tests/calendar.fbs
index 203ee43..203ee43 100644
--- a/store/test/calendar.fbs
+++ b/tests/calendar.fbs
diff --git a/store/test/storagebenchmark.cpp b/tests/storagebenchmark.cpp
index 0bad138..f9fea7c 100644
--- a/store/test/storagebenchmark.cpp
+++ b/tests/storagebenchmark.cpp
@@ -9,7 +9,7 @@
 #include <QString>
 #include <QTime>
 
-#include "store/database.h"
+#include "common/storage.h"
 
 using namespace Calendar;
 using namespace flatbuffers;
@@ -59,8 +59,8 @@ private Q_SLOTS:
 
     void cleanupTestCase()
     {
-        Database db(testDataPath, dbName);
-        db.removeFromDisk();
+        Storage store(testDataPath, dbName);
+        store.removeFromDisk();
     }
 
     void testWriteRead_data()
@@ -77,9 +77,9 @@ private Q_SLOTS:
         QFETCH(bool, useDb);
         QFETCH(int, count);
 
-        Database *db = 0;
+        Storage *store = 0;
         if (useDb) {
-            db = new Database(testDataPath, dbName);
+            store = new Storage(testDataPath, dbName);
         }
 
         std::ofstream myfile;
@@ -92,22 +92,22 @@ private Q_SLOTS:
         {
             auto event = createEvent();
             for (int i = 0; i < count; i++) {
-                if (db) {
+                if (store) {
                     if (i % 10000 == 0) {
                         if (i > 0) {
-                            db->commitTransaction();
+                            store->commitTransaction();
                         }
-                        db->startTransaction();
+                        store->startTransaction();
                     }
 
-                    db->write(keyPrefix + std::to_string(i), event);
+                    store->write(keyPrefix + std::to_string(i), event);
                 } else {
                     myfile << event;
                 }
             }
 
-            if (db) {
-                db->commitTransaction();
+            if (store) {
+                store->commitTransaction();
             } else {
                 myfile.close();
             }
@@ -117,20 +117,20 @@ private Q_SLOTS:
 
         {
             for (int i = 0; i < count; i++) {
-                if (db) {
-                    db->read(keyPrefix + std::to_string(i), [](std::string value){});
+                if (store) {
+                    store->read(keyPrefix + std::to_string(i), [](std::string value){});
                 }
             }
         }
         const int readDuration = time.restart();
 
-        if (db) {
+        if (store) {
             qDebug() << "Reading took[ms]: " << readDuration;
         } else {
             qDebug() << "File reading is not implemented.";
         }
 
-        delete db;
+        delete store;
     }
 
     void testBufferCreation()
@@ -149,8 +149,8 @@ private Q_SLOTS:
 
     void testSizes()
     {
-        Database db(testDataPath, dbName);
-        qDebug() << "Database size [kb]: " << db.diskUsage()/1024;
+        Storage store(testDataPath, dbName);
+        qDebug() << "Database size [kb]: " << store.diskUsage()/1024;
 
         QFileInfo fileInfo(filePath);
         qDebug() << "File size [kb]: " << fileInfo.size()/1024;
diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp
new file mode 100644
index 0000000..2b2805d
--- /dev/null
+++ b/tests/storagetest.cpp
@@ -0,0 +1,112 @@
+#include <QtTest>
+
+#include <iostream>
+
+#include <QDebug>
+#include <QString>
+#include <QtConcurrent/QtConcurrentRun>
+
+#include "common/storage.h"
+
+class StorageTest : public QObject
+{
+    Q_OBJECT
+private:
+    //This should point to a directory on disk and not a ramdisk (since we're measuring performance)
+    QString testDataPath;
+    QString dbName;
+    const char *keyPrefix = "key";
+
+    void populate(int count)
+    {
+        Storage storage(testDataPath, dbName);
+        for (int i = 0; i < count; i++) {
+            //This should perhaps become an implementation detail of the db?
+            if (i % 10000 == 0) {
+                if (i > 0) {
+                    storage.commitTransaction();
+                }
+                storage.startTransaction();
+            }
+            storage.write(keyPrefix + std::to_string(i), keyPrefix + std::to_string(i));
+        }
+        storage.commitTransaction();
+    }
+
+    bool verify(Storage &storage, int i)
+    {
+        bool success = true;
+        bool keyMatch = true;
+        const auto reference = keyPrefix + std::to_string(i);
+        success = storage.read(keyPrefix + std::to_string(i), [&error, &reference](const std::string &value) {
+            if (value != reference) {
+                qDebug() << "Mismatch while reading";
+                keyMatch = false;
+            }
+        });
+        return succes && keyMatch;
+    }
+
+private Q_SLOTS:
+    void initTestCase()
+    {
+        testDataPath = "./testdb";
+        dbName = "test";
+    }
+
+    void cleanupTestCase()
+    {
+        Database db(testDataPath, dbName);
+        db.removeFromDisk();
+    }
+
+
+    void testRead()
+    {
+        const int count = 100;
+
+        populate(count);
+
+        //ensure we can read everything back correctly
+        {
+            Database db(testDataPath, dbName);
+            for (int i = 0; i < count; i++) {
+                QVERIFY(verify(db, i));
+            }
+        }
+
+        Database db(testDataPath, dbName);
+        db.removeFromDisk();
+    }
+
+    void testConcurrentRead()
+    {
+        const int count = 10000;
+
+        populate(count);
+
+        //Try to concurrently read
+        QList<QFuture<void> > futures;
+        const int concurrencyLevel = 4;
+        for (int num = 0; num < concurrencyLevel; num++) {
+            futures << QtConcurrent::run([this, count](){
+                Database db(testDataPath, dbName);
+                for (int i = 0; i < count; i++) {
+                    if (!verify(db, i)) {
+                        qWarning() << "invalid value";
+                        break;
+                    }
+                }
+            });
+        }
+        for(auto future : futures) {
+            future.waitForFinished();
+        }
+
+        Database db(testDataPath, dbName);
+        db.removeFromDisk();
+    }
+};
+
+QTEST_MAIN(StorageTest)
+#include "storagetest.moc"