From f57dca821e2a33783ec0dd0af2f5c31daef4fa64 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Thu, 4 Dec 2014 19:23:09 +0100 Subject: kyoto! --- store/test/CMakeLists.txt | 2 +- store/test/storagebenchmark.cpp | 21 +++++++++------------ 2 files changed, 10 insertions(+), 13 deletions(-) (limited to 'store/test') diff --git a/store/test/CMakeLists.txt b/store/test/CMakeLists.txt index 4743cfb..1b9dc9e 100644 --- a/store/test/CMakeLists.txt +++ b/store/test/CMakeLists.txt @@ -12,7 +12,7 @@ macro(manual_tests) foreach(_testname ${ARGN}) add_executable(${_testname} ${_testname}.cpp ${store_SRCS}) qt5_use_modules(${_testname} Core Test) - target_link_libraries(${_testname} lmdb) + target_link_libraries(${_testname} kyotocabinet) endforeach(_testname) endmacro(auto_tests) diff --git a/store/test/storagebenchmark.cpp b/store/test/storagebenchmark.cpp index 3be90e9..d42dea8 100644 --- a/store/test/storagebenchmark.cpp +++ b/store/test/storagebenchmark.cpp @@ -46,21 +46,16 @@ class StorageBenchmark : public QObject private: //This should point to a directory on disk and not a ramdisk (since we're measuring performance) QString testDataPath; - QString dbPath; + QString dbName; QString filePath; const int count = 50000; private Q_SLOTS: void initTestCase() { - testDataPath = "./"; - dbPath = testDataPath + "testdb"; + testDataPath = "./testdb"; + dbName = "test"; filePath = testDataPath + "buffer.fb"; - - QDir dir(testDataPath); - dir.remove("testdb/data.mdb"); - dir.remove("testdb/lock.mdb"); - dir.remove(filePath); } void testWriteRead_data() @@ -79,7 +74,7 @@ private Q_SLOTS: Database *db = 0; if (useDb) { - db = new Database(dbPath); + db = new Database(testDataPath, dbName); } std::ofstream myfile; @@ -118,7 +113,7 @@ private Q_SLOTS: { for (int i = 0; i < count; i++) { if (db) { - db->read(keyPrefix + std::to_string(i), [](void *ptr, int size){}); + db->read(keyPrefix + std::to_string(i), [](std::string value){}); } } } @@ -149,9 +144,11 @@ private Q_SLOTS: void testSizes() { - QFileInfo dbInfo(dbPath, "data.mdb"); + Database db(testDataPath, dbName); + qDebug() << "Database size [kb]: " << db.diskUsage()/1024; + db.removeFromDisk(); + QFileInfo fileInfo(filePath); - qDebug() << "Database size [kb]: " << dbInfo.size()/1024; qDebug() << "File size [kb]: " << fileInfo.size()/1024; } }; -- cgit v1.2.3 From 767312e2063f4e58af3de0f27aba52de49e14295 Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 5 Dec 2014 09:17:46 +0100 Subject: major reorg that puts Storage (previously Database) into common there is now a top-level tests dir, and a compile time switch for lmdb vs kyotocabinet --- store/test/CMakeLists.txt | 21 ------ store/test/calendar.fbs | 12 --- store/test/storagebenchmark.cpp | 157 ---------------------------------------- 3 files changed, 190 deletions(-) delete mode 100644 store/test/CMakeLists.txt delete mode 100644 store/test/calendar.fbs delete mode 100644 store/test/storagebenchmark.cpp (limited to 'store/test') diff --git a/store/test/CMakeLists.txt b/store/test/CMakeLists.txt deleted file mode 100644 index 1b9dc9e..0000000 --- a/store/test/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -set(CMAKE_AUTOMOC ON) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -set(store_path "../") -set(store_SRCS - ${store_path}/database.cpp -) - -generate_flatbuffers(calendar) - -macro(manual_tests) - foreach(_testname ${ARGN}) - add_executable(${_testname} ${_testname}.cpp ${store_SRCS}) - qt5_use_modules(${_testname} Core Test) - target_link_libraries(${_testname} kyotocabinet) - endforeach(_testname) -endmacro(auto_tests) - -manual_tests ( - storagebenchmark -) diff --git a/store/test/calendar.fbs b/store/test/calendar.fbs deleted file mode 100644 index 203ee43..0000000 --- a/store/test/calendar.fbs +++ /dev/null @@ -1,12 +0,0 @@ -// example IDL file - -namespace Calendar; - -table Event { - summary:string; - description:string; - attachment:[byte]; -} - -root_type Event; -file_identifier "AKFB"; diff --git a/store/test/storagebenchmark.cpp b/store/test/storagebenchmark.cpp deleted file mode 100644 index d42dea8..0000000 --- a/store/test/storagebenchmark.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include - -#include "calendar_generated.h" - -#include -#include - -#include -#include -#include -#include - -#include "store/database.h" - -using namespace Calendar; -using namespace flatbuffers; - -static std::string createEvent() -{ - FlatBufferBuilder fbb; - { - auto summary = fbb.CreateString("summary"); - - const int attachmentSize = 1024*2; // 2KB - int8_t rawData[attachmentSize]; - auto data = fbb.CreateVector(rawData, attachmentSize); - - Calendar::EventBuilder eventBuilder(fbb); - eventBuilder.add_summary(summary); - eventBuilder.add_attachment(data); - auto eventLocation = eventBuilder.Finish(); - FinishEventBuffer(fbb, eventLocation); - } - return std::string(reinterpret_cast(fbb.GetBufferPointer()), fbb.GetSize()); -} - -// static void readEvent(const std::string &data) -// { -// auto readEvent = GetEvent(data.c_str()); -// std::cout << readEvent->summary()->c_str() << std::endl; -// } - -class StorageBenchmark : public QObject -{ - Q_OBJECT -private: - //This should point to a directory on disk and not a ramdisk (since we're measuring performance) - QString testDataPath; - QString dbName; - QString filePath; - const int count = 50000; - -private Q_SLOTS: - void initTestCase() - { - testDataPath = "./testdb"; - dbName = "test"; - filePath = testDataPath + "buffer.fb"; - } - - void testWriteRead_data() - { - QTest::addColumn("useDb"); - QTest::addColumn("count"); - - QTest::newRow("db, 50k") << true << count; - QTest::newRow("file, 50k") << false << count; - } - - void testWriteRead() - { - QFETCH(bool, useDb); - QFETCH(int, count); - - Database *db = 0; - if (useDb) { - db = new Database(testDataPath, dbName); - } - - std::ofstream myfile; - myfile.open(filePath.toStdString()); - const char *keyPrefix = "key"; - - QTime time; - - time.start(); - { - auto event = createEvent(); - for (int i = 0; i < count; i++) { - if (db) { - if (i % 10000 == 0) { - if (i > 0) { - db->commitTransaction(); - } - db->startTransaction(); - } - - db->write(keyPrefix + std::to_string(i), event); - } else { - myfile << event; - } - } - - if (db) { - db->commitTransaction(); - } else { - myfile.close(); - } - } - const int writeDuration = time.restart(); - qDebug() << "Writing took[ms]: " << writeDuration; - - { - for (int i = 0; i < count; i++) { - if (db) { - db->read(keyPrefix + std::to_string(i), [](std::string value){}); - } - } - } - const int readDuration = time.restart(); - - if (db) { - qDebug() << "Reading took[ms]: " << readDuration; - } else { - qDebug() << "File reading is not implemented."; - } - - delete db; - } - - void testBufferCreation() - { - QTime time; - - time.start(); - { - for (int i = 0; i < count; i++) { - auto event = createEvent(); - } - } - const int bufferDuration = time.elapsed(); - qDebug() << "Creating buffers took[ms]: " << bufferDuration; - } - - void testSizes() - { - Database db(testDataPath, dbName); - qDebug() << "Database size [kb]: " << db.diskUsage()/1024; - db.removeFromDisk(); - - QFileInfo fileInfo(filePath); - qDebug() << "File size [kb]: " << fileInfo.size()/1024; - } -}; - -QTEST_MAIN(StorageBenchmark) -#include "storagebenchmark.moc" -- cgit v1.2.3 From ee41f8d17bdc667fbbbc83deeff766faf048cf5e Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Fri, 5 Dec 2014 00:57:03 +0100 Subject: A storagetest including concurrency read test. Conflicts: tests/CMakeLists.txt --- store/test/storagetest.cpp | 111 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 store/test/storagetest.cpp (limited to 'store/test') diff --git a/store/test/storagetest.cpp b/store/test/storagetest.cpp new file mode 100644 index 0000000..1b105af --- /dev/null +++ b/store/test/storagetest.cpp @@ -0,0 +1,111 @@ +#include + +#include + +#include +#include +#include + +#include "store/database.h" + +class StorageTest : public QObject +{ + Q_OBJECT +private: + //This should point to a directory on disk and not a ramdisk (since we're measuring performance) + QString testDataPath; + QString dbName; + const char *keyPrefix = "key"; + + void populate(int count) + { + Database db(testDataPath, dbName); + for (int i = 0; i < count; i++) { + //This should perhaps become an implementation detail of the db? + if (i % 10000 == 0) { + if (i > 0) { + db.commitTransaction(); + } + db.startTransaction(); + } + db.write(keyPrefix + std::to_string(i), keyPrefix + std::to_string(i)); + } + db.commitTransaction(); + } + + bool verify(Database &db, int i) + { + bool error = false; + const auto reference = keyPrefix + std::to_string(i); + db.read(keyPrefix + std::to_string(i), [&error, &reference](const std::string &value) { + if (value != reference) { + qDebug() << "Mismatch while reading"; + error = true; + } + }); + return !error; + } + +private Q_SLOTS: + void initTestCase() + { + testDataPath = "./testdb"; + dbName = "test"; + } + + void cleanupTestCase() + { + Database db(testDataPath, dbName); + db.removeFromDisk(); + } + + + void testRead() + { + const int count = 100; + + populate(count); + + //ensure we can read everything back correctly + { + Database db(testDataPath, dbName); + for (int i = 0; i < count; i++) { + QVERIFY(verify(db, i)); + } + } + + Database db(testDataPath, dbName); + db.removeFromDisk(); + } + + void testConcurrentRead() + { + const int count = 10000; + + populate(count); + + //Try to concurrently read + QList > futures; + const int concurrencyLevel = 4; + for (int num = 0; num < concurrencyLevel; num++) { + futures << QtConcurrent::run([this, count](){ + Database db(testDataPath, dbName); + for (int i = 0; i < count; i++) { + if (!verify(db, i)) { + qWarning() << "invalid value"; + break; + } + } + }); + } + for(auto future : futures) { + future.waitForFinished(); + } + + Database db(testDataPath, dbName); + db.removeFromDisk(); + } +}; + +QTEST_MAIN(StorageTest) +#include "storagetest.moc" -- cgit v1.2.3 From 0c1400c7f0cf2f545a6cd7347314c1158fbfa36f Mon Sep 17 00:00:00 2001 From: Aaron Seigo Date: Fri, 5 Dec 2014 09:33:35 +0100 Subject: mv storagetest.cpp to the right location --- store/test/storagetest.cpp | 111 --------------------------------------------- 1 file changed, 111 deletions(-) delete mode 100644 store/test/storagetest.cpp (limited to 'store/test') diff --git a/store/test/storagetest.cpp b/store/test/storagetest.cpp deleted file mode 100644 index 1b105af..0000000 --- a/store/test/storagetest.cpp +++ /dev/null @@ -1,111 +0,0 @@ -#include - -#include - -#include -#include -#include - -#include "store/database.h" - -class StorageTest : public QObject -{ - Q_OBJECT -private: - //This should point to a directory on disk and not a ramdisk (since we're measuring performance) - QString testDataPath; - QString dbName; - const char *keyPrefix = "key"; - - void populate(int count) - { - Database db(testDataPath, dbName); - for (int i = 0; i < count; i++) { - //This should perhaps become an implementation detail of the db? - if (i % 10000 == 0) { - if (i > 0) { - db.commitTransaction(); - } - db.startTransaction(); - } - db.write(keyPrefix + std::to_string(i), keyPrefix + std::to_string(i)); - } - db.commitTransaction(); - } - - bool verify(Database &db, int i) - { - bool error = false; - const auto reference = keyPrefix + std::to_string(i); - db.read(keyPrefix + std::to_string(i), [&error, &reference](const std::string &value) { - if (value != reference) { - qDebug() << "Mismatch while reading"; - error = true; - } - }); - return !error; - } - -private Q_SLOTS: - void initTestCase() - { - testDataPath = "./testdb"; - dbName = "test"; - } - - void cleanupTestCase() - { - Database db(testDataPath, dbName); - db.removeFromDisk(); - } - - - void testRead() - { - const int count = 100; - - populate(count); - - //ensure we can read everything back correctly - { - Database db(testDataPath, dbName); - for (int i = 0; i < count; i++) { - QVERIFY(verify(db, i)); - } - } - - Database db(testDataPath, dbName); - db.removeFromDisk(); - } - - void testConcurrentRead() - { - const int count = 10000; - - populate(count); - - //Try to concurrently read - QList > futures; - const int concurrencyLevel = 4; - for (int num = 0; num < concurrencyLevel; num++) { - futures << QtConcurrent::run([this, count](){ - Database db(testDataPath, dbName); - for (int i = 0; i < count; i++) { - if (!verify(db, i)) { - qWarning() << "invalid value"; - break; - } - } - }); - } - for(auto future : futures) { - future.waitForFinished(); - } - - Database db(testDataPath, dbName); - db.removeFromDisk(); - } -}; - -QTEST_MAIN(StorageTest) -#include "storagetest.moc" -- cgit v1.2.3