summaryrefslogtreecommitdiffstats
path: root/common/storage_lmdb.cpp
diff options
context:
space:
mode:
authorChristian Mollekopf <chrigi_1@fastmail.fm>2016-07-11 11:55:29 +0200
committerChristian Mollekopf <chrigi_1@fastmail.fm>2016-07-11 11:55:29 +0200
commit3a3118e768e1447dc7524328e84b8d7faef81fe1 (patch)
treeaf5582170ed6164fffc9365f34b17bf449c0db40 /common/storage_lmdb.cpp
parentf9379318d801df204cc50385c5eca1f28e91755e (diff)
parentce2fd2666f084eebe443598f6f3740a02913091e (diff)
downloadsink-3a3118e768e1447dc7524328e84b8d7faef81fe1.tar.gz
sink-3a3118e768e1447dc7524328e84b8d7faef81fe1.zip
Merge branch 'feature/notifications' into develop
Diffstat (limited to 'common/storage_lmdb.cpp')
-rw-r--r--common/storage_lmdb.cpp61
1 files changed, 46 insertions, 15 deletions
diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp
index 3687594..79f4465 100644
--- a/common/storage_lmdb.cpp
+++ b/common/storage_lmdb.cpp
@@ -34,10 +34,8 @@
34#include <lmdb.h> 34#include <lmdb.h>
35#include "log.h" 35#include "log.h"
36 36
37#undef Trace 37SINK_DEBUG_AREA("storage")
38#define Trace() Trace_area("storage." + d->storageRoot.toLatin1() + '/' + d->name.toLatin1()) 38// SINK_DEBUG_COMPONENT(d->storageRoot.toLatin1() + '/' + d->name.toLatin1())
39#undef Warning
40#define Warning() Warning_area("storage")
41 39
42namespace Sink { 40namespace Sink {
43 41
@@ -103,6 +101,21 @@ Storage::NamedDatabase::NamedDatabase(NamedDatabase::Private *prv) : d(prv)
103{ 101{
104} 102}
105 103
104Storage::NamedDatabase::NamedDatabase(NamedDatabase &&other) : d(nullptr)
105{
106 *this = std::move(other);
107}
108
109Storage::NamedDatabase &Storage::NamedDatabase::operator=(Storage::NamedDatabase &&other)
110{
111 if (&other != this) {
112 delete d;
113 d = other.d;
114 other.d = nullptr;
115 }
116 return *this;
117}
118
106Storage::NamedDatabase::~NamedDatabase() 119Storage::NamedDatabase::~NamedDatabase()
107{ 120{
108 delete d; 121 delete d;
@@ -339,7 +352,7 @@ qint64 Storage::NamedDatabase::getSize()
339 MDB_stat stat; 352 MDB_stat stat;
340 rc = mdb_stat(d->transaction, d->dbi, &stat); 353 rc = mdb_stat(d->transaction, d->dbi, &stat);
341 if (rc) { 354 if (rc) {
342 Warning() << "Something went wrong " << QByteArray(mdb_strerror(rc)); 355 SinkWarning() << "Something went wrong " << QByteArray(mdb_strerror(rc));
343 } 356 }
344 // std::cout << "overflow_pages: " << stat.ms_overflow_pages << std::endl; 357 // std::cout << "overflow_pages: " << stat.ms_overflow_pages << std::endl;
345 // std::cout << "page size: " << stat.ms_psize << std::endl; 358 // std::cout << "page size: " << stat.ms_psize << std::endl;
@@ -398,6 +411,21 @@ Storage::Transaction::Transaction(Transaction::Private *prv) : d(prv)
398 d->startTransaction(); 411 d->startTransaction();
399} 412}
400 413
414Storage::Transaction::Transaction(Transaction &&other) : d(nullptr)
415{
416 *this = std::move(other);
417}
418
419Storage::Transaction &Storage::Transaction::operator=(Storage::Transaction &&other)
420{
421 if (&other != this) {
422 delete d;
423 d = other.d;
424 other.d = nullptr;
425 }
426 return *this;
427}
428
401Storage::Transaction::~Transaction() 429Storage::Transaction::~Transaction()
402{ 430{
403 if (d && d->transaction) { 431 if (d && d->transaction) {
@@ -452,7 +480,7 @@ static bool ensureCorrectDb(Storage::NamedDatabase &database, const QByteArray &
452 bool openedTheWrongDatabase = false; 480 bool openedTheWrongDatabase = false;
453 auto count = database.scan("__internal_dbname", [db, &openedTheWrongDatabase](const QByteArray &key, const QByteArray &value) ->bool { 481 auto count = database.scan("__internal_dbname", [db, &openedTheWrongDatabase](const QByteArray &key, const QByteArray &value) ->bool {
454 if (value != db) { 482 if (value != db) {
455 Warning() << "Opened the wrong database, got " << value << " instead of " << db; 483 SinkWarning() << "Opened the wrong database, got " << value << " instead of " << db;
456 openedTheWrongDatabase = true; 484 openedTheWrongDatabase = true;
457 } 485 }
458 return false; 486 return false;
@@ -475,7 +503,7 @@ bool Storage::Transaction::validateNamedDatabases()
475 for (const auto &dbName : databases) { 503 for (const auto &dbName : databases) {
476 auto db = openDatabase(dbName); 504 auto db = openDatabase(dbName);
477 if (!db) { 505 if (!db) {
478 Warning() << "Failed to open the database: " << dbName; 506 SinkWarning() << "Failed to open the database: " << dbName;
479 return false; 507 return false;
480 } 508 }
481 } 509 }
@@ -497,7 +525,7 @@ Storage::NamedDatabase Storage::Transaction::openDatabase(const QByteArray &db,
497 } 525 }
498 auto database = Storage::NamedDatabase(p); 526 auto database = Storage::NamedDatabase(p);
499 if (!ensureCorrectDb(database, db, d->requestedRead)) { 527 if (!ensureCorrectDb(database, db, d->requestedRead)) {
500 Warning() << "Failed to open the database" << db; 528 SinkWarning() << "Failed to open the database" << db;
501 return Storage::NamedDatabase(); 529 return Storage::NamedDatabase();
502 } 530 }
503 return database; 531 return database;
@@ -506,7 +534,7 @@ Storage::NamedDatabase Storage::Transaction::openDatabase(const QByteArray &db,
506QList<QByteArray> Storage::Transaction::getDatabaseNames() const 534QList<QByteArray> Storage::Transaction::getDatabaseNames() const
507{ 535{
508 if (!d) { 536 if (!d) {
509 Warning() << "Invalid transaction"; 537 SinkWarning() << "Invalid transaction";
510 return QList<QByteArray>(); 538 return QList<QByteArray>();
511 } 539 }
512 540
@@ -529,11 +557,12 @@ QList<QByteArray> Storage::Transaction::getDatabaseNames() const
529 rc = 0; 557 rc = 0;
530 } 558 }
531 if (rc) { 559 if (rc) {
532 Warning() << "Failed to get a value" << rc; 560 SinkWarning() << "Failed to get a value" << rc;
533 } 561 }
534 } 562 }
563 mdb_cursor_close(cursor);
535 } else { 564 } else {
536 Warning() << "Failed to open db" << rc << QByteArray(mdb_strerror(rc)); 565 SinkWarning() << "Failed to open db" << rc << QByteArray(mdb_strerror(rc));
537 } 566 }
538 return list; 567 return list;
539} 568}
@@ -580,7 +609,7 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m) : st
580 int rc = 0; 609 int rc = 0;
581 if ((rc = mdb_env_create(&env))) { 610 if ((rc = mdb_env_create(&env))) {
582 // TODO: handle error 611 // TODO: handle error
583 Warning() << "mdb_env_create: " << rc << " " << mdb_strerror(rc); 612 SinkWarning() << "mdb_env_create: " << rc << " " << mdb_strerror(rc);
584 } else { 613 } else {
585 mdb_env_set_maxdbs(env, 50); 614 mdb_env_set_maxdbs(env, 50);
586 unsigned int flags = MDB_NOTLS; 615 unsigned int flags = MDB_NOTLS;
@@ -588,11 +617,13 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m) : st
588 flags |= MDB_RDONLY; 617 flags |= MDB_RDONLY;
589 } 618 }
590 if ((rc = mdb_env_open(env, fullPath.toStdString().data(), flags, 0664))) { 619 if ((rc = mdb_env_open(env, fullPath.toStdString().data(), flags, 0664))) {
591 Warning() << "mdb_env_open: " << rc << " " << mdb_strerror(rc); 620 SinkWarning() << "mdb_env_open: " << rc << " " << mdb_strerror(rc);
592 mdb_env_close(env); 621 mdb_env_close(env);
593 env = 0; 622 env = 0;
594 } else { 623 } else {
595 // FIXME: dynamic resize 624 // FIXME: dynamic resize
625 // In order to run valgrind this size must be smaller than half your available RAM
626 // https://github.com/BVLC/caffe/issues/2404
596 const size_t dbSize = (size_t)10485760 * (size_t)8000; // 1MB * 8000 627 const size_t dbSize = (size_t)10485760 * (size_t)8000; // 1MB * 8000
597 mdb_env_set_mapsize(env, dbSize); 628 mdb_env_set_mapsize(env, dbSize);
598 sEnvironments.insert(fullPath, env); 629 sEnvironments.insert(fullPath, env);
@@ -648,7 +679,7 @@ qint64 Storage::diskUsage() const
648{ 679{
649 QFileInfo info(d->storageRoot + '/' + d->name + "/data.mdb"); 680 QFileInfo info(d->storageRoot + '/' + d->name + "/data.mdb");
650 if (!info.exists()) { 681 if (!info.exists()) {
651 Warning() << "Tried to get filesize for non-existant file: " << info.path(); 682 SinkWarning() << "Tried to get filesize for non-existant file: " << info.path();
652 } 683 }
653 return info.size(); 684 return info.size();
654} 685}
@@ -658,7 +689,7 @@ void Storage::removeFromDisk() const
658 const QString fullPath(d->storageRoot + '/' + d->name); 689 const QString fullPath(d->storageRoot + '/' + d->name);
659 QMutexLocker locker(&d->sMutex); 690 QMutexLocker locker(&d->sMutex);
660 QDir dir(fullPath); 691 QDir dir(fullPath);
661 Trace() << "Removing database from disk: " << fullPath; 692 SinkTrace() << "Removing database from disk: " << fullPath;
662 if (!dir.removeRecursively()) { 693 if (!dir.removeRecursively()) {
663 Error error(d->name.toLatin1(), ErrorCodes::GenericError, QString("Failed to remove directory %1 %2").arg(d->storageRoot).arg(d->name).toLatin1()); 694 Error error(d->name.toLatin1(), ErrorCodes::GenericError, QString("Failed to remove directory %1 %2").arg(d->storageRoot).arg(d->name).toLatin1());
664 defaultErrorHandler()(error); 695 defaultErrorHandler()(error);