diff options
author | Christian Mollekopf <chrigi_1@fastmail.fm> | 2016-07-11 11:55:29 +0200 |
---|---|---|
committer | Christian Mollekopf <chrigi_1@fastmail.fm> | 2016-07-11 11:55:29 +0200 |
commit | 3a3118e768e1447dc7524328e84b8d7faef81fe1 (patch) | |
tree | af5582170ed6164fffc9365f34b17bf449c0db40 /common/storage_lmdb.cpp | |
parent | f9379318d801df204cc50385c5eca1f28e91755e (diff) | |
parent | ce2fd2666f084eebe443598f6f3740a02913091e (diff) | |
download | sink-3a3118e768e1447dc7524328e84b8d7faef81fe1.tar.gz sink-3a3118e768e1447dc7524328e84b8d7faef81fe1.zip |
Merge branch 'feature/notifications' into develop
Diffstat (limited to 'common/storage_lmdb.cpp')
-rw-r--r-- | common/storage_lmdb.cpp | 61 |
1 files changed, 46 insertions, 15 deletions
diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index 3687594..79f4465 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp | |||
@@ -34,10 +34,8 @@ | |||
34 | #include <lmdb.h> | 34 | #include <lmdb.h> |
35 | #include "log.h" | 35 | #include "log.h" |
36 | 36 | ||
37 | #undef Trace | 37 | SINK_DEBUG_AREA("storage") |
38 | #define Trace() Trace_area("storage." + d->storageRoot.toLatin1() + '/' + d->name.toLatin1()) | 38 | // SINK_DEBUG_COMPONENT(d->storageRoot.toLatin1() + '/' + d->name.toLatin1()) |
39 | #undef Warning | ||
40 | #define Warning() Warning_area("storage") | ||
41 | 39 | ||
42 | namespace Sink { | 40 | namespace Sink { |
43 | 41 | ||
@@ -103,6 +101,21 @@ Storage::NamedDatabase::NamedDatabase(NamedDatabase::Private *prv) : d(prv) | |||
103 | { | 101 | { |
104 | } | 102 | } |
105 | 103 | ||
104 | Storage::NamedDatabase::NamedDatabase(NamedDatabase &&other) : d(nullptr) | ||
105 | { | ||
106 | *this = std::move(other); | ||
107 | } | ||
108 | |||
109 | Storage::NamedDatabase &Storage::NamedDatabase::operator=(Storage::NamedDatabase &&other) | ||
110 | { | ||
111 | if (&other != this) { | ||
112 | delete d; | ||
113 | d = other.d; | ||
114 | other.d = nullptr; | ||
115 | } | ||
116 | return *this; | ||
117 | } | ||
118 | |||
106 | Storage::NamedDatabase::~NamedDatabase() | 119 | Storage::NamedDatabase::~NamedDatabase() |
107 | { | 120 | { |
108 | delete d; | 121 | delete d; |
@@ -339,7 +352,7 @@ qint64 Storage::NamedDatabase::getSize() | |||
339 | MDB_stat stat; | 352 | MDB_stat stat; |
340 | rc = mdb_stat(d->transaction, d->dbi, &stat); | 353 | rc = mdb_stat(d->transaction, d->dbi, &stat); |
341 | if (rc) { | 354 | if (rc) { |
342 | Warning() << "Something went wrong " << QByteArray(mdb_strerror(rc)); | 355 | SinkWarning() << "Something went wrong " << QByteArray(mdb_strerror(rc)); |
343 | } | 356 | } |
344 | // std::cout << "overflow_pages: " << stat.ms_overflow_pages << std::endl; | 357 | // std::cout << "overflow_pages: " << stat.ms_overflow_pages << std::endl; |
345 | // std::cout << "page size: " << stat.ms_psize << std::endl; | 358 | // std::cout << "page size: " << stat.ms_psize << std::endl; |
@@ -398,6 +411,21 @@ Storage::Transaction::Transaction(Transaction::Private *prv) : d(prv) | |||
398 | d->startTransaction(); | 411 | d->startTransaction(); |
399 | } | 412 | } |
400 | 413 | ||
414 | Storage::Transaction::Transaction(Transaction &&other) : d(nullptr) | ||
415 | { | ||
416 | *this = std::move(other); | ||
417 | } | ||
418 | |||
419 | Storage::Transaction &Storage::Transaction::operator=(Storage::Transaction &&other) | ||
420 | { | ||
421 | if (&other != this) { | ||
422 | delete d; | ||
423 | d = other.d; | ||
424 | other.d = nullptr; | ||
425 | } | ||
426 | return *this; | ||
427 | } | ||
428 | |||
401 | Storage::Transaction::~Transaction() | 429 | Storage::Transaction::~Transaction() |
402 | { | 430 | { |
403 | if (d && d->transaction) { | 431 | if (d && d->transaction) { |
@@ -452,7 +480,7 @@ static bool ensureCorrectDb(Storage::NamedDatabase &database, const QByteArray & | |||
452 | bool openedTheWrongDatabase = false; | 480 | bool openedTheWrongDatabase = false; |
453 | auto count = database.scan("__internal_dbname", [db, &openedTheWrongDatabase](const QByteArray &key, const QByteArray &value) ->bool { | 481 | auto count = database.scan("__internal_dbname", [db, &openedTheWrongDatabase](const QByteArray &key, const QByteArray &value) ->bool { |
454 | if (value != db) { | 482 | if (value != db) { |
455 | Warning() << "Opened the wrong database, got " << value << " instead of " << db; | 483 | SinkWarning() << "Opened the wrong database, got " << value << " instead of " << db; |
456 | openedTheWrongDatabase = true; | 484 | openedTheWrongDatabase = true; |
457 | } | 485 | } |
458 | return false; | 486 | return false; |
@@ -475,7 +503,7 @@ bool Storage::Transaction::validateNamedDatabases() | |||
475 | for (const auto &dbName : databases) { | 503 | for (const auto &dbName : databases) { |
476 | auto db = openDatabase(dbName); | 504 | auto db = openDatabase(dbName); |
477 | if (!db) { | 505 | if (!db) { |
478 | Warning() << "Failed to open the database: " << dbName; | 506 | SinkWarning() << "Failed to open the database: " << dbName; |
479 | return false; | 507 | return false; |
480 | } | 508 | } |
481 | } | 509 | } |
@@ -497,7 +525,7 @@ Storage::NamedDatabase Storage::Transaction::openDatabase(const QByteArray &db, | |||
497 | } | 525 | } |
498 | auto database = Storage::NamedDatabase(p); | 526 | auto database = Storage::NamedDatabase(p); |
499 | if (!ensureCorrectDb(database, db, d->requestedRead)) { | 527 | if (!ensureCorrectDb(database, db, d->requestedRead)) { |
500 | Warning() << "Failed to open the database" << db; | 528 | SinkWarning() << "Failed to open the database" << db; |
501 | return Storage::NamedDatabase(); | 529 | return Storage::NamedDatabase(); |
502 | } | 530 | } |
503 | return database; | 531 | return database; |
@@ -506,7 +534,7 @@ Storage::NamedDatabase Storage::Transaction::openDatabase(const QByteArray &db, | |||
506 | QList<QByteArray> Storage::Transaction::getDatabaseNames() const | 534 | QList<QByteArray> Storage::Transaction::getDatabaseNames() const |
507 | { | 535 | { |
508 | if (!d) { | 536 | if (!d) { |
509 | Warning() << "Invalid transaction"; | 537 | SinkWarning() << "Invalid transaction"; |
510 | return QList<QByteArray>(); | 538 | return QList<QByteArray>(); |
511 | } | 539 | } |
512 | 540 | ||
@@ -529,11 +557,12 @@ QList<QByteArray> Storage::Transaction::getDatabaseNames() const | |||
529 | rc = 0; | 557 | rc = 0; |
530 | } | 558 | } |
531 | if (rc) { | 559 | if (rc) { |
532 | Warning() << "Failed to get a value" << rc; | 560 | SinkWarning() << "Failed to get a value" << rc; |
533 | } | 561 | } |
534 | } | 562 | } |
563 | mdb_cursor_close(cursor); | ||
535 | } else { | 564 | } else { |
536 | Warning() << "Failed to open db" << rc << QByteArray(mdb_strerror(rc)); | 565 | SinkWarning() << "Failed to open db" << rc << QByteArray(mdb_strerror(rc)); |
537 | } | 566 | } |
538 | return list; | 567 | return list; |
539 | } | 568 | } |
@@ -580,7 +609,7 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m) : st | |||
580 | int rc = 0; | 609 | int rc = 0; |
581 | if ((rc = mdb_env_create(&env))) { | 610 | if ((rc = mdb_env_create(&env))) { |
582 | // TODO: handle error | 611 | // TODO: handle error |
583 | Warning() << "mdb_env_create: " << rc << " " << mdb_strerror(rc); | 612 | SinkWarning() << "mdb_env_create: " << rc << " " << mdb_strerror(rc); |
584 | } else { | 613 | } else { |
585 | mdb_env_set_maxdbs(env, 50); | 614 | mdb_env_set_maxdbs(env, 50); |
586 | unsigned int flags = MDB_NOTLS; | 615 | unsigned int flags = MDB_NOTLS; |
@@ -588,11 +617,13 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m) : st | |||
588 | flags |= MDB_RDONLY; | 617 | flags |= MDB_RDONLY; |
589 | } | 618 | } |
590 | if ((rc = mdb_env_open(env, fullPath.toStdString().data(), flags, 0664))) { | 619 | if ((rc = mdb_env_open(env, fullPath.toStdString().data(), flags, 0664))) { |
591 | Warning() << "mdb_env_open: " << rc << " " << mdb_strerror(rc); | 620 | SinkWarning() << "mdb_env_open: " << rc << " " << mdb_strerror(rc); |
592 | mdb_env_close(env); | 621 | mdb_env_close(env); |
593 | env = 0; | 622 | env = 0; |
594 | } else { | 623 | } else { |
595 | // FIXME: dynamic resize | 624 | // FIXME: dynamic resize |
625 | // In order to run valgrind this size must be smaller than half your available RAM | ||
626 | // https://github.com/BVLC/caffe/issues/2404 | ||
596 | const size_t dbSize = (size_t)10485760 * (size_t)8000; // 1MB * 8000 | 627 | const size_t dbSize = (size_t)10485760 * (size_t)8000; // 1MB * 8000 |
597 | mdb_env_set_mapsize(env, dbSize); | 628 | mdb_env_set_mapsize(env, dbSize); |
598 | sEnvironments.insert(fullPath, env); | 629 | sEnvironments.insert(fullPath, env); |
@@ -648,7 +679,7 @@ qint64 Storage::diskUsage() const | |||
648 | { | 679 | { |
649 | QFileInfo info(d->storageRoot + '/' + d->name + "/data.mdb"); | 680 | QFileInfo info(d->storageRoot + '/' + d->name + "/data.mdb"); |
650 | if (!info.exists()) { | 681 | if (!info.exists()) { |
651 | Warning() << "Tried to get filesize for non-existant file: " << info.path(); | 682 | SinkWarning() << "Tried to get filesize for non-existant file: " << info.path(); |
652 | } | 683 | } |
653 | return info.size(); | 684 | return info.size(); |
654 | } | 685 | } |
@@ -658,7 +689,7 @@ void Storage::removeFromDisk() const | |||
658 | const QString fullPath(d->storageRoot + '/' + d->name); | 689 | const QString fullPath(d->storageRoot + '/' + d->name); |
659 | QMutexLocker locker(&d->sMutex); | 690 | QMutexLocker locker(&d->sMutex); |
660 | QDir dir(fullPath); | 691 | QDir dir(fullPath); |
661 | Trace() << "Removing database from disk: " << fullPath; | 692 | SinkTrace() << "Removing database from disk: " << fullPath; |
662 | if (!dir.removeRecursively()) { | 693 | if (!dir.removeRecursively()) { |
663 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, QString("Failed to remove directory %1 %2").arg(d->storageRoot).arg(d->name).toLatin1()); | 694 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, QString("Failed to remove directory %1 %2").arg(d->storageRoot).arg(d->name).toLatin1()); |
664 | defaultErrorHandler()(error); | 695 | defaultErrorHandler()(error); |