summaryrefslogtreecommitdiffstats
path: root/common/storage_lmdb.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'common/storage_lmdb.cpp')
-rw-r--r--common/storage_lmdb.cpp132
1 files changed, 49 insertions, 83 deletions
diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp
index 2d8b187..878a5d9 100644
--- a/common/storage_lmdb.cpp
+++ b/common/storage_lmdb.cpp
@@ -56,17 +56,12 @@ class Storage::NamedDatabase::Private
56{ 56{
57public: 57public:
58 Private(const QByteArray &_db, bool _allowDuplicates, const std::function<void(const Storage::Error &error)> &_defaultErrorHandler, const QString &_name, MDB_txn *_txn) 58 Private(const QByteArray &_db, bool _allowDuplicates, const std::function<void(const Storage::Error &error)> &_defaultErrorHandler, const QString &_name, MDB_txn *_txn)
59 : db(_db), 59 : db(_db), transaction(_txn), allowDuplicates(_allowDuplicates), defaultErrorHandler(_defaultErrorHandler), name(_name)
60 transaction(_txn),
61 allowDuplicates(_allowDuplicates),
62 defaultErrorHandler(_defaultErrorHandler),
63 name(_name)
64 { 60 {
65 } 61 }
66 62
67 ~Private() 63 ~Private()
68 { 64 {
69
70 } 65 }
71 66
72 QByteArray db; 67 QByteArray db;
@@ -88,7 +83,7 @@ public:
88 if (const int rc = mdb_dbi_open(transaction, db.constData(), flags, &dbi)) { 83 if (const int rc = mdb_dbi_open(transaction, db.constData(), flags, &dbi)) {
89 dbi = 0; 84 dbi = 0;
90 transaction = 0; 85 transaction = 0;
91 //The database is not existing, ignore in read-only mode 86 // The database is not existing, ignore in read-only mode
92 if (!(readOnly && rc == MDB_NOTFOUND)) { 87 if (!(readOnly && rc == MDB_NOTFOUND)) {
93 Error error(name.toLatin1(), ErrorCodes::GenericError, "Error while opening database: " + QByteArray(mdb_strerror(rc))); 88 Error error(name.toLatin1(), ErrorCodes::GenericError, "Error while opening database: " + QByteArray(mdb_strerror(rc)));
94 errorHandler ? errorHandler(error) : defaultErrorHandler(error); 89 errorHandler ? errorHandler(error) : defaultErrorHandler(error);
@@ -99,14 +94,11 @@ public:
99 } 94 }
100}; 95};
101 96
102Storage::NamedDatabase::NamedDatabase() 97Storage::NamedDatabase::NamedDatabase() : d(nullptr)
103 : d(nullptr)
104{ 98{
105
106} 99}
107 100
108Storage::NamedDatabase::NamedDatabase(NamedDatabase::Private *prv) 101Storage::NamedDatabase::NamedDatabase(NamedDatabase::Private *prv) : d(prv)
109 : d(prv)
110{ 102{
111} 103}
112 104
@@ -138,9 +130,9 @@ bool Storage::NamedDatabase::write(const QByteArray &sKey, const QByteArray &sVa
138 int rc; 130 int rc;
139 MDB_val key, data; 131 MDB_val key, data;
140 key.mv_size = keySize; 132 key.mv_size = keySize;
141 key.mv_data = const_cast<void*>(keyPtr); 133 key.mv_data = const_cast<void *>(keyPtr);
142 data.mv_size = valueSize; 134 data.mv_size = valueSize;
143 data.mv_data = const_cast<void*>(valuePtr); 135 data.mv_data = const_cast<void *>(valuePtr);
144 rc = mdb_put(d->transaction, d->dbi, &key, &data, 0); 136 rc = mdb_put(d->transaction, d->dbi, &key, &data, 0);
145 137
146 if (rc) { 138 if (rc) {
@@ -151,14 +143,12 @@ bool Storage::NamedDatabase::write(const QByteArray &sKey, const QByteArray &sVa
151 return !rc; 143 return !rc;
152} 144}
153 145
154void Storage::NamedDatabase::remove(const QByteArray &k, 146void Storage::NamedDatabase::remove(const QByteArray &k, const std::function<void(const Storage::Error &error)> &errorHandler)
155 const std::function<void(const Storage::Error &error)> &errorHandler)
156{ 147{
157 remove(k, QByteArray(), errorHandler); 148 remove(k, QByteArray(), errorHandler);
158} 149}
159 150
160void Storage::NamedDatabase::remove(const QByteArray &k, const QByteArray &value, 151void Storage::NamedDatabase::remove(const QByteArray &k, const QByteArray &value, const std::function<void(const Storage::Error &error)> &errorHandler)
161 const std::function<void(const Storage::Error &error)> &errorHandler)
162{ 152{
163 if (!d || !d->transaction) { 153 if (!d || !d->transaction) {
164 if (d) { 154 if (d) {
@@ -171,13 +161,13 @@ void Storage::NamedDatabase::remove(const QByteArray &k, const QByteArray &value
171 int rc; 161 int rc;
172 MDB_val key; 162 MDB_val key;
173 key.mv_size = k.size(); 163 key.mv_size = k.size();
174 key.mv_data = const_cast<void*>(static_cast<const void*>(k.data())); 164 key.mv_data = const_cast<void *>(static_cast<const void *>(k.data()));
175 if (value.isEmpty()) { 165 if (value.isEmpty()) {
176 rc = mdb_del(d->transaction, d->dbi, &key, 0); 166 rc = mdb_del(d->transaction, d->dbi, &key, 0);
177 } else { 167 } else {
178 MDB_val data; 168 MDB_val data;
179 data.mv_size = value.size(); 169 data.mv_size = value.size();
180 data.mv_data = const_cast<void*>(static_cast<const void*>(value.data())); 170 data.mv_data = const_cast<void *>(static_cast<const void *>(value.data()));
181 rc = mdb_del(d->transaction, d->dbi, &key, &data); 171 rc = mdb_del(d->transaction, d->dbi, &key, &data);
182 } 172 }
183 173
@@ -187,13 +177,11 @@ void Storage::NamedDatabase::remove(const QByteArray &k, const QByteArray &value
187 } 177 }
188} 178}
189 179
190int Storage::NamedDatabase::scan(const QByteArray &k, 180int Storage::NamedDatabase::scan(const QByteArray &k, const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler,
191 const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler, 181 const std::function<void(const Storage::Error &error)> &errorHandler, bool findSubstringKeys) const
192 const std::function<void(const Storage::Error &error)> &errorHandler,
193 bool findSubstringKeys) const
194{ 182{
195 if (!d || !d->transaction) { 183 if (!d || !d->transaction) {
196 //Not an error. We rely on this to read nothing from non-existing databases. 184 // Not an error. We rely on this to read nothing from non-existing databases.
197 return 0; 185 return 0;
198 } 186 }
199 187
@@ -202,7 +190,7 @@ int Storage::NamedDatabase::scan(const QByteArray &k,
202 MDB_val data; 190 MDB_val data;
203 MDB_cursor *cursor; 191 MDB_cursor *cursor;
204 192
205 key.mv_data = (void*)k.constData(); 193 key.mv_data = (void *)k.constData();
206 key.mv_size = k.size(); 194 key.mv_size = k.size();
207 195
208 rc = mdb_cursor_open(d->transaction, d->dbi, &cursor); 196 rc = mdb_cursor_open(d->transaction, d->dbi, &cursor);
@@ -220,21 +208,21 @@ int Storage::NamedDatabase::scan(const QByteArray &k,
220 op = MDB_SET_RANGE; 208 op = MDB_SET_RANGE;
221 } 209 }
222 if ((rc = mdb_cursor_get(cursor, &key, &data, op)) == 0) { 210 if ((rc = mdb_cursor_get(cursor, &key, &data, op)) == 0) {
223 //The first lookup will find a key that is equal or greather than our key 211 // The first lookup will find a key that is equal or greather than our key
224 if (QByteArray::fromRawData((char*)key.mv_data, key.mv_size).startsWith(k)) { 212 if (QByteArray::fromRawData((char *)key.mv_data, key.mv_size).startsWith(k)) {
225 numberOfRetrievedValues++; 213 numberOfRetrievedValues++;
226 if (resultHandler(QByteArray::fromRawData((char*)key.mv_data, key.mv_size), QByteArray::fromRawData((char*)data.mv_data, data.mv_size))) { 214 if (resultHandler(QByteArray::fromRawData((char *)key.mv_data, key.mv_size), QByteArray::fromRawData((char *)data.mv_data, data.mv_size))) {
227 if (findSubstringKeys) { 215 if (findSubstringKeys) {
228 //Reset the key to what we search for 216 // Reset the key to what we search for
229 key.mv_data = (void*)k.constData(); 217 key.mv_data = (void *)k.constData();
230 key.mv_size = k.size(); 218 key.mv_size = k.size();
231 } 219 }
232 MDB_cursor_op nextOp = (d->allowDuplicates && !findSubstringKeys) ? MDB_NEXT_DUP : MDB_NEXT; 220 MDB_cursor_op nextOp = (d->allowDuplicates && !findSubstringKeys) ? MDB_NEXT_DUP : MDB_NEXT;
233 while ((rc = mdb_cursor_get(cursor, &key, &data, nextOp)) == 0) { 221 while ((rc = mdb_cursor_get(cursor, &key, &data, nextOp)) == 0) {
234 //Every consequent lookup simply iterates through the list 222 // Every consequent lookup simply iterates through the list
235 if (QByteArray::fromRawData((char*)key.mv_data, key.mv_size).startsWith(k)) { 223 if (QByteArray::fromRawData((char *)key.mv_data, key.mv_size).startsWith(k)) {
236 numberOfRetrievedValues++; 224 numberOfRetrievedValues++;
237 if (!resultHandler(QByteArray::fromRawData((char*)key.mv_data, key.mv_size), QByteArray::fromRawData((char*)data.mv_data, data.mv_size))) { 225 if (!resultHandler(QByteArray::fromRawData((char *)key.mv_data, key.mv_size), QByteArray::fromRawData((char *)data.mv_data, data.mv_size))) {
238 break; 226 break;
239 } 227 }
240 } 228 }
@@ -243,14 +231,14 @@ int Storage::NamedDatabase::scan(const QByteArray &k,
243 } 231 }
244 } 232 }
245 233
246 //We never find the last value 234 // We never find the last value
247 if (rc == MDB_NOTFOUND) { 235 if (rc == MDB_NOTFOUND) {
248 rc = 0; 236 rc = 0;
249 } 237 }
250 } else { 238 } else {
251 if ((rc = mdb_cursor_get(cursor, &key, &data, MDB_SET)) == 0) { 239 if ((rc = mdb_cursor_get(cursor, &key, &data, MDB_SET)) == 0) {
252 numberOfRetrievedValues++; 240 numberOfRetrievedValues++;
253 resultHandler(QByteArray::fromRawData((char*)key.mv_data, key.mv_size), QByteArray::fromRawData((char*)data.mv_data, data.mv_size)); 241 resultHandler(QByteArray::fromRawData((char *)key.mv_data, key.mv_size), QByteArray::fromRawData((char *)data.mv_data, data.mv_size));
254 } 242 }
255 } 243 }
256 244
@@ -264,12 +252,11 @@ int Storage::NamedDatabase::scan(const QByteArray &k,
264 return numberOfRetrievedValues; 252 return numberOfRetrievedValues;
265} 253}
266 254
267void Storage::NamedDatabase::findLatest(const QByteArray &k, 255void Storage::NamedDatabase::findLatest(const QByteArray &k, const std::function<void(const QByteArray &key, const QByteArray &value)> &resultHandler,
268 const std::function<void(const QByteArray &key, const QByteArray &value)> &resultHandler, 256 const std::function<void(const Storage::Error &error)> &errorHandler) const
269 const std::function<void(const Storage::Error &error)> &errorHandler) const
270{ 257{
271 if (!d || !d->transaction) { 258 if (!d || !d->transaction) {
272 //Not an error. We rely on this to read nothing from non-existing databases. 259 // Not an error. We rely on this to read nothing from non-existing databases.
273 return; 260 return;
274 } 261 }
275 262
@@ -278,7 +265,7 @@ void Storage::NamedDatabase::findLatest(const QByteArray &k,
278 MDB_val data; 265 MDB_val data;
279 MDB_cursor *cursor; 266 MDB_cursor *cursor;
280 267
281 key.mv_data = (void*)k.constData(); 268 key.mv_data = (void *)k.constData();
282 key.mv_size = k.size(); 269 key.mv_size = k.size();
283 270
284 rc = mdb_cursor_open(d->transaction, d->dbi, &cursor); 271 rc = mdb_cursor_open(d->transaction, d->dbi, &cursor);
@@ -290,10 +277,10 @@ void Storage::NamedDatabase::findLatest(const QByteArray &k,
290 277
291 MDB_cursor_op op = MDB_SET_RANGE; 278 MDB_cursor_op op = MDB_SET_RANGE;
292 if ((rc = mdb_cursor_get(cursor, &key, &data, op)) == 0) { 279 if ((rc = mdb_cursor_get(cursor, &key, &data, op)) == 0) {
293 //The first lookup will find a key that is equal or greather than our key 280 // The first lookup will find a key that is equal or greather than our key
294 if (QByteArray::fromRawData((char*)key.mv_data, key.mv_size).startsWith(k)) { 281 if (QByteArray::fromRawData((char *)key.mv_data, key.mv_size).startsWith(k)) {
295 bool advanced = false; 282 bool advanced = false;
296 while (QByteArray::fromRawData((char*)key.mv_data, key.mv_size).startsWith(k)) { 283 while (QByteArray::fromRawData((char *)key.mv_data, key.mv_size).startsWith(k)) {
297 advanced = true; 284 advanced = true;
298 MDB_cursor_op nextOp = MDB_NEXT; 285 MDB_cursor_op nextOp = MDB_NEXT;
299 rc = mdb_cursor_get(cursor, &key, &data, nextOp); 286 rc = mdb_cursor_get(cursor, &key, &data, nextOp);
@@ -303,17 +290,17 @@ void Storage::NamedDatabase::findLatest(const QByteArray &k,
303 } 290 }
304 if (advanced) { 291 if (advanced) {
305 MDB_cursor_op prefOp = MDB_PREV; 292 MDB_cursor_op prefOp = MDB_PREV;
306 //We read past the end above, just take the last value 293 // We read past the end above, just take the last value
307 if (rc == MDB_NOTFOUND) { 294 if (rc == MDB_NOTFOUND) {
308 prefOp = MDB_LAST; 295 prefOp = MDB_LAST;
309 } 296 }
310 rc = mdb_cursor_get(cursor, &key, &data, prefOp); 297 rc = mdb_cursor_get(cursor, &key, &data, prefOp);
311 resultHandler(QByteArray::fromRawData((char*)key.mv_data, key.mv_size), QByteArray::fromRawData((char*)data.mv_data, data.mv_size)); 298 resultHandler(QByteArray::fromRawData((char *)key.mv_data, key.mv_size), QByteArray::fromRawData((char *)data.mv_data, data.mv_size));
312 } 299 }
313 } 300 }
314 } 301 }
315 302
316 //We never find the last value 303 // We never find the last value
317 if (rc == MDB_NOTFOUND) { 304 if (rc == MDB_NOTFOUND) {
318 rc = 0; 305 rc = 0;
319 } 306 }
@@ -350,25 +337,15 @@ qint64 Storage::NamedDatabase::getSize()
350} 337}
351 338
352 339
353
354
355class Storage::Transaction::Private 340class Storage::Transaction::Private
356{ 341{
357public: 342public:
358 Private(bool _requestRead, const std::function<void(const Storage::Error &error)> &_defaultErrorHandler, const QString &_name, MDB_env *_env) 343 Private(bool _requestRead, const std::function<void(const Storage::Error &error)> &_defaultErrorHandler, const QString &_name, MDB_env *_env)
359 : env(_env), 344 : env(_env), requestedRead(_requestRead), defaultErrorHandler(_defaultErrorHandler), name(_name), implicitCommit(false), error(false), modificationCounter(0)
360 requestedRead(_requestRead),
361 defaultErrorHandler(_defaultErrorHandler),
362 name(_name),
363 implicitCommit(false),
364 error(false),
365 modificationCounter(0)
366 { 345 {
367
368 } 346 }
369 ~Private() 347 ~Private()
370 { 348 {
371
372 } 349 }
373 350
374 MDB_env *env; 351 MDB_env *env;
@@ -391,14 +368,11 @@ public:
391 } 368 }
392}; 369};
393 370
394Storage::Transaction::Transaction() 371Storage::Transaction::Transaction() : d(nullptr)
395 : d(nullptr)
396{ 372{
397
398} 373}
399 374
400Storage::Transaction::Transaction(Transaction::Private *prv) 375Storage::Transaction::Transaction(Transaction::Private *prv) : d(prv)
401 : d(prv)
402{ 376{
403 d->startTransaction(); 377 d->startTransaction();
404} 378}
@@ -449,7 +423,7 @@ Storage::NamedDatabase Storage::Transaction::openDatabase(const QByteArray &db,
449 if (!d) { 423 if (!d) {
450 return Storage::NamedDatabase(); 424 return Storage::NamedDatabase();
451 } 425 }
452 //We don't now if anything changed 426 // We don't now if anything changed
453 d->implicitCommit = true; 427 d->implicitCommit = true;
454 auto p = new Storage::NamedDatabase::Private(db, allowDuplicates, d->defaultErrorHandler, d->name, d->transaction); 428 auto p = new Storage::NamedDatabase::Private(db, allowDuplicates, d->defaultErrorHandler, d->name, d->transaction);
455 if (!p->openDatabase(d->requestedRead, errorHandler)) { 429 if (!p->openDatabase(d->requestedRead, errorHandler)) {
@@ -475,9 +449,9 @@ QList<QByteArray> Storage::Transaction::getDatabaseNames() const
475 449
476 mdb_cursor_open(d->transaction, d->dbi, &cursor); 450 mdb_cursor_open(d->transaction, d->dbi, &cursor);
477 if ((rc = mdb_cursor_get(cursor, &key, &data, MDB_FIRST)) == 0) { 451 if ((rc = mdb_cursor_get(cursor, &key, &data, MDB_FIRST)) == 0) {
478 list << QByteArray::fromRawData((char*)key.mv_data, key.mv_size); 452 list << QByteArray::fromRawData((char *)key.mv_data, key.mv_size);
479 while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { 453 while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
480 list << QByteArray::fromRawData((char*)key.mv_data, key.mv_size); 454 list << QByteArray::fromRawData((char *)key.mv_data, key.mv_size);
481 } 455 }
482 } else { 456 } else {
483 Warning() << "Failed to get a value" << rc; 457 Warning() << "Failed to get a value" << rc;
@@ -489,9 +463,6 @@ QList<QByteArray> Storage::Transaction::getDatabaseNames() const
489} 463}
490 464
491 465
492
493
494
495class Storage::Private 466class Storage::Private
496{ 467{
497public: 468public:
@@ -504,17 +475,13 @@ public:
504 MDB_env *env; 475 MDB_env *env;
505 AccessMode mode; 476 AccessMode mode;
506 static QMutex sMutex; 477 static QMutex sMutex;
507 static QHash<QString, MDB_env*> sEnvironments; 478 static QHash<QString, MDB_env *> sEnvironments;
508}; 479};
509 480
510QMutex Storage::Private::sMutex; 481QMutex Storage::Private::sMutex;
511QHash<QString, MDB_env*> Storage::Private::sEnvironments; 482QHash<QString, MDB_env *> Storage::Private::sEnvironments;
512 483
513Storage::Private::Private(const QString &s, const QString &n, AccessMode m) 484Storage::Private::Private(const QString &s, const QString &n, AccessMode m) : storageRoot(s), name(n), env(0), mode(m)
514 : storageRoot(s),
515 name(n),
516 env(0),
517 mode(m)
518{ 485{
519 const QString fullPath(storageRoot + '/' + name); 486 const QString fullPath(storageRoot + '/' + name);
520 QFileInfo dirInfo(fullPath); 487 QFileInfo dirInfo(fullPath);
@@ -525,11 +492,11 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m)
525 if (mode == ReadWrite && !dirInfo.permission(QFile::WriteOwner)) { 492 if (mode == ReadWrite && !dirInfo.permission(QFile::WriteOwner)) {
526 qCritical() << fullPath << "does not have write permissions. Aborting"; 493 qCritical() << fullPath << "does not have write permissions. Aborting";
527 } else if (dirInfo.exists()) { 494 } else if (dirInfo.exists()) {
528 //Ensure the environment is only created once 495 // Ensure the environment is only created once
529 QMutexLocker locker(&sMutex); 496 QMutexLocker locker(&sMutex);
530 497
531 /* 498 /*
532 * It seems we can only ever have one environment open in the process. 499 * It seems we can only ever have one environment open in the process.
533 * Otherwise multi-threading breaks. 500 * Otherwise multi-threading breaks.
534 */ 501 */
535 env = sEnvironments.value(fullPath); 502 env = sEnvironments.value(fullPath);
@@ -549,8 +516,8 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m)
549 mdb_env_close(env); 516 mdb_env_close(env);
550 env = 0; 517 env = 0;
551 } else { 518 } else {
552 //FIXME: dynamic resize 519 // FIXME: dynamic resize
553 const size_t dbSize = (size_t)10485760 * (size_t)8000; //1MB * 8000 520 const size_t dbSize = (size_t)10485760 * (size_t)8000; // 1MB * 8000
554 mdb_env_set_mapsize(env, dbSize); 521 mdb_env_set_mapsize(env, dbSize);
555 sEnvironments.insert(fullPath, env); 522 sEnvironments.insert(fullPath, env);
556 } 523 }
@@ -561,7 +528,7 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m)
561 528
562Storage::Private::~Private() 529Storage::Private::~Private()
563{ 530{
564 //Since we can have only one environment open per process, we currently leak the environments. 531 // Since we can have only one environment open per process, we currently leak the environments.
565 // if (env) { 532 // if (env) {
566 // //mdb_dbi_close should not be necessary and is potentially dangerous (see docs) 533 // //mdb_dbi_close should not be necessary and is potentially dangerous (see docs)
567 // mdb_dbi_close(env, dbi); 534 // mdb_dbi_close(env, dbi);
@@ -569,8 +536,7 @@ Storage::Private::~Private()
569 // } 536 // }
570} 537}
571 538
572Storage::Storage(const QString &storageRoot, const QString &name, AccessMode mode) 539Storage::Storage(const QString &storageRoot, const QString &name, AccessMode mode) : d(new Private(storageRoot, name, mode))
573 : d(new Private(storageRoot, name, mode))
574{ 540{
575} 541}
576 542