diff options
-rw-r--r-- | common/storage.h | 61 | ||||
-rw-r--r-- | common/storage_lmdb.cpp | 297 | ||||
-rw-r--r-- | tests/indextest.cpp | 21 | ||||
-rw-r--r-- | tests/storagebenchmark.cpp | 8 | ||||
-rw-r--r-- | tests/storagetest.cpp | 114 |
5 files changed, 362 insertions, 139 deletions
diff --git a/common/storage.h b/common/storage.h index a7241a7..2f7a2df 100644 --- a/common/storage.h +++ b/common/storage.h | |||
@@ -51,16 +51,12 @@ public: | |||
51 | int code; | 51 | int code; |
52 | }; | 52 | }; |
53 | 53 | ||
54 | class Transaction | 54 | class Transaction; |
55 | class NamedDatabase | ||
55 | { | 56 | { |
56 | public: | 57 | public: |
57 | Transaction(); | 58 | NamedDatabase(); |
58 | ~Transaction(); | 59 | ~NamedDatabase(); |
59 | bool commit(const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()); | ||
60 | void abort(); | ||
61 | |||
62 | void setAutocommit(int interval); | ||
63 | |||
64 | /** | 60 | /** |
65 | * Write a value | 61 | * Write a value |
66 | */ | 62 | */ |
@@ -73,22 +69,57 @@ public: | |||
73 | const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()); | 69 | const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()); |
74 | /** | 70 | /** |
75 | * Read values with a given key. | 71 | * Read values with a given key. |
76 | * | 72 | * |
77 | * * An empty @param key results in a full scan | 73 | * * An empty @param key results in a full scan |
78 | * * If duplicates are existing (revisions), all values are returned. | 74 | * * If duplicates are existing (revisions), all values are returned. |
79 | * * The pointers of the returned values are valid during the execution of the @param resultHandler | 75 | * * The pointers of the returned values are valid during the execution of the @param resultHandler |
80 | * | 76 | * |
81 | * @return The number of values retrieved. | 77 | * @return The number of values retrieved. |
82 | */ | 78 | */ |
83 | int scan(const QByteArray &k, | 79 | int scan(const QByteArray &k, |
84 | const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler, | 80 | const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler, |
85 | const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()) const; | 81 | const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()) const; |
86 | 82 | ||
83 | NamedDatabase(NamedDatabase&& other) : d(other.d) | ||
84 | { | ||
85 | d = other.d; | ||
86 | other.d = nullptr; | ||
87 | } | ||
88 | |||
89 | NamedDatabase& operator=(NamedDatabase&& other) { | ||
90 | d = other.d; | ||
91 | other.d = nullptr; | ||
92 | return *this; | ||
93 | } | ||
94 | |||
95 | operator bool() const { | ||
96 | return (d != nullptr); | ||
97 | } | ||
98 | |||
99 | private: | ||
100 | friend Transaction; | ||
101 | NamedDatabase(NamedDatabase& other); | ||
102 | NamedDatabase& operator=(NamedDatabase& other); | ||
103 | class Private; | ||
104 | NamedDatabase(Private*); | ||
105 | Private *d; | ||
106 | }; | ||
107 | |||
108 | class Transaction | ||
109 | { | ||
110 | public: | ||
111 | Transaction(); | ||
112 | ~Transaction(); | ||
113 | bool commit(const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()); | ||
114 | void abort(); | ||
115 | |||
116 | NamedDatabase openDatabase(const QByteArray &name = QByteArray("default"), const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()) const; | ||
117 | |||
87 | Transaction(Transaction&& other) : d(other.d) | 118 | Transaction(Transaction&& other) : d(other.d) |
88 | { | 119 | { |
89 | d = other.d; | 120 | d = other.d; |
90 | other.d = nullptr; | 121 | other.d = nullptr; |
91 | } | 122 | } |
92 | Transaction& operator=(Transaction&& other) { | 123 | Transaction& operator=(Transaction&& other) { |
93 | d = other.d; | 124 | d = other.d; |
94 | other.d = nullptr; | 125 | other.d = nullptr; |
@@ -98,6 +129,14 @@ public: | |||
98 | operator bool() const { | 129 | operator bool() const { |
99 | return (d != nullptr); | 130 | return (d != nullptr); |
100 | } | 131 | } |
132 | |||
133 | bool write(const QByteArray &key, const QByteArray &value, const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()); | ||
134 | |||
135 | void remove(const QByteArray &key, | ||
136 | const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()); | ||
137 | int scan(const QByteArray &k, | ||
138 | const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler, | ||
139 | const std::function<void(const Storage::Error &error)> &errorHandler = std::function<void(const Storage::Error &error)>()) const; | ||
101 | private: | 140 | private: |
102 | Transaction(Transaction& other); | 141 | Transaction(Transaction& other); |
103 | Transaction& operator=(Transaction& other); | 142 | Transaction& operator=(Transaction& other); |
diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index 0618d61..ebb3be3 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp | |||
@@ -47,120 +47,69 @@ int getErrorCode(int e) | |||
47 | return -1; | 47 | return -1; |
48 | } | 48 | } |
49 | 49 | ||
50 | 50 | class Storage::NamedDatabase::Private | |
51 | |||
52 | class Storage::Transaction::Private | ||
53 | { | 51 | { |
54 | public: | 52 | public: |
55 | Private(bool _requestRead, bool _allowDuplicates, const std::function<void(const Storage::Error &error)> &_defaultErrorHandler, const QString &_name, MDB_env *_env) | 53 | Private(const QByteArray &_db, bool _allowDuplicates, const std::function<void(const Storage::Error &error)> &_defaultErrorHandler, const QString &_name, MDB_txn *_txn) |
56 | : env(_env), | 54 | : db(_db), |
57 | requestedRead(_requestRead), | 55 | transaction(_txn), |
58 | allowDuplicates(_allowDuplicates), | 56 | allowDuplicates(_allowDuplicates), |
59 | defaultErrorHandler(_defaultErrorHandler), | 57 | defaultErrorHandler(_defaultErrorHandler), |
60 | name(_name), | 58 | name(_name) |
61 | implicitCommit(false), | ||
62 | error(false), | ||
63 | autoCommitInterval(0), | ||
64 | modificationCounter(0) | ||
65 | { | 59 | { |
66 | |||
67 | } | 60 | } |
61 | |||
68 | ~Private() | 62 | ~Private() |
69 | { | 63 | { |
70 | 64 | ||
71 | } | 65 | } |
72 | 66 | ||
73 | MDB_env *env; | 67 | QByteArray db; |
74 | MDB_txn *transaction; | 68 | MDB_txn *transaction; |
75 | MDB_dbi dbi; | 69 | MDB_dbi dbi; |
76 | bool requestedRead; | ||
77 | bool allowDuplicates; | 70 | bool allowDuplicates; |
78 | std::function<void(const Storage::Error &error)> defaultErrorHandler; | 71 | std::function<void(const Storage::Error &error)> defaultErrorHandler; |
79 | QString name; | 72 | QString name; |
80 | bool implicitCommit; | ||
81 | bool error; | ||
82 | int autoCommitInterval; | ||
83 | int modificationCounter; | ||
84 | 73 | ||
85 | void startTransaction() | 74 | bool openDatabase(std::function<void(const Storage::Error &error)> errorHandler) |
86 | { | 75 | { |
87 | const int rc = mdb_txn_begin(env, NULL, requestedRead ? MDB_RDONLY : 0, &transaction); | 76 | unsigned int flags = MDB_CREATE; |
88 | if (rc) { | 77 | if (allowDuplicates) { |
89 | defaultErrorHandler(Error(name.toLatin1(), ErrorCodes::GenericError, "Error while opening transaction: " + QByteArray(mdb_strerror(rc)))); | 78 | flags |= MDB_DUPSORT; |
90 | } | 79 | } |
91 | } | 80 | if (const int rc = mdb_dbi_open(transaction, db.constData(), flags, &dbi)) { |
92 | 81 | qWarning() << "Failed to open: " << rc << db; | |
93 | void openDatabase() | 82 | dbi = 0; |
94 | { | 83 | transaction = 0; |
95 | const int rc = mdb_dbi_open(transaction, NULL, allowDuplicates ? MDB_DUPSORT : 0, &dbi); | 84 | Error error(name.toLatin1(), ErrorCodes::GenericError, "Error while opening database: " + QByteArray(mdb_strerror(rc))); |
96 | if (rc) { | 85 | errorHandler ? errorHandler(error) : defaultErrorHandler(error); |
97 | defaultErrorHandler(Error(name.toLatin1(), ErrorCodes::GenericError, "Error while opening database: " + QByteArray(mdb_strerror(rc)))); | 86 | return false; |
98 | } | 87 | } |
88 | return true; | ||
99 | } | 89 | } |
100 | }; | 90 | }; |
101 | 91 | ||
102 | Storage::Transaction::Transaction() | 92 | Storage::NamedDatabase::NamedDatabase() |
103 | : d(0) | 93 | : d(nullptr) |
104 | { | 94 | { |
105 | 95 | ||
106 | } | 96 | } |
107 | 97 | ||
108 | Storage::Transaction::Transaction(Transaction::Private *prv) | 98 | Storage::NamedDatabase::NamedDatabase(NamedDatabase::Private *prv) |
109 | : d(prv) | 99 | : d(prv) |
110 | { | 100 | { |
111 | d->startTransaction(); | ||
112 | d->openDatabase(); | ||
113 | } | 101 | } |
114 | 102 | ||
115 | Storage::Transaction::~Transaction() | 103 | Storage::NamedDatabase::~NamedDatabase() |
116 | { | 104 | { |
117 | if (d && d->transaction) { | ||
118 | if (d->implicitCommit && !d->error) { | ||
119 | commit(); | ||
120 | } else { | ||
121 | mdb_txn_abort(d->transaction); | ||
122 | } | ||
123 | } | ||
124 | delete d; | 105 | delete d; |
125 | } | 106 | } |
126 | 107 | ||
127 | bool Storage::Transaction::commit(const std::function<void(const Storage::Error &error)> &errorHandler) | 108 | bool Storage::NamedDatabase::write(const QByteArray &sKey, const QByteArray &sValue, const std::function<void(const Storage::Error &error)> &errorHandler) |
128 | { | ||
129 | if (!d) { | ||
130 | return false; | ||
131 | } | ||
132 | |||
133 | const int rc = mdb_txn_commit(d->transaction); | ||
134 | if (rc) { | ||
135 | mdb_txn_abort(d->transaction); | ||
136 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, "Error during transaction commit: " + QByteArray(mdb_strerror(rc))); | ||
137 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); | ||
138 | } | ||
139 | d->transaction = nullptr; | ||
140 | |||
141 | return !rc; | ||
142 | } | ||
143 | |||
144 | void Storage::Transaction::abort() | ||
145 | { | ||
146 | if (!d || !d->transaction) { | ||
147 | return; | ||
148 | } | ||
149 | |||
150 | mdb_txn_abort(d->transaction); | ||
151 | d->transaction = nullptr; | ||
152 | } | ||
153 | |||
154 | void Storage::Transaction::setAutocommit(int interval) | ||
155 | { | ||
156 | if (d) { | ||
157 | d->autoCommitInterval = interval; | ||
158 | } | ||
159 | } | ||
160 | |||
161 | bool Storage::Transaction::write(const QByteArray &sKey, const QByteArray &sValue, const std::function<void(const Storage::Error &error)> &errorHandler) | ||
162 | { | 109 | { |
163 | if (!d || !d->transaction) { | 110 | if (!d || !d->transaction) { |
111 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, "Not open"); | ||
112 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); | ||
164 | return false; | 113 | return false; |
165 | } | 114 | } |
166 | const void *keyPtr = sKey.data(); | 115 | const void *keyPtr = sKey.data(); |
@@ -183,33 +132,43 @@ bool Storage::Transaction::write(const QByteArray &sKey, const QByteArray &sValu | |||
183 | rc = mdb_put(d->transaction, d->dbi, &key, &data, 0); | 132 | rc = mdb_put(d->transaction, d->dbi, &key, &data, 0); |
184 | 133 | ||
185 | if (rc) { | 134 | if (rc) { |
186 | d->error = true; | ||
187 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, "mdb_put: " + QByteArray(mdb_strerror(rc))); | 135 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, "mdb_put: " + QByteArray(mdb_strerror(rc))); |
188 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); | 136 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); |
189 | } else { | ||
190 | d->implicitCommit = true; | ||
191 | } | 137 | } |
192 | 138 | ||
193 | if (d->autoCommitInterval > 0) { | 139 | return !rc; |
194 | d->modificationCounter++; | 140 | } |
195 | if (d->modificationCounter >= d->autoCommitInterval) { | 141 | |
196 | commit(); | 142 | void Storage::NamedDatabase::remove(const QByteArray &k, |
197 | d->startTransaction(); | 143 | const std::function<void(const Storage::Error &error)> &errorHandler) |
198 | d->openDatabase(); | 144 | { |
199 | d->modificationCounter = 0; | 145 | if (!d || !d->transaction) { |
200 | } | 146 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, "Not open"); |
147 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); | ||
148 | return; | ||
201 | } | 149 | } |
202 | 150 | ||
203 | return !rc; | 151 | int rc; |
152 | MDB_val key; | ||
153 | key.mv_size = k.size(); | ||
154 | key.mv_data = const_cast<void*>(static_cast<const void*>(k.data())); | ||
155 | rc = mdb_del(d->transaction, d->dbi, &key, 0); | ||
156 | |||
157 | if (rc) { | ||
158 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, QString("Error on mdb_del: %1 %2").arg(rc).arg(mdb_strerror(rc)).toLatin1()); | ||
159 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); | ||
160 | } | ||
161 | |||
162 | return; | ||
204 | } | 163 | } |
205 | 164 | ||
206 | int Storage::Transaction::scan(const QByteArray &k, | 165 | int Storage::NamedDatabase::scan(const QByteArray &k, |
207 | const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler, | 166 | const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler, |
208 | const std::function<void(const Storage::Error &error)> &errorHandler) const | 167 | const std::function<void(const Storage::Error &error)> &errorHandler) const |
209 | { | 168 | { |
210 | if (!d || !d->transaction) { | 169 | if (!d || !d->transaction) { |
211 | Error error(d->name.toLatin1(), ErrorCodes::NotOpen, "Not open"); | 170 | // Error error(d->name.toLatin1(), ErrorCodes::NotOpen, "Not open"); |
212 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); | 171 | // errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); |
213 | return 0; | 172 | return 0; |
214 | } | 173 | } |
215 | 174 | ||
@@ -231,7 +190,7 @@ int Storage::Transaction::scan(const QByteArray &k, | |||
231 | int numberOfRetrievedValues = 0; | 190 | int numberOfRetrievedValues = 0; |
232 | 191 | ||
233 | if (k.isEmpty() || d->allowDuplicates) { | 192 | if (k.isEmpty() || d->allowDuplicates) { |
234 | if ((rc = mdb_cursor_get(cursor, &key, &data, d->allowDuplicates ? MDB_SET_RANGE : MDB_FIRST)) == 0) { | 193 | if ((rc = mdb_cursor_get(cursor, &key, &data, d->allowDuplicates ? MDB_SET : MDB_FIRST)) == 0) { |
235 | numberOfRetrievedValues++; | 194 | numberOfRetrievedValues++; |
236 | if (resultHandler(QByteArray::fromRawData((char*)key.mv_data, key.mv_size), QByteArray::fromRawData((char*)data.mv_data, data.mv_size))) { | 195 | if (resultHandler(QByteArray::fromRawData((char*)key.mv_data, key.mv_size), QByteArray::fromRawData((char*)data.mv_data, data.mv_size))) { |
237 | while ((rc = mdb_cursor_get(cursor, &key, &data, d->allowDuplicates ? MDB_NEXT_DUP : MDB_NEXT)) == 0) { | 196 | while ((rc = mdb_cursor_get(cursor, &key, &data, d->allowDuplicates ? MDB_NEXT_DUP : MDB_NEXT)) == 0) { |
@@ -264,34 +223,142 @@ int Storage::Transaction::scan(const QByteArray &k, | |||
264 | return numberOfRetrievedValues; | 223 | return numberOfRetrievedValues; |
265 | } | 224 | } |
266 | 225 | ||
267 | void Storage::Transaction::remove(const QByteArray &k, | 226 | |
268 | const std::function<void(const Storage::Error &error)> &errorHandler) | 227 | |
228 | |||
229 | class Storage::Transaction::Private | ||
230 | { | ||
231 | public: | ||
232 | Private(bool _requestRead, bool _allowDuplicates, const std::function<void(const Storage::Error &error)> &_defaultErrorHandler, const QString &_name, MDB_env *_env) | ||
233 | : env(_env), | ||
234 | requestedRead(_requestRead), | ||
235 | allowDuplicates(_allowDuplicates), | ||
236 | defaultErrorHandler(_defaultErrorHandler), | ||
237 | name(_name), | ||
238 | implicitCommit(false), | ||
239 | error(false), | ||
240 | modificationCounter(0) | ||
241 | { | ||
242 | |||
243 | } | ||
244 | ~Private() | ||
245 | { | ||
246 | |||
247 | } | ||
248 | |||
249 | MDB_env *env; | ||
250 | MDB_txn *transaction; | ||
251 | MDB_dbi dbi; | ||
252 | bool requestedRead; | ||
253 | bool allowDuplicates; | ||
254 | std::function<void(const Storage::Error &error)> defaultErrorHandler; | ||
255 | QString name; | ||
256 | bool implicitCommit; | ||
257 | bool error; | ||
258 | int modificationCounter; | ||
259 | |||
260 | void startTransaction() | ||
261 | { | ||
262 | // qDebug() << "Opening transaction " << requestedRead; | ||
263 | const int rc = mdb_txn_begin(env, NULL, requestedRead ? MDB_RDONLY : 0, &transaction); | ||
264 | if (rc) { | ||
265 | defaultErrorHandler(Error(name.toLatin1(), ErrorCodes::GenericError, "Error while opening transaction: " + QByteArray(mdb_strerror(rc)))); | ||
266 | } | ||
267 | } | ||
268 | }; | ||
269 | |||
270 | Storage::Transaction::Transaction() | ||
271 | : d(nullptr) | ||
272 | { | ||
273 | |||
274 | } | ||
275 | |||
276 | Storage::Transaction::Transaction(Transaction::Private *prv) | ||
277 | : d(prv) | ||
278 | { | ||
279 | d->startTransaction(); | ||
280 | } | ||
281 | |||
282 | Storage::Transaction::~Transaction() | ||
283 | { | ||
284 | if (d && d->transaction) { | ||
285 | if (d->implicitCommit && !d->error) { | ||
286 | // qDebug() << "implicit commit"; | ||
287 | commit(); | ||
288 | } else { | ||
289 | // qDebug() << "Aorting transaction"; | ||
290 | mdb_txn_abort(d->transaction); | ||
291 | } | ||
292 | } | ||
293 | delete d; | ||
294 | } | ||
295 | |||
296 | bool Storage::Transaction::commit(const std::function<void(const Storage::Error &error)> &errorHandler) | ||
269 | { | 297 | { |
270 | if (!d || !d->transaction) { | 298 | if (!d || !d->transaction) { |
271 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, "Not open"); | 299 | return false; |
300 | } | ||
301 | |||
302 | const int rc = mdb_txn_commit(d->transaction); | ||
303 | if (rc) { | ||
304 | mdb_txn_abort(d->transaction); | ||
305 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, "Error during transaction commit: " + QByteArray(mdb_strerror(rc))); | ||
272 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); | 306 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); |
307 | } | ||
308 | d->transaction = nullptr; | ||
309 | |||
310 | return !rc; | ||
311 | } | ||
312 | |||
313 | void Storage::Transaction::abort() | ||
314 | { | ||
315 | if (!d || !d->transaction) { | ||
273 | return; | 316 | return; |
274 | } | 317 | } |
275 | 318 | ||
276 | int rc; | 319 | mdb_txn_abort(d->transaction); |
277 | MDB_val key; | 320 | d->transaction = nullptr; |
278 | key.mv_size = k.size(); | 321 | } |
279 | key.mv_data = const_cast<void*>(static_cast<const void*>(k.data())); | ||
280 | rc = mdb_del(d->transaction, d->dbi, &key, 0); | ||
281 | 322 | ||
282 | if (rc) { | 323 | Storage::NamedDatabase Storage::Transaction::openDatabase(const QByteArray &db, const std::function<void(const Storage::Error &error)> &errorHandler) const |
324 | { | ||
325 | if (!d) { | ||
326 | return Storage::NamedDatabase(); | ||
327 | } | ||
328 | //We don't now if anything changed | ||
329 | d->implicitCommit = true; | ||
330 | auto p = new Storage::NamedDatabase::Private(db, d->allowDuplicates, d->defaultErrorHandler, d->name, d->transaction); | ||
331 | p->openDatabase(errorHandler); | ||
332 | return Storage::NamedDatabase(p); | ||
333 | } | ||
334 | |||
335 | bool Storage::Transaction::write(const QByteArray &key, const QByteArray &value, const std::function<void(const Storage::Error &error)> &errorHandler) | ||
336 | { | ||
337 | openDatabase().write(key, value, [this, errorHandler](const Storage::Error &error) { | ||
283 | d->error = true; | 338 | d->error = true; |
284 | Error error(d->name.toLatin1(), ErrorCodes::GenericError, QString("Error on mdb_del: %1 %2").arg(rc).arg(mdb_strerror(rc)).toLatin1()); | ||
285 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); | 339 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); |
286 | } else { | 340 | }); |
287 | d->implicitCommit = true; | 341 | d->implicitCommit = true; |
288 | } | ||
289 | 342 | ||
290 | return; | 343 | return !d->error; |
291 | } | 344 | } |
292 | 345 | ||
346 | void Storage::Transaction::remove(const QByteArray &k, | ||
347 | const std::function<void(const Storage::Error &error)> &errorHandler) | ||
348 | { | ||
349 | openDatabase().remove(k, [this, errorHandler](const Storage::Error &error) { | ||
350 | d->error = true; | ||
351 | errorHandler ? errorHandler(error) : d->defaultErrorHandler(error); | ||
352 | }); | ||
353 | d->implicitCommit = true; | ||
354 | } | ||
293 | 355 | ||
294 | 356 | int Storage::Transaction::scan(const QByteArray &k, | |
357 | const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler, | ||
358 | const std::function<void(const Storage::Error &error)> &errorHandler) const | ||
359 | { | ||
360 | return openDatabase().scan(k, resultHandler, errorHandler); | ||
361 | } | ||
295 | 362 | ||
296 | 363 | ||
297 | 364 | ||
@@ -310,9 +377,7 @@ public: | |||
310 | QString storageRoot; | 377 | QString storageRoot; |
311 | QString name; | 378 | QString name; |
312 | 379 | ||
313 | MDB_dbi dbi; | ||
314 | MDB_env *env; | 380 | MDB_env *env; |
315 | MDB_txn *transaction; | ||
316 | AccessMode mode; | 381 | AccessMode mode; |
317 | bool readTransaction; | 382 | bool readTransaction; |
318 | bool firstOpen; | 383 | bool firstOpen; |
@@ -328,7 +393,6 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m, bool | |||
328 | : storageRoot(s), | 393 | : storageRoot(s), |
329 | name(n), | 394 | name(n), |
330 | env(0), | 395 | env(0), |
331 | transaction(0), | ||
332 | mode(m), | 396 | mode(m), |
333 | readTransaction(false), | 397 | readTransaction(false), |
334 | firstOpen(true), | 398 | firstOpen(true), |
@@ -357,7 +421,12 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m, bool | |||
357 | // TODO: handle error | 421 | // TODO: handle error |
358 | std::cerr << "mdb_env_create: " << rc << " " << mdb_strerror(rc) << std::endl; | 422 | std::cerr << "mdb_env_create: " << rc << " " << mdb_strerror(rc) << std::endl; |
359 | } else { | 423 | } else { |
360 | if ((rc = mdb_env_open(env, fullPath.toStdString().data(), mode == ReadOnly ? MDB_RDONLY : 0 | MDB_NOTLS, 0664))) { | 424 | mdb_env_set_maxdbs(env, 10); |
425 | unsigned int flags = MDB_NOTLS; | ||
426 | if (mode == ReadOnly) { | ||
427 | flags |= MDB_RDONLY; | ||
428 | } | ||
429 | if ((rc = mdb_env_open(env, fullPath.toStdString().data(), flags, 0664))) { | ||
361 | std::cerr << "mdb_env_open: " << rc << " " << mdb_strerror(rc) << std::endl; | 430 | std::cerr << "mdb_env_open: " << rc << " " << mdb_strerror(rc) << std::endl; |
362 | mdb_env_close(env); | 431 | mdb_env_close(env); |
363 | env = 0; | 432 | env = 0; |
@@ -374,10 +443,6 @@ Storage::Private::Private(const QString &s, const QString &n, AccessMode m, bool | |||
374 | 443 | ||
375 | Storage::Private::~Private() | 444 | Storage::Private::~Private() |
376 | { | 445 | { |
377 | if (transaction) { | ||
378 | mdb_txn_abort(transaction); | ||
379 | } | ||
380 | |||
381 | //Since we can have only one environment open per process, we currently leak the environments. | 446 | //Since we can have only one environment open per process, we currently leak the environments. |
382 | // if (env) { | 447 | // if (env) { |
383 | // //mdb_dbi_close should not be necessary and is potentially dangerous (see docs) | 448 | // //mdb_dbi_close should not be necessary and is potentially dangerous (see docs) |
diff --git a/tests/indextest.cpp b/tests/indextest.cpp index 24f90c8..e3eabcc 100644 --- a/tests/indextest.cpp +++ b/tests/indextest.cpp | |||
@@ -13,38 +13,39 @@ class IndexTest : public QObject | |||
13 | private Q_SLOTS: | 13 | private Q_SLOTS: |
14 | void initTestCase() | 14 | void initTestCase() |
15 | { | 15 | { |
16 | Akonadi2::Storage store(Akonadi2::Store::storageLocation(), "org.kde.dummy.testindex", Akonadi2::Storage::ReadWrite); | 16 | Akonadi2::Storage store("./testindex", "org.kde.dummy.testindex", Akonadi2::Storage::ReadWrite); |
17 | store.removeFromDisk(); | 17 | store.removeFromDisk(); |
18 | } | 18 | } |
19 | 19 | ||
20 | void cleanup() | 20 | void cleanup() |
21 | { | 21 | { |
22 | Akonadi2::Storage store(Akonadi2::Store::storageLocation(), "org.kde.dummy.testindex", Akonadi2::Storage::ReadWrite); | 22 | Akonadi2::Storage store("./testindex", "org.kde.dummy.testindex", Akonadi2::Storage::ReadWrite); |
23 | store.removeFromDisk(); | 23 | store.removeFromDisk(); |
24 | } | 24 | } |
25 | 25 | ||
26 | void testIndex() | 26 | void testIndex() |
27 | { | 27 | { |
28 | Index index(Akonadi2::Store::storageLocation(), "org.kde.dummy.testindex", Akonadi2::Storage::ReadWrite); | 28 | Index index("./testindex", "org.kde.dummy.testindex", Akonadi2::Storage::ReadWrite); |
29 | index.add("key1", "value1"); | 29 | //The first key is specifically a substring of the second key |
30 | index.add("key1", "value2"); | 30 | index.add("key", "value1"); |
31 | index.add("key2", "value3"); | 31 | index.add("keyFoo", "value2"); |
32 | index.add("keyFoo", "value3"); | ||
32 | 33 | ||
33 | { | 34 | { |
34 | QList<QByteArray> values; | 35 | QList<QByteArray> values; |
35 | index.lookup(QByteArray("key1"), [&values](const QByteArray &value) { | 36 | index.lookup(QByteArray("key"), [&values](const QByteArray &value) { |
36 | values << value; | 37 | values << value; |
37 | }, | 38 | }, |
38 | [](const Index::Error &error){ qWarning() << "Error: "; }); | 39 | [](const Index::Error &error){ qWarning() << "Error: "; }); |
39 | QCOMPARE(values.size(), 2); | 40 | QCOMPARE(values.size(), 1); |
40 | } | 41 | } |
41 | { | 42 | { |
42 | QList<QByteArray> values; | 43 | QList<QByteArray> values; |
43 | index.lookup(QByteArray("key2"), [&values](const QByteArray &value) { | 44 | index.lookup(QByteArray("keyFoo"), [&values](const QByteArray &value) { |
44 | values << value; | 45 | values << value; |
45 | }, | 46 | }, |
46 | [](const Index::Error &error){ qWarning() << "Error: "; }); | 47 | [](const Index::Error &error){ qWarning() << "Error: "; }); |
47 | QCOMPARE(values.size(), 1); | 48 | QCOMPARE(values.size(), 2); |
48 | } | 49 | } |
49 | { | 50 | { |
50 | QList<QByteArray> values; | 51 | QList<QByteArray> values; |
diff --git a/tests/storagebenchmark.cpp b/tests/storagebenchmark.cpp index ce1005d..f143c4d 100644 --- a/tests/storagebenchmark.cpp +++ b/tests/storagebenchmark.cpp | |||
@@ -97,9 +97,12 @@ private Q_SLOTS: | |||
97 | auto event = createEvent(); | 97 | auto event = createEvent(); |
98 | if (store) { | 98 | if (store) { |
99 | auto transaction = store->createTransaction(Akonadi2::Storage::ReadWrite); | 99 | auto transaction = store->createTransaction(Akonadi2::Storage::ReadWrite); |
100 | transaction.setAutocommit(10000); | ||
101 | for (int i = 0; i < count; i++) { | 100 | for (int i = 0; i < count; i++) { |
102 | transaction.write(keyPrefix + QByteArray::number(i), event); | 101 | transaction.write(keyPrefix + QByteArray::number(i), event); |
102 | if ((i % 10000) == 0) { | ||
103 | transaction.commit(); | ||
104 | transaction = store->createTransaction(Akonadi2::Storage::ReadWrite); | ||
105 | } | ||
103 | } | 106 | } |
104 | transaction.commit(); | 107 | transaction.commit(); |
105 | } else { | 108 | } else { |
@@ -116,8 +119,9 @@ private Q_SLOTS: | |||
116 | { | 119 | { |
117 | if (store) { | 120 | if (store) { |
118 | auto transaction = store->createTransaction(Akonadi2::Storage::ReadOnly); | 121 | auto transaction = store->createTransaction(Akonadi2::Storage::ReadOnly); |
122 | auto db = transaction.openDatabase(); | ||
119 | for (int i = 0; i < count; i++) { | 123 | for (int i = 0; i < count; i++) { |
120 | transaction.scan(keyPrefix + QByteArray::number(i), [](const QByteArray &key, const QByteArray &value) -> bool { return true; }); | 124 | db.scan(keyPrefix + QByteArray::number(i), [](const QByteArray &key, const QByteArray &value) -> bool { return true; }); |
121 | } | 125 | } |
122 | } | 126 | } |
123 | } | 127 | } |
diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp index 55ec888..fe80bb7 100644 --- a/tests/storagetest.cpp +++ b/tests/storagetest.cpp | |||
@@ -211,6 +211,120 @@ private Q_SLOTS: | |||
211 | storage2.removeFromDisk(); | 211 | storage2.removeFromDisk(); |
212 | } | 212 | } |
213 | } | 213 | } |
214 | |||
215 | void testNoDuplicates() | ||
216 | { | ||
217 | bool gotResult = false; | ||
218 | bool gotError = false; | ||
219 | Akonadi2::Storage store(testDataPath, dbName, Akonadi2::Storage::ReadWrite, false); | ||
220 | auto transaction = store.createTransaction(Akonadi2::Storage::ReadWrite); | ||
221 | auto db = transaction.openDatabase(); | ||
222 | db.write("key","value"); | ||
223 | db.write("key","value"); | ||
224 | |||
225 | int numValues = db.scan("", [&](const QByteArray &key, const QByteArray &value) -> bool { | ||
226 | gotResult = true; | ||
227 | return true; | ||
228 | }, | ||
229 | [&](const Akonadi2::Storage::Error &error) { | ||
230 | qDebug() << error.message; | ||
231 | gotError = true; | ||
232 | }); | ||
233 | |||
234 | QCOMPARE(numValues, 1); | ||
235 | QVERIFY(!gotError); | ||
236 | } | ||
237 | |||
238 | void testDuplicates() | ||
239 | { | ||
240 | bool gotResult = false; | ||
241 | bool gotError = false; | ||
242 | Akonadi2::Storage store(testDataPath, dbName, Akonadi2::Storage::ReadWrite, true); | ||
243 | auto transaction = store.createTransaction(Akonadi2::Storage::ReadWrite); | ||
244 | auto db = transaction.openDatabase(); | ||
245 | db.write("key","value1"); | ||
246 | db.write("key","value2"); | ||
247 | int numValues = db.scan("key", [&](const QByteArray &key, const QByteArray &value) -> bool { | ||
248 | gotResult = true; | ||
249 | return true; | ||
250 | }, | ||
251 | [&](const Akonadi2::Storage::Error &error) { | ||
252 | qDebug() << error.message; | ||
253 | gotError = true; | ||
254 | }); | ||
255 | |||
256 | QCOMPARE(numValues, 2); | ||
257 | QVERIFY(!gotError); | ||
258 | } | ||
259 | |||
260 | void testNonexitingNamedDb() | ||
261 | { | ||
262 | bool gotResult = false; | ||
263 | bool gotError = false; | ||
264 | Akonadi2::Storage store(testDataPath, dbName, Akonadi2::Storage::ReadOnly); | ||
265 | int numValues = store.createTransaction(Akonadi2::Storage::ReadOnly).openDatabase("test").scan("", [&](const QByteArray &key, const QByteArray &value) -> bool { | ||
266 | gotResult = true; | ||
267 | return false; | ||
268 | }, | ||
269 | [&](const Akonadi2::Storage::Error &error) { | ||
270 | qDebug() << error.message; | ||
271 | gotError = true; | ||
272 | }); | ||
273 | QCOMPARE(numValues, 0); | ||
274 | QVERIFY(!gotResult); | ||
275 | QVERIFY(!gotError); | ||
276 | } | ||
277 | |||
278 | void testWriteToNamedDb() | ||
279 | { | ||
280 | bool gotError = false; | ||
281 | Akonadi2::Storage store(testDataPath, dbName, Akonadi2::Storage::ReadWrite); | ||
282 | store.createTransaction(Akonadi2::Storage::ReadWrite).openDatabase("test").write("key1", "value1", [&](const Akonadi2::Storage::Error &error) { | ||
283 | qDebug() << error.message; | ||
284 | gotError = true; | ||
285 | }); | ||
286 | QVERIFY(!gotError); | ||
287 | } | ||
288 | |||
289 | void testWriteDuplicatesToNamedDb() | ||
290 | { | ||
291 | bool gotError = false; | ||
292 | Akonadi2::Storage store(testDataPath, dbName, Akonadi2::Storage::ReadWrite, true); | ||
293 | store.createTransaction(Akonadi2::Storage::ReadWrite).openDatabase("test").write("key1", "value1", [&](const Akonadi2::Storage::Error &error) { | ||
294 | qDebug() << error.message; | ||
295 | gotError = true; | ||
296 | }); | ||
297 | QVERIFY(!gotError); | ||
298 | } | ||
299 | |||
300 | //By default we want only exact matches | ||
301 | void testSubstringKeys() | ||
302 | { | ||
303 | Akonadi2::Storage store(testDataPath, dbName, Akonadi2::Storage::ReadWrite, true); | ||
304 | auto transaction = store.createTransaction(Akonadi2::Storage::ReadWrite); | ||
305 | auto db = transaction.openDatabase(); | ||
306 | db.write("sub","value1"); | ||
307 | db.write("subsub","value2"); | ||
308 | int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { | ||
309 | return true; | ||
310 | }); | ||
311 | |||
312 | QCOMPARE(numValues, 1); | ||
313 | } | ||
314 | |||
315 | //Ensure we don't retrieve a key that is greater than the current key. We only want equal keys. | ||
316 | void testKeyRange() | ||
317 | { | ||
318 | Akonadi2::Storage store(testDataPath, dbName, Akonadi2::Storage::ReadWrite, true); | ||
319 | auto transaction = store.createTransaction(Akonadi2::Storage::ReadWrite); | ||
320 | auto db = transaction.openDatabase(); | ||
321 | db.write("sub1","value1"); | ||
322 | int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { | ||
323 | return true; | ||
324 | }); | ||
325 | |||
326 | QCOMPARE(numValues, 0); | ||
327 | } | ||
214 | }; | 328 | }; |
215 | 329 | ||
216 | QTEST_MAIN(StorageTest) | 330 | QTEST_MAIN(StorageTest) |