Separate UIDs and Revisions in main databases

Summary:
- Change revision type from `qint64` to `size_t` for LMDB in a couple of places (LMDB supports `unsigned int` or `size_t` which are `long unsigned int` on my machine)
- Better support for database flags (duplicate, integer keys, integer values for now but is extensible)
- Main databases' keys are now revisions
- Some databases switched to integer keys databases:
  - Main databases
  - the revision to uid mapping database
  - the revision to entity type mapping database
- Refactor the entity type's `typeDatabases` method (if in the future we need to change the main databases' flags again)
- New uid to revision mapping database (`uidsToRevisions`):
  - Stores all revisions (not uid to latest revision) because we need it for cleaning old revisions
  - Flags are: duplicates + integer values (so findLatest finds the latest revision for the given uid)

~~Problems to fix before merging:~~
All Fixed!

- ~~Sometimes Sink can't read what has just been written to the database (maybe because of transactions race conditions)~~
  - ~~Most of the times, this results in Sink not able to find the uid for a given revision by reading the `revisions` database~~
- ~~`pipelinetest`'s `testModifyWithConflict` fails because the local changes are overridden~~

~~The first problem prevents me from running benchmarks~~

Reviewers: cmollekopf

Tags: #sink

Differential Revision: https://phabricator.kde.org/D14974

2 files changed, 17 insertions, 13 deletions

diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp
index aedf889..6e14501 100644
--- a/common/domain/typeimplementations.cpp
+++ b/common/domain/typeimplementations.cpp
@@ -80,7 +80,11 @@ typedef IndexConfig<Calendar,
         ValueIndex<Calendar::Name>
     > CalendarIndexConfig;
 
-
+template <typename EntityType, typename EntityIndexConfig>
+QMap<QByteArray, int> defaultTypeDatabases()
+{
+    return merge(QMap<QByteArray, int>{{QByteArray{EntityType::name} + ".main", Storage::IntegerKeys}}, EntityIndexConfig::databases());
+}
 
 void TypeImplementation<Mail>::configure(TypeIndex &index)
 {
@@ -89,7 +93,7 @@ void TypeImplementation<Mail>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Mail>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Mail::name} + ".main", 0}}, MailIndexConfig::databases());
+    return defaultTypeDatabases<Mail, MailIndexConfig>();
 }
 
 void TypeImplementation<Mail>::configure(IndexPropertyMapper &indexPropertyMapper)
@@ -132,7 +136,7 @@ void TypeImplementation<Folder>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Folder>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Folder::name} + ".main", 0}}, FolderIndexConfig::databases());
+    return defaultTypeDatabases<Folder, FolderIndexConfig>();
 }
 
 void TypeImplementation<Folder>::configure(PropertyMapper &propertyMapper)
@@ -157,7 +161,7 @@ void TypeImplementation<Contact>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Contact>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Contact::name} + ".main", 0}}, ContactIndexConfig::databases());
+    return defaultTypeDatabases<Contact, ContactIndexConfig>();
 }
 
 void TypeImplementation<Contact>::configure(PropertyMapper &propertyMapper)
@@ -185,7 +189,7 @@ void TypeImplementation<Addressbook>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Addressbook>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Addressbook::name} + ".main", 0}}, AddressbookIndexConfig::databases());
+    return defaultTypeDatabases<Addressbook, AddressbookIndexConfig>();
 }
 
 void TypeImplementation<Addressbook>::configure(PropertyMapper &propertyMapper)
@@ -207,7 +211,7 @@ void TypeImplementation<Event>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Event>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Event::name} + ".main", 0}}, EventIndexConfig::databases());
+    return defaultTypeDatabases<Event, EventIndexConfig>();
 }
 
 void TypeImplementation<Event>::configure(PropertyMapper &propertyMapper)
@@ -235,7 +239,7 @@ void TypeImplementation<Todo>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Todo>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Todo::name} + ".main", 0}}, TodoIndexConfig::databases());
+    return defaultTypeDatabases<Todo, TodoIndexConfig>();
 }
 
 void TypeImplementation<Todo>::configure(PropertyMapper &propertyMapper)
@@ -266,7 +270,7 @@ void TypeImplementation<Calendar>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Calendar>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Calendar::name} + ".main", 0}}, CalendarIndexConfig::databases());
+    return defaultTypeDatabases<Calendar, CalendarIndexConfig>();
 }
 
 void TypeImplementation<Calendar>::configure(PropertyMapper &propertyMapper)
diff --git a/common/domain/typeimplementations_p.h b/common/domain/typeimplementations_p.h
index 51af113..bfdea77 100644
--- a/common/domain/typeimplementations_p.h
+++ b/common/domain/typeimplementations_p.h
@@ -57,7 +57,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + Property::name, 1}};
+        return {{QByteArray{EntityType::name} +".index." + Property::name, Sink::Storage::AllowDuplicates}};
     }
 };
 
@@ -74,7 +74,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + Property::name + ".sort." + SortProperty::name, 1}};
+        return {{QByteArray{EntityType::name} +".index." + Property::name + ".sort." + SortProperty::name, Sink::Storage::AllowDuplicates}};
     }
 };
 
@@ -90,7 +90,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + SortProperty::name + ".sorted", 1}};
+        return {{QByteArray{EntityType::name} +".index." + SortProperty::name + ".sorted", Sink::Storage::AllowDuplicates}};
     }
 };
 
@@ -106,7 +106,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + Property::name + SecondaryProperty::name, 1}};
+        return {{QByteArray{EntityType::name} +".index." + Property::name + SecondaryProperty::name, Sink::Storage::AllowDuplicates}};
     }
 };
 
@@ -142,7 +142,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + RangeBeginProperty::name + ".range." + RangeEndProperty::name, 1}};
+        return {{QByteArray{EntityType::name} +".index." + RangeBeginProperty::name + ".range." + RangeEndProperty::name, Sink::Storage::AllowDuplicates}};
     }
 };