Refactor how the storage is used.

This is the initial refactoring to improve how we deal with the storage.

It does a couple of things:
* Rename Sink::Storage to Sink::Storage::DataStore to free up the
  Sink::Storage namespace
* Introduce a Sink::ResourceContext to have a single object that can be
  passed around containing everything that is necessary to operate on a
  resource. This is a lot better than the multiple separate parameters
  that we used to pass around all over the place, while still allowing
  for dependency injection for tests.
* Tie storage access together using the new EntityStore that directly
  works with ApplicationDomainTypes. This gives us a central place where
  main storage, indexes and buffer adaptors are tied together, which
  will also give us a place to implement external indexes, such as a
  fulltextindex using xapian.
* Use ApplicationDomainTypes as the default way to pass around entities.
  Instead of using various ways to pass around entities (buffers,
  buffer adaptors, ApplicationDomainTypes), only use a single way.
  The old approach was confusing, and was only done as:
  * optimization; really shouldn't be necessary and otherwise I'm sure
  we can find better ways to optimize ApplicationDomainType itself.
  * a way to account for entities that have multiple buffers, a concept
  that I no longer deem relevant.

While this commit does the bulk of the work to get there, the following
commits will refactor more stuff to get things back to normal.

8 files changed, 77 insertions, 43 deletions

diff --git a/common/domain/applicationdomaintype.cpp b/common/domain/applicationdomaintype.cpp
index 2a0d977..3109966 100644
--- a/common/domain/applicationdomaintype.cpp
+++ b/common/domain/applicationdomaintype.cpp
@@ -73,7 +73,7 @@ ApplicationDomainType::~ApplicationDomainType()
 
 QByteArray ApplicationDomainType::generateUid()
 {
-    return Sink::Storage::generateUid();
+    return Sink::Storage::DataStore::generateUid();
 }
 
 bool ApplicationDomainType::hasProperty(const QByteArray &key) const
diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h
index e581e07..39ce2b9 100644
--- a/common/domain/applicationdomaintype.h
+++ b/common/domain/applicationdomaintype.h
@@ -241,6 +241,8 @@ struct SINK_EXPORT SinkResource : public ApplicationDomainType {
 struct SINK_EXPORT Entity : public ApplicationDomainType {
     typedef QSharedPointer<Entity> Ptr;
     using ApplicationDomainType::ApplicationDomainType;
+    Entity() = default;
+    Entity(const ApplicationDomainType &other) : ApplicationDomainType(other) {}
     virtual ~Entity();
 };
 
diff --git a/common/domain/event.cpp b/common/domain/event.cpp
index f3abd62..d801592 100644
--- a/common/domain/event.cpp
+++ b/common/domain/event.cpp
@@ -42,23 +42,28 @@ static QMutex sMutex;
 
 using namespace Sink::ApplicationDomain;
 
+void TypeImplementation<Event>::configureIndex(TypeIndex &index)
+{
+    index.addProperty<QByteArray>(Event::Uid::name);
+}
+
 static TypeIndex &getIndex()
 {
     QMutexLocker locker(&sMutex);
     static TypeIndex *index = 0;
     if (!index) {
         index = new TypeIndex("event");
-        index->addProperty<QByteArray>("uid");
+        TypeImplementation<Event>::configureIndex(*index);
     }
     return *index;
 }
 
-void TypeImplementation<Event>::index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction)
+void TypeImplementation<Event>::index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction)
 {
     return getIndex().add(identifier, bufferAdaptor, transaction);
 }
 
-void TypeImplementation<Event>::removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction)
+void TypeImplementation<Event>::removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction)
 {
     return getIndex().remove(identifier, bufferAdaptor, transaction);
 }
@@ -83,10 +88,10 @@ QSharedPointer<WritePropertyMapper<TypeImplementation<Event>::BufferBuilder> > T
     return propertyMapper;
 }
 
-DataStoreQuery::Ptr TypeImplementation<Event>::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction)
+DataStoreQuery::Ptr TypeImplementation<Event>::prepareQuery(const Sink::Query &query, Sink::Storage::EntityStore::Ptr store)
 {
     auto mapper = initializeReadPropertyMapper();
-    return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName<Event>(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) {
+    return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName<Event>(), store, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) {
 
         const auto localBuffer = Sink::EntityBuffer::readBuffer<Buffer>(entity.local());
         return mapper->getProperty(property, localBuffer);
diff --git a/common/domain/event.h b/common/domain/event.h
index 684b58e..ce9691d 100644
--- a/common/domain/event.h
+++ b/common/domain/event.h
@@ -21,6 +21,7 @@
 #include "applicationdomaintype.h"
 
 #include "storage.h"
+#include "storage/entitystore.h"
 
 class ResultSet;
 class QByteArray;
@@ -32,6 +33,8 @@ class WritePropertyMapper;
 
 class DataStoreQuery;
 
+class TypeIndex;
+
 namespace Sink {
     class Query;
 
@@ -51,10 +54,12 @@ class TypeImplementation<Sink::ApplicationDomain::Event> {
 public:
     typedef Sink::ApplicationDomain::Buffer::Event Buffer;
     typedef Sink::ApplicationDomain::Buffer::EventBuilder BufferBuilder;
+    static void configureIndex(TypeIndex &index);
     static QSet<QByteArray> indexedProperties();
-    static QSharedPointer<DataStoreQuery> prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction);
-    static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
-    static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
+    static QSharedPointer<DataStoreQuery> prepareQuery(const Sink::Query &query, Sink::Storage::EntityStore::Ptr store);
+
+    static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction);
+    static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction);
     static QSharedPointer<ReadPropertyMapper<Buffer> > initializeReadPropertyMapper();
     static QSharedPointer<WritePropertyMapper<BufferBuilder> > initializeWritePropertyMapper();
 };
diff --git a/common/domain/folder.cpp b/common/domain/folder.cpp
index 824fa0b..f04a3e7 100644
--- a/common/domain/folder.cpp
+++ b/common/domain/folder.cpp
@@ -44,25 +44,30 @@ static QMutex sMutex;
 
 using namespace Sink::ApplicationDomain;
 
+void TypeImplementation<Folder>::configureIndex(TypeIndex &index)
+{
+    index.addProperty<QByteArray>(Folder::Parent::name);
+    index.addProperty<QString>(Folder::Name::name);
+}
+
 static TypeIndex &getIndex()
 {
     QMutexLocker locker(&sMutex);
     static TypeIndex *index = 0;
     if (!index) {
         index = new TypeIndex("folder");
-        index->addProperty<QByteArray>("parent");
-        index->addProperty<QString>("name");
+        TypeImplementation<Folder>::configureIndex(*index);
     }
     return *index;
 }
 
-void TypeImplementation<Folder>::index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction)
+void TypeImplementation<Folder>::index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction)
 {
     SinkTrace() << "Indexing " << identifier;
     getIndex().add(identifier, bufferAdaptor, transaction);
 }
 
-void TypeImplementation<Folder>::removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction)
+void TypeImplementation<Folder>::removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction)
 {
     getIndex().remove(identifier, bufferAdaptor, transaction);
 }
@@ -87,10 +92,10 @@ QSharedPointer<WritePropertyMapper<TypeImplementation<Folder>::BufferBuilder> >
     return propertyMapper;
 }
 
-DataStoreQuery::Ptr TypeImplementation<Folder>::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction)
+DataStoreQuery::Ptr TypeImplementation<Folder>::prepareQuery(const Sink::Query &query, Sink::Storage::EntityStore::Ptr store)
 {
     auto mapper = initializeReadPropertyMapper();
-    return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName<Folder>(), transaction, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) {
+    return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName<Folder>(), store, getIndex(), [mapper](const Sink::Entity &entity, const QByteArray &property) {
         const auto localBuffer = Sink::EntityBuffer::readBuffer<Buffer>(entity.local());
         return mapper->getProperty(property, localBuffer);
     });
diff --git a/common/domain/folder.h b/common/domain/folder.h
index e4631de..0a52b01 100644
--- a/common/domain/folder.h
+++ b/common/domain/folder.h
@@ -21,6 +21,7 @@
 #include "applicationdomaintype.h"
 
 #include "storage.h"
+#include "storage/entitystore.h"
 
 class ResultSet;
 class QByteArray;
@@ -31,6 +32,8 @@ class ReadPropertyMapper;
 template<typename T>
 class WritePropertyMapper;
 
+class TypeIndex;
+
 namespace Sink {
     class Query;
 
@@ -45,10 +48,11 @@ class TypeImplementation<Sink::ApplicationDomain::Folder> {
 public:
     typedef Sink::ApplicationDomain::Buffer::Folder Buffer;
     typedef Sink::ApplicationDomain::Buffer::FolderBuilder BufferBuilder;
-    static QSharedPointer<DataStoreQuery> prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction);
+    static void configureIndex(TypeIndex &index);
+    static QSharedPointer<DataStoreQuery> prepareQuery(const Sink::Query &query, Sink::Storage::EntityStore::Ptr store);
     static QSet<QByteArray> indexedProperties();
-    static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
-    static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
+    static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction);
+    static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction);
     static QSharedPointer<ReadPropertyMapper<Buffer> > initializeReadPropertyMapper();
     static QSharedPointer<WritePropertyMapper<BufferBuilder> > initializeWritePropertyMapper();
 };
diff --git a/common/domain/mail.cpp b/common/domain/mail.cpp
index 2b6eb84..1b46e28 100644
--- a/common/domain/mail.cpp
+++ b/common/domain/mail.cpp
@@ -45,25 +45,31 @@ static QMutex sMutex;
 using namespace Sink;
 using namespace Sink::ApplicationDomain;
 
+void TypeImplementation<Mail>::configureIndex(TypeIndex &index)
+{
+    index.addProperty<QByteArray>(Mail::Uid::name);
+    index.addProperty<QByteArray>(Mail::Sender::name);
+    index.addProperty<QByteArray>(Mail::SenderName::name);
+    /* index->addProperty<QString>(Mail::Subject::name); */
+    /* index->addFulltextProperty<QString>(Mail::Subject::name); */
+    index.addProperty<QDateTime>(Mail::Date::name);
+    index.addProperty<QByteArray>(Mail::Folder::name);
+    index.addPropertyWithSorting<QByteArray, QDateTime>(Mail::Folder::name, Mail::Date::name);
+    index.addProperty<QByteArray>(Mail::MessageId::name);
+    index.addProperty<QByteArray>(Mail::ParentMessageId::name);
+
+    index.addProperty<Mail::MessageId>();
+    index.addSecondaryProperty<Mail::MessageId, Mail::ThreadId>();
+    index.addSecondaryProperty<Mail::ThreadId, Mail::MessageId>();
+}
+
 static TypeIndex &getIndex()
 {
     QMutexLocker locker(&sMutex);
     static TypeIndex *index = 0;
     if (!index) {
         index = new TypeIndex("mail");
-        index->addProperty<QByteArray>(Mail::Uid::name);
-        index->addProperty<QByteArray>(Mail::Sender::name);
-        index->addProperty<QByteArray>(Mail::SenderName::name);
-        index->addProperty<QString>(Mail::Subject::name);
-        index->addProperty<QDateTime>(Mail::Date::name);
-        index->addProperty<QByteArray>(Mail::Folder::name);
-        index->addPropertyWithSorting<QByteArray, QDateTime>(Mail::Folder::name, Mail::Date::name);
-        index->addProperty<QByteArray>(Mail::MessageId::name);
-        index->addProperty<QByteArray>(Mail::ParentMessageId::name);
-
-        index->addProperty<Mail::MessageId>();
-        index->addSecondaryProperty<Mail::MessageId, Mail::ThreadId>();
-        index->addSecondaryProperty<Mail::ThreadId, Mail::MessageId>();
+        TypeImplementation<Mail>::configureIndex(*index);
     }
     return *index;
 }
@@ -122,7 +128,7 @@ static QString stripOffPrefixes(const QString &subject)
 }
 
 
-static void updateThreadingIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction)
+static void updateThreadingIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction)
 {
     auto messageId = bufferAdaptor.getProperty(Mail::MessageId::name);
     auto parentMessageId = bufferAdaptor.getProperty(Mail::ParentMessageId::name);
@@ -164,16 +170,17 @@ static void updateThreadingIndex(const QByteArray &identifier, const BufferAdapt
     }
 }
 
-void TypeImplementation<Mail>::index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction)
+void TypeImplementation<Mail>::index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction)
 {
     SinkTrace() << "Indexing " << identifier;
     getIndex().add(identifier, bufferAdaptor, transaction);
     updateThreadingIndex(identifier, bufferAdaptor, transaction);
 }
 
-void TypeImplementation<Mail>::removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction)
+void TypeImplementation<Mail>::removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction)
 {
     getIndex().remove(identifier, bufferAdaptor, transaction);
+    //TODO cleanup threading index
 }
 
 QSharedPointer<ReadPropertyMapper<TypeImplementation<Mail>::Buffer> > TypeImplementation<Mail>::initializeReadPropertyMapper()
@@ -218,18 +225,21 @@ QSharedPointer<WritePropertyMapper<TypeImplementation<Mail>::BufferBuilder> > Ty
 }
 
 
-DataStoreQuery::Ptr TypeImplementation<Mail>::prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction)
+DataStoreQuery::Ptr TypeImplementation<Mail>::prepareQuery(const Sink::Query &query, Sink::Storage::EntityStore::Ptr store)
 {
     auto mapper = initializeReadPropertyMapper();
-    return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName<Mail>(), transaction, getIndex(), [mapper, &transaction](const Sink::Entity &entity, const QByteArray &property) -> QVariant {
+    return DataStoreQuery::Ptr::create(query, ApplicationDomain::getTypeName<Mail>(), store, getIndex(), [mapper, store](const Sink::Entity &entity, const QByteArray &property) -> QVariant {
         if (property == Mail::ThreadId::name) {
             const auto localBuffer = Sink::EntityBuffer::readBuffer<Buffer>(entity.local());
             Q_ASSERT(localBuffer);
             auto messageId = mapper->getProperty(Mail::MessageId::name, localBuffer);
+            //FIXME
             //This is an index property that we have too lookup
-            auto thread = getIndex().secondaryLookup<Mail::MessageId, Mail::ThreadId>(messageId, transaction);
-            Q_ASSERT(!thread.isEmpty());
-            return thread.first();
+            /* auto thread = getIndex().secondaryLookup<Mail::MessageId, Mail::ThreadId>(messageId); */
+            /* auto thread = store->secondaryLookup<Mail::MessageId, Mail::ThreadId>(messageId); */
+            /* Q_ASSERT(!thread.isEmpty()); */
+            /* return thread.first(); */
+            return QVariant();
         } else {
             const auto localBuffer = Sink::EntityBuffer::readBuffer<Buffer>(entity.local());
             Q_ASSERT(localBuffer);
diff --git a/common/domain/mail.h b/common/domain/mail.h
index ea3ef9e..6c1f670 100644
--- a/common/domain/mail.h
+++ b/common/domain/mail.h
@@ -21,7 +21,7 @@
 #include "applicationdomaintype.h"
 
 #include "storage.h"
-#include "datastorequery.h"
+#include "storage/entitystore.h"
 
 class ResultSet;
 class QByteArray;
@@ -32,6 +32,8 @@ class ReadPropertyMapper;
 template<typename T>
 class WritePropertyMapper;
 
+class TypeIndex;
+
 namespace Sink {
     class Query;
 
@@ -46,10 +48,11 @@ class TypeImplementation<Sink::ApplicationDomain::Mail> {
 public:
     typedef Sink::ApplicationDomain::Buffer::Mail Buffer;
     typedef Sink::ApplicationDomain::Buffer::MailBuilder BufferBuilder;
-    static QSharedPointer<DataStoreQuery> prepareQuery(const Sink::Query &query, Sink::Storage::Transaction &transaction);
+    static void configureIndex(TypeIndex &index);
+    static QSharedPointer<DataStoreQuery> prepareQuery(const Sink::Query &query, Sink::Storage::EntityStore::Ptr storage);
     static QSet<QByteArray> indexedProperties();
-    static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
-    static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::Transaction &transaction);
+    static void index(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction);
+    static void removeIndex(const QByteArray &identifier, const BufferAdaptor &bufferAdaptor, Sink::Storage::DataStore::Transaction &transaction);
     static QSharedPointer<ReadPropertyMapper<Buffer> > initializeReadPropertyMapper();
     static QSharedPointer<WritePropertyMapper<BufferBuilder> > initializeWritePropertyMapper();
 };