Move the BLOB property handling to the entitystore.

This is really part of the storage, and will help us to cleanly
implement features like moving properties into a temporary place when
reading in a clean way as well.

9 files changed, 74 insertions, 65 deletions

diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h
index 7c3800d..1848224 100644
--- a/common/domain/applicationdomaintype.h
+++ b/common/domain/applicationdomaintype.h
@@ -99,7 +99,15 @@ struct SINK_EXPORT Progress {
 };
 
 struct BLOB {
+    BLOB() = default;
+    BLOB(const BLOB &) = default;
+    BLOB(const QString &id) : value(id) {};
+    ~BLOB() = default;
+    bool operator==(const BLOB &other) const {
+        return value == other.value && isExternal == other.isExternal;
+    }
     QString value;
+    bool isExternal = true;
 };
 
 /**
diff --git a/common/mailpreprocessor.cpp b/common/mailpreprocessor.cpp
index 575ff4b..bde8a64 100644
--- a/common/mailpreprocessor.cpp
+++ b/common/mailpreprocessor.cpp
@@ -152,49 +152,3 @@ void MailPropertyExtractor::modifiedEntity(const Sink::ApplicationDomain::Mail &
     }
 }
 
-
-MimeMessageMover::MimeMessageMover() : Sink::EntityPreprocessor<ApplicationDomain::Mail>()
-{
-}
-
-QString MimeMessageMover::moveMessage(const QString &oldPath, const Sink::ApplicationDomain::Mail &mail)
-{
-    const auto directory = Sink::resourceStorageLocation(resourceInstanceIdentifier());
-    const auto filePath = directory + "/" + mail.identifier();
-    if (oldPath != filePath) {
-        if (!QDir().mkpath(directory)) {
-            SinkWarning() << "Failed to create the directory: " << directory;
-        }
-        QFile::remove(filePath);
-        QFile origFile(oldPath);
-        if (!origFile.open(QIODevice::ReadWrite)) {
-            SinkWarning() << "Failed to open the original file with write rights: " << origFile.errorString();
-        }
-        if (!origFile.rename(filePath)) {
-            SinkWarning() << "Failed to move the file from: " << oldPath << " to " << filePath << ". " << origFile.errorString();
-        }
-        origFile.close();
-        return filePath;
-    }
-    return oldPath;
-}
-
-void MimeMessageMover::newEntity(Sink::ApplicationDomain::Mail &mail)
-{
-    if (!mail.getMimeMessagePath().isEmpty()) {
-        mail.setMimeMessagePath(moveMessage(mail.getMimeMessagePath(), mail));
-    }
-}
-
-void MimeMessageMover::modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail)
-{
-    if (!newMail.getMimeMessagePath().isEmpty()) {
-        newMail.setMimeMessagePath(moveMessage(newMail.getMimeMessagePath(), newMail));
-    }
-}
-
-void MimeMessageMover::deletedEntity(const Sink::ApplicationDomain::Mail &mail)
-{
-    QFile::remove(mail.getMimeMessagePath());
-}
-
diff --git a/common/mailpreprocessor.h b/common/mailpreprocessor.h
index f979f22..a24a8d3 100644
--- a/common/mailpreprocessor.h
+++ b/common/mailpreprocessor.h
@@ -30,16 +30,3 @@ protected:
     virtual QString getFilePathFromMimeMessagePath(const QString &) const;
 };
 
-class SINK_EXPORT MimeMessageMover : public Sink::EntityPreprocessor<Sink::ApplicationDomain::Mail>
-{
-public:
-    MimeMessageMover();
-    virtual ~MimeMessageMover(){}
-
-    void newEntity(Sink::ApplicationDomain::Mail &mail) Q_DECL_OVERRIDE;
-    void modifiedEntity(const Sink::ApplicationDomain::Mail &oldMail, Sink::ApplicationDomain::Mail &newMail) Q_DECL_OVERRIDE;
-    void deletedEntity(const Sink::ApplicationDomain::Mail &mail) Q_DECL_OVERRIDE;
-
-private:
-    QString moveMessage(const QString &oldPath, const Sink::ApplicationDomain::Mail &mail);
-};
diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp
index 999bb2c..d8b1121 100644
--- a/common/storage/entitystore.cpp
+++ b/common/storage/entitystore.cpp
@@ -19,6 +19,9 @@
  */
 #include "entitystore.h"
 
+#include <QDir>
+#include <QFile>
+
 #include "entitybuffer.h"
 #include "log.h"
 #include "typeindex.h"
@@ -90,6 +93,11 @@ public:
         return ApplicationDomain::ApplicationDomainType{resourceContext.instanceId(), uid, revision, adaptor};
     }
 
+    QString entityBlobStoragePath(const QByteArray &id)
+    {
+        return Sink::resourceStorageLocation(resourceContext.instanceId()) + "/blob/" + id;
+    }
+
 };
 
 EntityStore::EntityStore(const ResourceContext &context)
@@ -120,6 +128,38 @@ void EntityStore::abortTransaction()
     d->transaction = Storage::DataStore::Transaction();
 }
 
+void EntityStore::copyBlobs(ApplicationDomain::ApplicationDomainType &entity, qint64 newRevision)
+{
+    const auto directory = d->entityBlobStoragePath(entity.identifier());
+    if (!QDir().mkpath(directory)) {
+        SinkWarning() << "Failed to create the directory: " << directory;
+    }
+
+    for (const auto &property : entity.changedProperties()) {
+        const auto value = entity.getProperty(property);
+        if (value.canConvert<ApplicationDomain::BLOB>()) {
+            const auto blob = value.value<ApplicationDomain::BLOB>();
+            bool blobIsExternal = blob.isExternal;
+            //Any blob that is not part of the storage yet has to be moved there.
+            if (blob.isExternal) {
+                auto oldPath = blob.value;
+                auto filePath = directory + QString("/%1%2.blob").arg(QString::number(newRevision)).arg(QString::fromLatin1(property));
+                //In case we hit the same revision again due to a rollback.
+                QFile::remove(filePath);
+                QFile origFile(oldPath);
+                if (!origFile.open(QIODevice::ReadWrite)) {
+                    SinkWarning() << "Failed to open the original file with write rights: " << origFile.errorString();
+                }
+                if (!origFile.rename(filePath)) {
+                    SinkWarning() << "Failed to move the file from: " << oldPath << " to " << filePath << ". " << origFile.errorString();
+                }
+                origFile.close();
+                entity.setProperty(property, QVariant::fromValue(ApplicationDomain::BLOB{filePath}));
+            }
+        }
+    }
+}
+
 bool EntityStore::add(const QByteArray &type, const ApplicationDomain::ApplicationDomainType &entity_, bool replayToSource, const PreprocessCreation &preprocess)
 {
     if (entity_.identifier().isEmpty()) {
@@ -138,6 +178,8 @@ bool EntityStore::add(const QByteArray &type, const ApplicationDomain::Applicati
     //The maxRevision may have changed meanwhile if the entity created sub-entities
     const qint64 newRevision = maxRevision() + 1;
 
+    copyBlobs(entity, newRevision);
+
     // Add metadata buffer
     flatbuffers::FlatBufferBuilder metadataFbb;
     auto metadataBuilder = MetadataBuilder(metadataFbb);
@@ -192,6 +234,8 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::Applic
 
     const qint64 newRevision = DataStore::maxRevision(d->transaction) + 1;
 
+    copyBlobs(newEntity, newRevision);
+
     // Add metadata buffer
     flatbuffers::FlatBufferBuilder metadataFbb;
     {
@@ -297,6 +341,13 @@ void EntityStore::cleanupRevision(qint64 revision)
                         DataStore::removeRevision(d->transaction, rev);
                         DataStore::mainDatabase(d->transaction, bufferType).remove(key);
                     }
+                    if (metadata->operation() == Operation_Removal) {
+                        const auto directory = d->entityBlobStoragePath(uid);
+                        QDir dir(directory);
+                        if (!dir.removeRecursively()) {
+                            SinkError() << "Failed to cleanup: " << directory;
+                        }
+                    }
                 }
 
                 return true;
diff --git a/common/storage/entitystore.h b/common/storage/entitystore.h
index 06ca8c4..c626ebc 100644
--- a/common/storage/entitystore.h
+++ b/common/storage/entitystore.h
@@ -110,6 +110,7 @@ public:
     qint64 maxRevision();
 
 private:
+    void copyBlobs(ApplicationDomain::ApplicationDomainType &entity, qint64 newRevision);
     class Private;
     const QSharedPointer<Private> d;
 };
diff --git a/examples/dummyresource/resourcefactory.cpp b/examples/dummyresource/resourcefactory.cpp
index 07021b9..03238ef 100644
--- a/examples/dummyresource/resourcefactory.cpp
+++ b/examples/dummyresource/resourcefactory.cpp
@@ -163,7 +163,7 @@ DummyResource::DummyResource(const Sink::ResourceContext &resourceContext, const
     setupSynchronizer(QSharedPointer<DummySynchronizer>::create(resourceContext));
     setupInspector(QSharedPointer<DummyInspector>::create(resourceContext));
     setupPreprocessors(ENTITY_TYPE_MAIL,
-            QVector<Sink::Preprocessor*>() << new MailPropertyExtractor << new MimeMessageMover << new SpecialPurposeProcessor{resourceContext.resourceType, resourceContext.instanceId()});
+            QVector<Sink::Preprocessor*>() << new MailPropertyExtractor << new SpecialPurposeProcessor{resourceContext.resourceType, resourceContext.instanceId()});
     setupPreprocessors(ENTITY_TYPE_FOLDER,
             QVector<Sink::Preprocessor*>());
     setupPreprocessors(ENTITY_TYPE_EVENT,
diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp
index 238fb0c..252b910 100644
--- a/examples/imapresource/imapresource.cpp
+++ b/examples/imapresource/imapresource.cpp
@@ -737,7 +737,7 @@ ImapResource::ImapResource(const ResourceContext &resourceContext)
     inspector->mPassword = password;
     setupInspector(inspector);
 
-    setupPreprocessors(ENTITY_TYPE_MAIL, QVector<Sink::Preprocessor*>() << new SpecialPurposeProcessor(resourceContext.resourceType, resourceContext.instanceId()) << new MimeMessageMover << new MailPropertyExtractor);
+    setupPreprocessors(ENTITY_TYPE_MAIL, QVector<Sink::Preprocessor*>() << new SpecialPurposeProcessor(resourceContext.resourceType, resourceContext.instanceId()) << new MailPropertyExtractor);
     setupPreprocessors(ENTITY_TYPE_FOLDER, QVector<Sink::Preprocessor*>());
 }
 
diff --git a/examples/maildirresource/maildirresource.cpp b/examples/maildirresource/maildirresource.cpp
index a05afc6..6a03263 100644
--- a/examples/maildirresource/maildirresource.cpp
+++ b/examples/maildirresource/maildirresource.cpp
@@ -132,7 +132,10 @@ public:
         auto mail = newEntity.cast<ApplicationDomain::Mail>();
         const auto mimeMessage = mail.getMimeMessagePath();
         if (!mimeMessage.isNull()) {
-            mail.setMimeMessagePath(moveMessage(mimeMessage, mail.getFolder()));
+            const auto path = moveMessage(mimeMessage, mail.getFolder());
+            auto blob = ApplicationDomain::BLOB{path};
+            blob.isExternal = false;
+            mail.setProperty(ApplicationDomain::Mail::MimeMessage::name, QVariant::fromValue(blob));
         }
     }
 
@@ -149,7 +152,9 @@ public:
             auto newPath = moveMessage(mimeMessage, newMail.getFolder());
             if (newPath != oldMail.getMimeMessagePath()) {
                 const auto oldPath = getFilePathFromMimeMessagePath(oldMail.getMimeMessagePath());
-                newMail.setMimeMessagePath(newPath);
+                auto blob = ApplicationDomain::BLOB{newPath};
+                blob.isExternal = false;
+                newMail.setProperty(ApplicationDomain::Mail::MimeMessage::name, QVariant::fromValue(blob));
                 //Remove the olde mime message if there is a new one
                 QFile::remove(oldPath);
             }
@@ -323,7 +328,10 @@ public:
             Sink::ApplicationDomain::Mail mail;
             mail.setFolder(folderLocalId);
             //We only store the directory path + key, so we facade can add the changing bits (flags)
-            mail.setMimeMessagePath(KPIM::Maildir::getDirectoryFromFile(filePath) + maildirKey);
+            auto path = KPIM::Maildir::getDirectoryFromFile(filePath) + maildirKey;
+            auto blob = ApplicationDomain::BLOB{path};
+            blob.isExternal = false;
+            mail.setProperty(ApplicationDomain::Mail::MimeMessage::name, QVariant::fromValue(blob));
             mail.setUnread(!flags.testFlag(KPIM::Maildir::Seen));
             mail.setImportant(flags.testFlag(KPIM::Maildir::Flagged));
 
diff --git a/examples/mailtransportresource/mailtransportresource.cpp b/examples/mailtransportresource/mailtransportresource.cpp
index 08a8748..b7ee77a 100644
--- a/examples/mailtransportresource/mailtransportresource.cpp
+++ b/examples/mailtransportresource/mailtransportresource.cpp
@@ -182,7 +182,7 @@ MailtransportResource::MailtransportResource(const Sink::ResourceContext &resour
     setupSynchronizer(synchronizer);
     setupInspector(QSharedPointer<MailtransportInspector>::create(resourceContext));
 
-    setupPreprocessors(ENTITY_TYPE_MAIL, QVector<Sink::Preprocessor*>() << new MimeMessageMover << new MailPropertyExtractor);
+    setupPreprocessors(ENTITY_TYPE_MAIL, QVector<Sink::Preprocessor*>() << new MailPropertyExtractor);
 }
 
 MailtransportResourceFactory::MailtransportResourceFactory(QObject *parent)