Removed all traces of BLOB properties

10 files changed, 5 insertions, 197 deletions

diff --git a/common/domain/applicationdomaintype.cpp b/common/domain/applicationdomaintype.cpp
index c1ac7e8..8bb74e3 100644
--- a/common/domain/applicationdomaintype.cpp
+++ b/common/domain/applicationdomaintype.cpp
@@ -58,12 +58,6 @@ QDebug Sink::ApplicationDomain::operator<< (QDebug d, const Sink::ApplicationDom
     return d;
 }
 
-QDebug Sink::ApplicationDomain::operator<< (QDebug d, const Sink::ApplicationDomain::BLOB &blob)
-{
-    d << blob.value << "external:" << blob.isExternal ;
-    return d;
-}
-
 template <typename DomainType, typename Property>
 int registerProperty() {
     Sink::Private::PropertyRegistry::instance().registerProperty<Property>(Sink::ApplicationDomain::getTypeName<DomainType>());
@@ -130,13 +124,12 @@ static const int foo = [] {
     QMetaType::registerEqualsComparator<Reference>();
     QMetaType::registerDebugStreamOperator<Reference>();
     QMetaType::registerConverter<Reference, QByteArray>();
-    QMetaType::registerDebugStreamOperator<BLOB>();
     QMetaType::registerDebugStreamOperator<Mail::Contact>();
     qRegisterMetaTypeStreamOperators<Sink::ApplicationDomain::Reference>();
     return 0;
 }();
 
-void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::ApplicationDomain::BufferAdaptor &memoryAdaptor, const QList<QByteArray> &properties, bool copyBlobs, bool pruneReferences)
+void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::ApplicationDomain::BufferAdaptor &memoryAdaptor, const QList<QByteArray> &properties, bool pruneReferences)
 {
     auto propertiesToCopy = properties;
     if (properties.isEmpty()) {
@@ -144,14 +137,7 @@ void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::Applicatio
     }
     for (const auto &property : propertiesToCopy) {
         const auto value = buffer.getProperty(property);
-        if (copyBlobs && value.canConvert<BLOB>()) {
-            const auto oldPath = value.value<BLOB>().value;
-            const auto newPath = Sink::temporaryFileLocation() + "/" + createUuid();
-            if (!QFile::copy(oldPath, newPath)) {
-                SinkWarning() << "Failed to copy file from: " << oldPath << "to: " << newPath;
-            }
-            memoryAdaptor.setProperty(property, QVariant::fromValue(BLOB{newPath}));
-        } else if (pruneReferences && value.canConvert<Reference>()) {
+        if (pruneReferences && value.canConvert<Reference>()) {
             continue;
         } else {
             memoryAdaptor.setProperty(property, value);
@@ -245,33 +231,6 @@ void ApplicationDomainType::setProperty(const QByteArray &key, const Application
     setProperty(key, QVariant::fromValue(Reference{value.identifier()}));
 }
 
-QByteArray ApplicationDomainType::getBlobProperty(const QByteArray &key) const
-{
-    const auto path = getProperty(key).value<BLOB>().value;
-    QFile file(path);
-    if (!file.open(QIODevice::ReadOnly)) {
-        SinkError() << "Failed to open the file for reading: " << file.errorString() << "Path:" << path << " For property:" << key;
-        return QByteArray();
-    }
-    return file.readAll();
-}
-
-void ApplicationDomainType::setBlobProperty(const QByteArray &key, const QByteArray &value)
-{
-    const auto path = Sink::temporaryFileLocation() + "/" + createUuid();
-    QFile file(path);
-    if (!file.open(QIODevice::WriteOnly)) {
-        SinkError() << "Failed to open the file for writing: " << file.errorString() << path << " For property " << key;
-        return;
-    }
-    if (file.write(value) < 0) {
-        SinkError() << "Failed to write to file: " << file.errorString() << path << " For property " << key;
-    }
-    //Ensure that the file is written to disk immediately
-    file.close();
-    setProperty(key, QVariant::fromValue(BLOB{path}));
-}
-
 void ApplicationDomainType::setChangedProperties(const QSet<QByteArray> &changeset)
 {
     *mChangeSet = changeset;
diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h
index 8a0daef..b4db54e 100644
--- a/common/domain/applicationdomaintype.h
+++ b/common/domain/applicationdomaintype.h
@@ -62,16 +62,6 @@
     void setStatus##NAME(const TYPE &value) { setProperty(NAME::name, QVariant::fromValue(value)); } \
     TYPE get##NAME() const { return getProperty(NAME::name).value<TYPE>(); } \
 
-#define SINK_BLOB_PROPERTY(NAME, LOWERCASENAME) \
-    struct NAME { \
-        static constexpr const char *name = #LOWERCASENAME; \
-        typedef BLOB Type; \
-    }; \
-    void set##NAME(const QByteArray &value) { setBlobProperty(NAME::name, value); } \
-    void set##NAME##Path(const QString &path) { setProperty(NAME::name, QVariant::fromValue(BLOB{path})); } \
-    QByteArray get##NAME() const { return getBlobProperty(NAME::name); } \
-    QString get##NAME##Path() const { return getProperty(NAME::name).value<BLOB>().value; } \
-
 #define SINK_REFERENCE_PROPERTY(TYPE, NAME, LOWERCASENAME) \
     struct NAME { \
         static constexpr const char *name = #LOWERCASENAME; \
@@ -145,23 +135,6 @@ struct SINK_EXPORT Progress {
 /**
  * Internal type.
  *
- * Represents a BLOB property.
- */
-struct BLOB {
-    BLOB() = default;
-    BLOB(const BLOB &) = default;
-    BLOB(const QString &id) : value(id) {};
-    ~BLOB() = default;
-    bool operator==(const BLOB &other) const {
-        return value == other.value && isExternal == other.isExternal;
-    }
-    QString value;
-    bool isExternal = true;
-};
-
-/**
- * Internal type.
- *
  * Represents a reference to another entity in the same resource.
  */
 struct Reference {
@@ -179,7 +152,7 @@ struct Reference {
     QByteArray value;
 };
 
-void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::ApplicationDomain::BufferAdaptor &memoryAdaptor, const QList<QByteArray> &properties = QList<QByteArray>(), bool copyBlobs = false, bool pruneReferences = false);
+void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::ApplicationDomain::BufferAdaptor &memoryAdaptor, const QList<QByteArray> &properties = QList<QByteArray>(), bool pruneReferences = false);
 
 /**
  * The domain type interface has two purposes:
@@ -223,7 +196,7 @@ public:
     static typename DomainType::Ptr getInMemoryRepresentation(const ApplicationDomainType &domainType, const QList<QByteArray> properties = QList<QByteArray>())
     {
         auto memoryAdaptor = QSharedPointer<Sink::ApplicationDomain::MemoryBufferAdaptor>::create();
-        copyBuffer(*(domainType.mAdaptor), *memoryAdaptor, properties, false, false);
+        copyBuffer(*(domainType.mAdaptor), *memoryAdaptor, properties, false);
         //mIdentifier internally still refers to the memory-mapped memory, we need to copy the memory or it will become invalid
         return QSharedPointer<DomainType>::create(domainType.mResourceInstanceIdentifier, QByteArray(domainType.mIdentifier.constData(), domainType.mIdentifier.size()), domainType.mRevision, memoryAdaptor);
     }
@@ -236,7 +209,7 @@ public:
     {
         auto memoryAdaptor = QSharedPointer<Sink::ApplicationDomain::MemoryBufferAdaptor>::create();
         Q_ASSERT(domainType.mAdaptor);
-        copyBuffer(*(domainType.mAdaptor), *memoryAdaptor, properties, true, true);
+        copyBuffer(*(domainType.mAdaptor), *memoryAdaptor, properties, true);
         return QSharedPointer<DomainType>::create(QByteArray{}, QByteArray{}, 0, memoryAdaptor);
     }
 
@@ -287,9 +260,6 @@ public:
      */
     void setProperty(const QByteArray &key, const ApplicationDomainType &value);
 
-    QByteArray getBlobProperty(const QByteArray &key) const;
-    void setBlobProperty(const QByteArray &key, const QByteArray &value);
-
     void setChangedProperties(const QSet<QByteArray> &changeset);
     QByteArrayList changedProperties() const;
     QByteArrayList availableProperties() const;
@@ -321,7 +291,6 @@ inline bool operator==(const ApplicationDomainType& lhs, const ApplicationDomain
 
 SINK_EXPORT QDebug operator<< (QDebug d, const ApplicationDomainType &type);
 SINK_EXPORT QDebug operator<< (QDebug d, const Reference &ref);
-SINK_EXPORT QDebug operator<< (QDebug d, const BLOB &blob);
 
 
 struct SINK_EXPORT SinkAccount : public ApplicationDomainType {
@@ -439,7 +408,6 @@ struct SINK_EXPORT Mail : public Entity {
     SINK_PROPERTY(bool, Unread, unread);
     SINK_PROPERTY(bool, Important, important);
     SINK_REFERENCE_PROPERTY(Folder, Folder, folder);
-    // SINK_BLOB_PROPERTY(MimeMessage, mimeMessage);
     SINK_PROPERTY(QByteArray, MimeMessage, mimeMessage);
     SINK_EXTRACTED_PROPERTY(bool, FullPayloadAvailable, fullPayloadAvailable);
     SINK_PROPERTY(bool, Draft, draft);
@@ -542,7 +510,6 @@ class SINK_EXPORT TypeImplementation;
 #undef SINK_ENTITY
 #undef SINK_PROPERTY
 #undef SINK_EXTRACTED_PROPERTY
-#undef SINK_BLOB_PROPERTY
 #undef SINK_REFERENCE_PROPERTY
 #undef SINK_INDEX_PROPERTY
 
@@ -576,5 +543,4 @@ Q_DECLARE_METATYPE(Sink::ApplicationDomain::Mail::Contact)
 Q_DECLARE_METATYPE(Sink::ApplicationDomain::Contact::Email)
 Q_DECLARE_METATYPE(Sink::ApplicationDomain::Error)
 Q_DECLARE_METATYPE(Sink::ApplicationDomain::Progress)
-Q_DECLARE_METATYPE(Sink::ApplicationDomain::BLOB)
 Q_DECLARE_METATYPE(Sink::ApplicationDomain::Reference)
diff --git a/common/domain/propertyregistry.cpp b/common/domain/propertyregistry.cpp
index 7b9b61a..755e7b4 100644
--- a/common/domain/propertyregistry.cpp
+++ b/common/domain/propertyregistry.cpp
@@ -46,15 +46,6 @@ QVariant parseString<Sink::ApplicationDomain::Reference>(const QString &s)
 }
 
 template <>
-QVariant parseString<Sink::ApplicationDomain::BLOB>(const QString &s)
-{
-    //TODO copy path
-    // return QVariant::fromValue(Sink::ApplicationDomain::BLOB{s.toLatin1()});
-    Q_ASSERT(false);
-    return QVariant{};
-}
-
-template <>
 QVariant parseString<bool>(const QString &s)
 {
     if (s == "true") {
diff --git a/common/domain/propertyregistry.h b/common/domain/propertyregistry.h
index 758c10d..be63989 100644
--- a/common/domain/propertyregistry.h
+++ b/common/domain/propertyregistry.h
@@ -43,9 +43,6 @@ template <>
 QVariant parseString<Sink::ApplicationDomain::Reference>(const QString &s);
 
 template <>
-QVariant parseString<Sink::ApplicationDomain::BLOB>(const QString &s);
-
-template <>
 QVariant parseString<bool>(const QString &s);
 
 template <>
diff --git a/common/propertymapper.cpp b/common/propertymapper.cpp
index dbf93a3..7c313cc 100644
--- a/common/propertymapper.cpp
+++ b/common/propertymapper.cpp
@@ -35,17 +35,6 @@ flatbuffers::uoffset_t variantToProperty<QString>(const QVariant &property, flat
 }
 
 template <>
-flatbuffers::uoffset_t variantToProperty<Sink::ApplicationDomain::BLOB>(const QVariant &property, flatbuffers::FlatBufferBuilder &fbb)
-{
-    if (property.isValid()) {
-        const auto blob = property.value<Sink::ApplicationDomain::BLOB>();
-        auto s = blob.value + (blob.isExternal ? ":ext" : ":int");
-        return fbb.CreateString(s.toStdString()).o;
-    }
-    return 0;
-}
-
-template <>
 flatbuffers::uoffset_t variantToProperty<Sink::ApplicationDomain::Reference>(const QVariant &property, flatbuffers::FlatBufferBuilder &fbb)
 {
     if (property.isValid()) {
@@ -153,22 +142,6 @@ QVariant propertyToVariant<QString>(const flatbuffers::String *property)
 }
 
 template <>
-QVariant propertyToVariant<Sink::ApplicationDomain::BLOB>(const flatbuffers::String *property)
-{
-    if (property) {
-        // We have to copy the memory, otherwise it would become eventually invalid
-        auto s = QString::fromStdString(property->str());
-        auto ext = s.endsWith(":ext");
-        s.chop(4);
-
-        auto blob = Sink::ApplicationDomain::BLOB{s};
-        blob.isExternal = ext;
-        return QVariant::fromValue(blob);
-    }
-    return QVariant();
-}
-
-template <>
 QVariant propertyToVariant<Sink::ApplicationDomain::Reference>(const flatbuffers::String *property)
 {
     if (property) {
diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp
index 5514e31..4ad3eaf 100644
--- a/common/storage/entitystore.cpp
+++ b/common/storage/entitystore.cpp
@@ -96,14 +96,6 @@ class EntityStore::Private {
 public:
     Private(const ResourceContext &context, const Sink::Log::Context &ctx) : resourceContext(context), logCtx(ctx.subContext("entitystore"))
     {
-        static bool initialized = false;
-        if (!initialized) {
-            if (QDir{}.mkpath(entityBlobStorageDir())) {
-                initialized = true;
-            } else {
-                SinkWarningCtx(logCtx) << "Failed to create the directory: " << entityBlobStorageDir();
-            }
-        }
     }
 
     ResourceContext resourceContext;
@@ -158,17 +150,6 @@ public:
         auto adaptor = resourceContext.adaptorFactory(type).createAdaptor(buffer.entity(), &typeIndex(type));
         return ApplicationDomain::ApplicationDomainType{resourceContext.instanceId(), uid, revision, adaptor};
     }
-
-    QString entityBlobStorageDir()
-    {
-        return Sink::resourceStorageLocation(resourceContext.instanceId()) + "/blob";
-    }
-
-    QString entityBlobStoragePath(const QByteArray &id)
-    {
-        return entityBlobStorageDir() +"/"+ id;
-    }
-
 };
 
 EntityStore::EntityStore(const ResourceContext &context, const Log::Context &ctx)
@@ -205,35 +186,6 @@ bool EntityStore::hasTransaction() const
     return d->transaction;
 }
 
-void EntityStore::copyBlobs(ApplicationDomain::ApplicationDomainType &entity, qint64 newRevision)
-{
-    const auto directory = d->entityBlobStoragePath(entity.identifier());
-
-    for (const auto &property : entity.changedProperties()) {
-        const auto value = entity.getProperty(property);
-        if (value.canConvert<ApplicationDomain::BLOB>()) {
-            const auto blob = value.value<ApplicationDomain::BLOB>();
-            //Any blob that is not part of the storage yet has to be moved there.
-            if (blob.isExternal) {
-                auto oldPath = blob.value;
-                auto filePath = directory + QString("_%1%2.blob").arg(QString::number(newRevision)).arg(QString::fromLatin1(property));
-                //In case we hit the same revision again due to a rollback.
-                QFile::remove(filePath);
-                QFile origFile(oldPath);
-                if (!origFile.open(QIODevice::ReadWrite)) {
-                    SinkWarningCtx(d->logCtx) << "Failed to open the original file with write rights: " << origFile.errorString();
-                }
-                if (!origFile.rename(filePath)) {
-                    SinkWarningCtx(d->logCtx) << "Failed to move the file from: " << oldPath << " to " << filePath << ". " << origFile.errorString();
-                }
-                origFile.close();
-                SinkTraceCtx(d->logCtx) << "Moved blob property: " << property << " from " << oldPath << "to" << filePath;
-                entity.setProperty(property, QVariant::fromValue(ApplicationDomain::BLOB{filePath}));
-            }
-        }
-    }
-}
-
 bool EntityStore::add(const QByteArray &type, ApplicationDomain::ApplicationDomainType entity, bool replayToSource)
 {
     if (entity.identifier().isEmpty()) {
@@ -248,8 +200,6 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomain::ApplicationDoma
     //The maxRevision may have changed meanwhile if the entity created sub-entities
     const qint64 newRevision = maxRevision() + 1;
 
-    copyBlobs(entity, newRevision);
-
     // Add metadata buffer
     flatbuffers::FlatBufferBuilder metadataFbb;
     auto metadataBuilder = MetadataBuilder(metadataFbb);
@@ -317,8 +267,6 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::Applic
 
     const qint64 newRevision = DataStore::maxRevision(d->transaction) + 1;
 
-    copyBlobs(newEntity, newRevision);
-
     // Add metadata buffer
     flatbuffers::FlatBufferBuilder metadataFbb;
     {
@@ -408,13 +356,6 @@ void EntityStore::cleanupEntityRevisionsUntil(qint64 revision)
                         DataStore::removeRevision(d->transaction, rev);
                         DataStore::mainDatabase(d->transaction, bufferType).remove(key);
                     }
-                    if (isRemoval) {
-                        QDir dir{d->entityBlobStorageDir()};
-                        const auto infoList = dir.entryInfoList(QStringList{} << QString{uid + "*"});
-                        for (const auto &fileInfo : infoList) {
-                            QFile::remove(fileInfo.filePath());
-                        }
-                    }
                     //Don't cleanup more than specified
                     if (rev >= revision) {
                         return false;
diff --git a/common/store.cpp b/common/store.cpp
index a2204fc..edca842 100644
--- a/common/store.cpp
+++ b/common/store.cpp
@@ -71,11 +71,6 @@ QString Store::storageLocation()
     return Sink::storageLocation();
 }
 
-QString Store::getTemporaryFilePath()
-{
-    return Sink::temporaryFileLocation() + "/" + createUuid();
-}
-
 
 template <class DomainType>
 KAsync::Job<void> queryResource(const QByteArray resourceType, const QByteArray &resourceInstanceIdentifier, const Query &query, typename AggregatingResultEmitter<typename DomainType::Ptr>::Ptr aggregatingEmitter, const Sink::Log::Context &ctx_)
diff --git a/common/store.h b/common/store.h
index 3ad547e..62858c5 100644
--- a/common/store.h
+++ b/common/store.h
@@ -43,11 +43,6 @@ namespace Store {
 
 QString SINK_EXPORT storageLocation();
 
-/**
- * Returns a unique path for a writable file that can be used to write BLOB properties.
- */
-QString SINK_EXPORT getTemporaryFilePath();
-
 // Must be the same as in ModelResult
 enum Roles
 {
diff --git a/sinksh/syntax_modules/sink_list.cpp b/sinksh/syntax_modules/sink_list.cpp
index e4300f1..8da921f 100644
--- a/sinksh/syntax_modules/sink_list.cpp
+++ b/sinksh/syntax_modules/sink_list.cpp
@@ -73,8 +73,6 @@ QStringList printToList(const Sink::ApplicationDomain::ApplicationDomainType &o,
         if (value.isValid()) {
             if (value.canConvert<Sink::ApplicationDomain::Reference>()) {
                 line << compressId(compact, value.toByteArray());
-            } else if (value.canConvert<Sink::ApplicationDomain::BLOB>()) {
-                line << value.value<Sink::ApplicationDomain::BLOB>().value;
             } else if (value.canConvert<QString>()) {
                 line << value.toString().mid(0, 75);
             } else if (value.canConvert<QByteArray>()) {
diff --git a/sinksh/syntax_modules/sink_stat.cpp b/sinksh/syntax_modules/sink_stat.cpp
index b4487a8..bb7fc08 100644
--- a/sinksh/syntax_modules/sink_stat.cpp
+++ b/sinksh/syntax_modules/sink_stat.cpp
@@ -68,13 +68,6 @@ void statResource(const QString &resource, const State &state)
     auto size = diskUsage / 1024;
     state.printLine(QObject::tr("Actual database file sizes total: %1 [kb]").arg(size), 1);
 
-    QDir dataDir{Sink::resourceStorageLocation(resource.toLatin1()) + "/blob/"};
-    Q_ASSERT(dataDir.exists());
-    qint64 dataSize = 0;
-    for (const auto &e : dataDir.entryInfoList(QDir::Files | QDir::NoSymLinks | QDir::NoDotAndDotDot)) {
-        dataSize += e.size();
-    }
-    state.printLine(QObject::tr("Total BLOB size [kb]: %1").arg(dataSize / 1024), 1);
     state.printLine();
 }