From 95854555f038708ddb3fe2e7244deaa57e9a4b8c Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 6 Feb 2018 08:18:38 +0100 Subject: Removed all traces of BLOB properties --- common/domain/applicationdomaintype.cpp | 45 ++----------------------- common/domain/applicationdomaintype.h | 40 ++-------------------- common/domain/propertyregistry.cpp | 9 ----- common/domain/propertyregistry.h | 3 -- common/propertymapper.cpp | 27 --------------- common/storage/entitystore.cpp | 59 --------------------------------- common/store.cpp | 5 --- common/store.h | 5 --- sinksh/syntax_modules/sink_list.cpp | 2 -- sinksh/syntax_modules/sink_stat.cpp | 7 ---- 10 files changed, 5 insertions(+), 197 deletions(-) diff --git a/common/domain/applicationdomaintype.cpp b/common/domain/applicationdomaintype.cpp index c1ac7e8..8bb74e3 100644 --- a/common/domain/applicationdomaintype.cpp +++ b/common/domain/applicationdomaintype.cpp @@ -58,12 +58,6 @@ QDebug Sink::ApplicationDomain::operator<< (QDebug d, const Sink::ApplicationDom return d; } -QDebug Sink::ApplicationDomain::operator<< (QDebug d, const Sink::ApplicationDomain::BLOB &blob) -{ - d << blob.value << "external:" << blob.isExternal ; - return d; -} - template int registerProperty() { Sink::Private::PropertyRegistry::instance().registerProperty(Sink::ApplicationDomain::getTypeName()); @@ -130,13 +124,12 @@ static const int foo = [] { QMetaType::registerEqualsComparator(); QMetaType::registerDebugStreamOperator(); QMetaType::registerConverter(); - QMetaType::registerDebugStreamOperator(); QMetaType::registerDebugStreamOperator(); qRegisterMetaTypeStreamOperators(); return 0; }(); -void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::ApplicationDomain::BufferAdaptor &memoryAdaptor, const QList &properties, bool copyBlobs, bool pruneReferences) +void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::ApplicationDomain::BufferAdaptor &memoryAdaptor, const QList &properties, bool pruneReferences) { auto propertiesToCopy = properties; if (properties.isEmpty()) { @@ -144,14 +137,7 @@ void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::Applicatio } for (const auto &property : propertiesToCopy) { const auto value = buffer.getProperty(property); - if (copyBlobs && value.canConvert()) { - const auto oldPath = value.value().value; - const auto newPath = Sink::temporaryFileLocation() + "/" + createUuid(); - if (!QFile::copy(oldPath, newPath)) { - SinkWarning() << "Failed to copy file from: " << oldPath << "to: " << newPath; - } - memoryAdaptor.setProperty(property, QVariant::fromValue(BLOB{newPath})); - } else if (pruneReferences && value.canConvert()) { + if (pruneReferences && value.canConvert()) { continue; } else { memoryAdaptor.setProperty(property, value); @@ -245,33 +231,6 @@ void ApplicationDomainType::setProperty(const QByteArray &key, const Application setProperty(key, QVariant::fromValue(Reference{value.identifier()})); } -QByteArray ApplicationDomainType::getBlobProperty(const QByteArray &key) const -{ - const auto path = getProperty(key).value().value; - QFile file(path); - if (!file.open(QIODevice::ReadOnly)) { - SinkError() << "Failed to open the file for reading: " << file.errorString() << "Path:" << path << " For property:" << key; - return QByteArray(); - } - return file.readAll(); -} - -void ApplicationDomainType::setBlobProperty(const QByteArray &key, const QByteArray &value) -{ - const auto path = Sink::temporaryFileLocation() + "/" + createUuid(); - QFile file(path); - if (!file.open(QIODevice::WriteOnly)) { - SinkError() << "Failed to open the file for writing: " << file.errorString() << path << " For property " << key; - return; - } - if (file.write(value) < 0) { - SinkError() << "Failed to write to file: " << file.errorString() << path << " For property " << key; - } - //Ensure that the file is written to disk immediately - file.close(); - setProperty(key, QVariant::fromValue(BLOB{path})); -} - void ApplicationDomainType::setChangedProperties(const QSet &changeset) { *mChangeSet = changeset; diff --git a/common/domain/applicationdomaintype.h b/common/domain/applicationdomaintype.h index 8a0daef..b4db54e 100644 --- a/common/domain/applicationdomaintype.h +++ b/common/domain/applicationdomaintype.h @@ -62,16 +62,6 @@ void setStatus##NAME(const TYPE &value) { setProperty(NAME::name, QVariant::fromValue(value)); } \ TYPE get##NAME() const { return getProperty(NAME::name).value(); } \ -#define SINK_BLOB_PROPERTY(NAME, LOWERCASENAME) \ - struct NAME { \ - static constexpr const char *name = #LOWERCASENAME; \ - typedef BLOB Type; \ - }; \ - void set##NAME(const QByteArray &value) { setBlobProperty(NAME::name, value); } \ - void set##NAME##Path(const QString &path) { setProperty(NAME::name, QVariant::fromValue(BLOB{path})); } \ - QByteArray get##NAME() const { return getBlobProperty(NAME::name); } \ - QString get##NAME##Path() const { return getProperty(NAME::name).value().value; } \ - #define SINK_REFERENCE_PROPERTY(TYPE, NAME, LOWERCASENAME) \ struct NAME { \ static constexpr const char *name = #LOWERCASENAME; \ @@ -142,23 +132,6 @@ struct SINK_EXPORT Progress { }; -/** - * Internal type. - * - * Represents a BLOB property. - */ -struct BLOB { - BLOB() = default; - BLOB(const BLOB &) = default; - BLOB(const QString &id) : value(id) {}; - ~BLOB() = default; - bool operator==(const BLOB &other) const { - return value == other.value && isExternal == other.isExternal; - } - QString value; - bool isExternal = true; -}; - /** * Internal type. * @@ -179,7 +152,7 @@ struct Reference { QByteArray value; }; -void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::ApplicationDomain::BufferAdaptor &memoryAdaptor, const QList &properties = QList(), bool copyBlobs = false, bool pruneReferences = false); +void copyBuffer(Sink::ApplicationDomain::BufferAdaptor &buffer, Sink::ApplicationDomain::BufferAdaptor &memoryAdaptor, const QList &properties = QList(), bool pruneReferences = false); /** * The domain type interface has two purposes: @@ -223,7 +196,7 @@ public: static typename DomainType::Ptr getInMemoryRepresentation(const ApplicationDomainType &domainType, const QList properties = QList()) { auto memoryAdaptor = QSharedPointer::create(); - copyBuffer(*(domainType.mAdaptor), *memoryAdaptor, properties, false, false); + copyBuffer(*(domainType.mAdaptor), *memoryAdaptor, properties, false); //mIdentifier internally still refers to the memory-mapped memory, we need to copy the memory or it will become invalid return QSharedPointer::create(domainType.mResourceInstanceIdentifier, QByteArray(domainType.mIdentifier.constData(), domainType.mIdentifier.size()), domainType.mRevision, memoryAdaptor); } @@ -236,7 +209,7 @@ public: { auto memoryAdaptor = QSharedPointer::create(); Q_ASSERT(domainType.mAdaptor); - copyBuffer(*(domainType.mAdaptor), *memoryAdaptor, properties, true, true); + copyBuffer(*(domainType.mAdaptor), *memoryAdaptor, properties, true); return QSharedPointer::create(QByteArray{}, QByteArray{}, 0, memoryAdaptor); } @@ -287,9 +260,6 @@ public: */ void setProperty(const QByteArray &key, const ApplicationDomainType &value); - QByteArray getBlobProperty(const QByteArray &key) const; - void setBlobProperty(const QByteArray &key, const QByteArray &value); - void setChangedProperties(const QSet &changeset); QByteArrayList changedProperties() const; QByteArrayList availableProperties() const; @@ -321,7 +291,6 @@ inline bool operator==(const ApplicationDomainType& lhs, const ApplicationDomain SINK_EXPORT QDebug operator<< (QDebug d, const ApplicationDomainType &type); SINK_EXPORT QDebug operator<< (QDebug d, const Reference &ref); -SINK_EXPORT QDebug operator<< (QDebug d, const BLOB &blob); struct SINK_EXPORT SinkAccount : public ApplicationDomainType { @@ -439,7 +408,6 @@ struct SINK_EXPORT Mail : public Entity { SINK_PROPERTY(bool, Unread, unread); SINK_PROPERTY(bool, Important, important); SINK_REFERENCE_PROPERTY(Folder, Folder, folder); - // SINK_BLOB_PROPERTY(MimeMessage, mimeMessage); SINK_PROPERTY(QByteArray, MimeMessage, mimeMessage); SINK_EXTRACTED_PROPERTY(bool, FullPayloadAvailable, fullPayloadAvailable); SINK_PROPERTY(bool, Draft, draft); @@ -542,7 +510,6 @@ class SINK_EXPORT TypeImplementation; #undef SINK_ENTITY #undef SINK_PROPERTY #undef SINK_EXTRACTED_PROPERTY -#undef SINK_BLOB_PROPERTY #undef SINK_REFERENCE_PROPERTY #undef SINK_INDEX_PROPERTY @@ -576,5 +543,4 @@ Q_DECLARE_METATYPE(Sink::ApplicationDomain::Mail::Contact) Q_DECLARE_METATYPE(Sink::ApplicationDomain::Contact::Email) Q_DECLARE_METATYPE(Sink::ApplicationDomain::Error) Q_DECLARE_METATYPE(Sink::ApplicationDomain::Progress) -Q_DECLARE_METATYPE(Sink::ApplicationDomain::BLOB) Q_DECLARE_METATYPE(Sink::ApplicationDomain::Reference) diff --git a/common/domain/propertyregistry.cpp b/common/domain/propertyregistry.cpp index 7b9b61a..755e7b4 100644 --- a/common/domain/propertyregistry.cpp +++ b/common/domain/propertyregistry.cpp @@ -45,15 +45,6 @@ QVariant parseString(const QString &s) return QVariant::fromValue(Sink::ApplicationDomain::Reference{s.toLatin1()}); } -template <> -QVariant parseString(const QString &s) -{ - //TODO copy path - // return QVariant::fromValue(Sink::ApplicationDomain::BLOB{s.toLatin1()}); - Q_ASSERT(false); - return QVariant{}; -} - template <> QVariant parseString(const QString &s) { diff --git a/common/domain/propertyregistry.h b/common/domain/propertyregistry.h index 758c10d..be63989 100644 --- a/common/domain/propertyregistry.h +++ b/common/domain/propertyregistry.h @@ -42,9 +42,6 @@ QVariant parseString(const QString &s); template <> QVariant parseString(const QString &s); -template <> -QVariant parseString(const QString &s); - template <> QVariant parseString(const QString &s); diff --git a/common/propertymapper.cpp b/common/propertymapper.cpp index dbf93a3..7c313cc 100644 --- a/common/propertymapper.cpp +++ b/common/propertymapper.cpp @@ -34,17 +34,6 @@ flatbuffers::uoffset_t variantToProperty(const QVariant &property, flat return 0; } -template <> -flatbuffers::uoffset_t variantToProperty(const QVariant &property, flatbuffers::FlatBufferBuilder &fbb) -{ - if (property.isValid()) { - const auto blob = property.value(); - auto s = blob.value + (blob.isExternal ? ":ext" : ":int"); - return fbb.CreateString(s.toStdString()).o; - } - return 0; -} - template <> flatbuffers::uoffset_t variantToProperty(const QVariant &property, flatbuffers::FlatBufferBuilder &fbb) { @@ -152,22 +141,6 @@ QVariant propertyToVariant(const flatbuffers::String *property) return QVariant(); } -template <> -QVariant propertyToVariant(const flatbuffers::String *property) -{ - if (property) { - // We have to copy the memory, otherwise it would become eventually invalid - auto s = QString::fromStdString(property->str()); - auto ext = s.endsWith(":ext"); - s.chop(4); - - auto blob = Sink::ApplicationDomain::BLOB{s}; - blob.isExternal = ext; - return QVariant::fromValue(blob); - } - return QVariant(); -} - template <> QVariant propertyToVariant(const flatbuffers::String *property) { diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 5514e31..4ad3eaf 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -96,14 +96,6 @@ class EntityStore::Private { public: Private(const ResourceContext &context, const Sink::Log::Context &ctx) : resourceContext(context), logCtx(ctx.subContext("entitystore")) { - static bool initialized = false; - if (!initialized) { - if (QDir{}.mkpath(entityBlobStorageDir())) { - initialized = true; - } else { - SinkWarningCtx(logCtx) << "Failed to create the directory: " << entityBlobStorageDir(); - } - } } ResourceContext resourceContext; @@ -158,17 +150,6 @@ public: auto adaptor = resourceContext.adaptorFactory(type).createAdaptor(buffer.entity(), &typeIndex(type)); return ApplicationDomain::ApplicationDomainType{resourceContext.instanceId(), uid, revision, adaptor}; } - - QString entityBlobStorageDir() - { - return Sink::resourceStorageLocation(resourceContext.instanceId()) + "/blob"; - } - - QString entityBlobStoragePath(const QByteArray &id) - { - return entityBlobStorageDir() +"/"+ id; - } - }; EntityStore::EntityStore(const ResourceContext &context, const Log::Context &ctx) @@ -205,35 +186,6 @@ bool EntityStore::hasTransaction() const return d->transaction; } -void EntityStore::copyBlobs(ApplicationDomain::ApplicationDomainType &entity, qint64 newRevision) -{ - const auto directory = d->entityBlobStoragePath(entity.identifier()); - - for (const auto &property : entity.changedProperties()) { - const auto value = entity.getProperty(property); - if (value.canConvert()) { - const auto blob = value.value(); - //Any blob that is not part of the storage yet has to be moved there. - if (blob.isExternal) { - auto oldPath = blob.value; - auto filePath = directory + QString("_%1%2.blob").arg(QString::number(newRevision)).arg(QString::fromLatin1(property)); - //In case we hit the same revision again due to a rollback. - QFile::remove(filePath); - QFile origFile(oldPath); - if (!origFile.open(QIODevice::ReadWrite)) { - SinkWarningCtx(d->logCtx) << "Failed to open the original file with write rights: " << origFile.errorString(); - } - if (!origFile.rename(filePath)) { - SinkWarningCtx(d->logCtx) << "Failed to move the file from: " << oldPath << " to " << filePath << ". " << origFile.errorString(); - } - origFile.close(); - SinkTraceCtx(d->logCtx) << "Moved blob property: " << property << " from " << oldPath << "to" << filePath; - entity.setProperty(property, QVariant::fromValue(ApplicationDomain::BLOB{filePath})); - } - } - } -} - bool EntityStore::add(const QByteArray &type, ApplicationDomain::ApplicationDomainType entity, bool replayToSource) { if (entity.identifier().isEmpty()) { @@ -248,8 +200,6 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomain::ApplicationDoma //The maxRevision may have changed meanwhile if the entity created sub-entities const qint64 newRevision = maxRevision() + 1; - copyBlobs(entity, newRevision); - // Add metadata buffer flatbuffers::FlatBufferBuilder metadataFbb; auto metadataBuilder = MetadataBuilder(metadataFbb); @@ -317,8 +267,6 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::Applic const qint64 newRevision = DataStore::maxRevision(d->transaction) + 1; - copyBlobs(newEntity, newRevision); - // Add metadata buffer flatbuffers::FlatBufferBuilder metadataFbb; { @@ -408,13 +356,6 @@ void EntityStore::cleanupEntityRevisionsUntil(qint64 revision) DataStore::removeRevision(d->transaction, rev); DataStore::mainDatabase(d->transaction, bufferType).remove(key); } - if (isRemoval) { - QDir dir{d->entityBlobStorageDir()}; - const auto infoList = dir.entryInfoList(QStringList{} << QString{uid + "*"}); - for (const auto &fileInfo : infoList) { - QFile::remove(fileInfo.filePath()); - } - } //Don't cleanup more than specified if (rev >= revision) { return false; diff --git a/common/store.cpp b/common/store.cpp index a2204fc..edca842 100644 --- a/common/store.cpp +++ b/common/store.cpp @@ -71,11 +71,6 @@ QString Store::storageLocation() return Sink::storageLocation(); } -QString Store::getTemporaryFilePath() -{ - return Sink::temporaryFileLocation() + "/" + createUuid(); -} - template KAsync::Job queryResource(const QByteArray resourceType, const QByteArray &resourceInstanceIdentifier, const Query &query, typename AggregatingResultEmitter::Ptr aggregatingEmitter, const Sink::Log::Context &ctx_) diff --git a/common/store.h b/common/store.h index 3ad547e..62858c5 100644 --- a/common/store.h +++ b/common/store.h @@ -43,11 +43,6 @@ namespace Store { QString SINK_EXPORT storageLocation(); -/** - * Returns a unique path for a writable file that can be used to write BLOB properties. - */ -QString SINK_EXPORT getTemporaryFilePath(); - // Must be the same as in ModelResult enum Roles { diff --git a/sinksh/syntax_modules/sink_list.cpp b/sinksh/syntax_modules/sink_list.cpp index e4300f1..8da921f 100644 --- a/sinksh/syntax_modules/sink_list.cpp +++ b/sinksh/syntax_modules/sink_list.cpp @@ -73,8 +73,6 @@ QStringList printToList(const Sink::ApplicationDomain::ApplicationDomainType &o, if (value.isValid()) { if (value.canConvert()) { line << compressId(compact, value.toByteArray()); - } else if (value.canConvert()) { - line << value.value().value; } else if (value.canConvert()) { line << value.toString().mid(0, 75); } else if (value.canConvert()) { diff --git a/sinksh/syntax_modules/sink_stat.cpp b/sinksh/syntax_modules/sink_stat.cpp index b4487a8..bb7fc08 100644 --- a/sinksh/syntax_modules/sink_stat.cpp +++ b/sinksh/syntax_modules/sink_stat.cpp @@ -68,13 +68,6 @@ void statResource(const QString &resource, const State &state) auto size = diskUsage / 1024; state.printLine(QObject::tr("Actual database file sizes total: %1 [kb]").arg(size), 1); - QDir dataDir{Sink::resourceStorageLocation(resource.toLatin1()) + "/blob/"}; - Q_ASSERT(dataDir.exists()); - qint64 dataSize = 0; - for (const auto &e : dataDir.entryInfoList(QDir::Files | QDir::NoSymLinks | QDir::NoDotAndDotDot)) { - dataSize += e.size(); - } - state.printLine(QObject::tr("Total BLOB size [kb]: %1").arg(dataSize / 1024), 1); state.printLine(); } -- cgit v1.2.3