1 files changed, 64 insertions, 63 deletions

diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp
index 909f1b2..b7309ab 100644
--- a/common/storage/entitystore.cpp
+++ b/common/storage/entitystore.cpp
@@ -31,8 +31,7 @@
 #include "bufferutils.h"
 #include "entity_generated.h"
 #include "applicationdomaintype_p.h"
-
-#include "domaintypes.h"
+#include "typeimplementations.h"
 
 using namespace Sink;
 using namespace Sink::Storage;
@@ -168,19 +167,15 @@ void EntityStore::copyBlobs(ApplicationDomain::ApplicationDomainType &entity, qi
     }
 }
 
-bool EntityStore::add(const QByteArray &type, const ApplicationDomain::ApplicationDomainType &entity_, bool replayToSource, const PreprocessCreation &preprocess)
+bool EntityStore::add(const QByteArray &type, ApplicationDomain::ApplicationDomainType entity, bool replayToSource)
 {
-    if (entity_.identifier().isEmpty()) {
+    if (entity.identifier().isEmpty()) {
         SinkWarningCtx(d->logCtx) << "Can't write entity with an empty identifier";
         return false;
     }
 
-    auto entity = *ApplicationDomain::ApplicationDomainType::getInMemoryRepresentation<ApplicationDomain::ApplicationDomainType>(entity_, entity_.availableProperties());
-    entity.setChangedProperties(entity.availableProperties().toSet());
-
     SinkTraceCtx(d->logCtx) << "New entity " << entity;
 
-    preprocess(entity);
     d->typeIndex(type).add(entity.identifier(), entity, d->transaction);
 
     //The maxRevision may have changed meanwhile if the entity created sub-entities
@@ -205,26 +200,20 @@ bool EntityStore::add(const QByteArray &type, const ApplicationDomain::Applicati
             [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << entity.identifier() << newRevision; });
     DataStore::setMaxRevision(d->transaction, newRevision);
     DataStore::recordRevision(d->transaction, newRevision, entity.identifier(), type);
+    DataStore::recordUid(d->transaction, entity.identifier());
     SinkTraceCtx(d->logCtx) << "Wrote entity: " << entity.identifier() << type << newRevision;
     return true;
 }
 
-bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::ApplicationDomainType &diff, const QByteArrayList &deletions, bool replayToSource, const PreprocessModification &preprocess)
+ApplicationDomain::ApplicationDomainType EntityStore::applyDiff(const QByteArray &type, const ApplicationDomain::ApplicationDomainType &current, const ApplicationDomain::ApplicationDomainType &diff, const QByteArrayList &deletions) const
 {
-    auto changeset = diff.changedProperties();
-    const auto current = readLatest(type, diff.identifier());
-    if (current.identifier().isEmpty()) {
-        SinkWarningCtx(d->logCtx) << "Failed to read current version: " << diff.identifier();
-        return false;
-    }
-
     auto newEntity = *ApplicationDomain::ApplicationDomainType::getInMemoryRepresentation<ApplicationDomain::ApplicationDomainType>(current, current.availableProperties());
 
     SinkTraceCtx(d->logCtx) << "Modified entity: " << newEntity;
 
     // Apply diff
     //SinkTrace() << "Applying changed properties: " << changeset;
-    for (const auto &property : changeset) {
+    for (const auto &property : diff.changedProperties()) {
         const auto value = diff.getProperty(property);
         if (value.isValid()) {
             //SinkTrace() << "Setting property: " << property;
@@ -237,8 +226,25 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::Applic
         //SinkTrace() << "Removing property: " << property;
         newEntity.setProperty(property, QVariant());
     }
+    return newEntity;
+}
+
+bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::ApplicationDomainType &diff, const QByteArrayList &deletions, bool replayToSource)
+{
+    const auto current = readLatest(type, diff.identifier());
+    if (current.identifier().isEmpty()) {
+        SinkWarningCtx(d->logCtx) << "Failed to read current version: " << diff.identifier();
+        return false;
+    }
+
+    auto newEntity = applyDiff(type, current, diff, deletions);
+    return modify(type, current, newEntity, replayToSource);
+}
+
+bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::ApplicationDomainType &current, ApplicationDomain::ApplicationDomainType newEntity, bool replayToSource)
+{
+    SinkTraceCtx(d->logCtx) << "Modified entity: " << newEntity;
 
-    preprocess(current, newEntity);
     d->typeIndex(type).remove(current.identifier(), current, d->transaction);
     d->typeIndex(type).add(newEntity.identifier(), newEntity, d->transaction);
 
@@ -250,7 +256,7 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::Applic
     flatbuffers::FlatBufferBuilder metadataFbb;
     {
         //We add availableProperties to account for the properties that have been changed by the preprocessors
-        auto modifiedProperties = BufferUtils::toVector(metadataFbb, changeset + newEntity.changedProperties());
+        auto modifiedProperties = BufferUtils::toVector(metadataFbb, newEntity.changedProperties());
         auto metadataBuilder = MetadataBuilder(metadataFbb);
         metadataBuilder.add_revision(newRevision);
         metadataBuilder.add_operation(Operation_Modification);
@@ -259,7 +265,7 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::Applic
         auto metadataBuffer = metadataBuilder.Finish();
         FinishMetadataBuffer(metadataFbb, metadataBuffer);
     }
-    SinkTraceCtx(d->logCtx) << "Changed properties: " << changeset + newEntity.changedProperties();
+    SinkTraceCtx(d->logCtx) << "Changed properties: " << newEntity.changedProperties();
 
     newEntity.setChangedProperties(newEntity.availableProperties().toSet());
 
@@ -275,36 +281,14 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomain::Applic
     return true;
 }
 
-bool EntityStore::remove(const QByteArray &type, const QByteArray &uid, bool replayToSource, const PreprocessRemoval &preprocess)
+bool EntityStore::remove(const QByteArray &type, const Sink::ApplicationDomain::ApplicationDomainType &current, bool replayToSource)
 {
-    bool found = false;
-    bool alreadyRemoved = false;
-    DataStore::mainDatabase(d->transaction, type)
-        .findLatest(uid,
-            [&found, &alreadyRemoved](const QByteArray &key, const QByteArray &data) -> bool {
-                auto entity = GetEntity(data.data());
-                if (entity && entity->metadata()) {
-                    auto metadata = GetMetadata(entity->metadata()->Data());
-                    found = true;
-                    if (metadata->operation() == Operation_Removal) {
-                        alreadyRemoved = true;
-                    }
-                }
-                return false;
-            },
-            [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to read old revision from storage: " << error.message; });
-
-    if (!found) {
-        SinkWarningCtx(d->logCtx) << "Remove: Failed to find entity " << uid;
-        return false;
-    }
-    if (alreadyRemoved) {
+    const auto uid = current.identifier();
+    if (!exists(type, uid)) {
         SinkWarningCtx(d->logCtx) << "Remove: Entity is already removed " << uid;
         return false;
     }
 
-    const auto current = readLatest(type, uid);
-    preprocess(current);
     d->typeIndex(type).remove(current.identifier(), current, d->transaction);
 
     SinkTraceCtx(d->logCtx) << "Removed entity " << current;
@@ -328,6 +312,7 @@ bool EntityStore::remove(const QByteArray &type, const QByteArray &uid, bool rep
             [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << uid << newRevision; });
     DataStore::setMaxRevision(d->transaction, newRevision);
     DataStore::recordRevision(d->transaction, newRevision, uid, type);
+    DataStore::removeUid(d->transaction, uid);
     return true;
 }
 
@@ -521,15 +506,9 @@ ApplicationDomain::ApplicationDomainType EntityStore::readEntity(const QByteArra
 
 void EntityStore::readAll(const QByteArray &type, const std::function<void(const ApplicationDomain::ApplicationDomainType &entity)> &callback)
 {
-    auto db = DataStore::mainDatabase(d->getTransaction(), type);
-    db.scan("",
-        [=](const QByteArray &key, const QByteArray &value) -> bool {
-            auto uid = DataStore::uidFromKey(key);
-            auto buffer = Sink::EntityBuffer{value.data(), value.size()};
-            callback(d->createApplicationDomainType(type, uid, DataStore::maxRevision(d->getTransaction()), buffer));
-            return true;
-        },
-        [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during query: " << error.message; });
+    readAllUids(type, [&] (const QByteArray &uid) {
+        readLatest(type, uid, callback);
+    });
 }
 
 void EntityStore::readRevisions(qint64 baseRevision, const QByteArray &expectedType, const std::function<void(const QByteArray &key)> &callback)
@@ -588,15 +567,7 @@ ApplicationDomain::ApplicationDomainType EntityStore::readPrevious(const QByteAr
 
 void EntityStore::readAllUids(const QByteArray &type, const std::function<void(const QByteArray &uid)> callback)
 {
-    //TODO use uid index instead
-    //FIXME we currently report each uid for every revision with the same uid
-    auto db = DataStore::mainDatabase(d->getTransaction(), type);
-    db.scan("",
-        [callback](const QByteArray &key, const QByteArray &) -> bool {
-            callback(Sink::Storage::DataStore::uidFromKey(key));
-            return true;
-        },
-        [&](const Sink::Storage::DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to read current value from storage: " << error.message; });
+    DataStore::getUids(d->getTransaction(), callback);
 }
 
 bool EntityStore::contains(const QByteArray &type, const QByteArray &uid)
@@ -604,6 +575,36 @@ bool EntityStore::contains(const QByteArray &type, const QByteArray &uid)
     return DataStore::mainDatabase(d->getTransaction(), type).contains(uid);
 }
 
+bool EntityStore::exists(const QByteArray &type, const QByteArray &uid)
+{
+    bool found = false;
+    bool alreadyRemoved = false;
+    DataStore::mainDatabase(d->transaction, type)
+        .findLatest(uid,
+            [&found, &alreadyRemoved](const QByteArray &key, const QByteArray &data) -> bool {
+                auto entity = GetEntity(data.data());
+                if (entity && entity->metadata()) {
+                    auto metadata = GetMetadata(entity->metadata()->Data());
+                    found = true;
+                    if (metadata->operation() == Operation_Removal) {
+                        alreadyRemoved = true;
+                    }
+                }
+                return false;
+            },
+            [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to read old revision from storage: " << error.message; });
+    if (!found) {
+        SinkTraceCtx(d->logCtx) << "Remove: Failed to find entity " << uid;
+        return false;
+    }
+    if (alreadyRemoved) {
+        SinkTraceCtx(d->logCtx) << "Remove: Entity is already removed " << uid;
+        return false;
+    }
+    return true;
+}
+
+
 qint64 EntityStore::maxRevision()
 {
     if (!d->exists()) {