Re-do fullScan function and test it

2 files changed, 52 insertions, 22 deletions

diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp
index 7776bfc..250f01d 100644
--- a/common/storage/entitystore.cpp
+++ b/common/storage/entitystore.cpp
@@ -440,30 +440,12 @@ QVector<Identifier> EntityStore::fullScan(const QByteArray &type)
         SinkTraceCtx(d->logCtx) << "Database is not existing: " << type;
         return {};
     }
-    //The scan can return duplicate results if we have multiple revisions, so we use a set to deduplicate.
-    QSet<Identifier> keys;
-    DataStore::mainDatabase(d->getTransaction(), type)
-        .scan(QByteArray(),
-            [&](const QByteArray &key, const QByteArray &data) -> bool {
-                EntityBuffer buffer(const_cast<const char *>(data.data()), data.size());
-                if (!buffer.isValid()) {
-                    SinkWarningCtx(d->logCtx) << "Read invalid buffer from disk";
-                    return true;
-                }
 
-                size_t revision = byteArrayToSizeT(key);
+    QSet<Identifier> keys;
 
-                const auto metadata = flatbuffers::GetRoot<Metadata>(buffer.metadataBuffer());
-                const QByteArray uid = DataStore::getUidFromRevision(d->getTransaction(), revision);
-                const auto identifier = Sink::Storage::Identifier::fromDisplayByteArray(uid);
-                if (keys.contains(identifier)) {
-                    //Not something that should persist if the replay works, so we keep a message for now.
-                    SinkTraceCtx(d->logCtx) << "Multiple revisions for uid: " << Sink::Storage::Key::fromInternalByteArray(key) << ". This is normal if changereplay has not completed yet.";
-                }
-                keys << identifier;
-                return true;
-            },
-            [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during fullScan query: " << error.message; });
+    DataStore::getUids(type, d->getTransaction(), [&keys] (const QByteArray &uid) {
+        keys << Identifier::fromDisplayByteArray(uid);
+    });
 
     SinkTraceCtx(d->logCtx) << "Full scan retrieved " << keys.size() << " results.";
     return keys.toList().toVector();
diff --git a/tests/entitystoretest.cpp b/tests/entitystoretest.cpp
index 03b940b..aab5b3b 100644
--- a/tests/entitystoretest.cpp
+++ b/tests/entitystoretest.cpp
@@ -29,6 +29,54 @@ private slots:
     {
     }
 
+    void testFullScan()
+    {
+        using namespace Sink;
+        ResourceContext resourceContext{resourceInstanceIdentifier.toUtf8(), "dummy", AdaptorFactoryRegistry::instance().getFactories("test")};
+        Storage::EntityStore store(resourceContext, {});
+
+        auto mail = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail.setExtractedMessageId("messageid");
+        mail.setExtractedSubject("boo");
+
+        auto mail2 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail2.setExtractedMessageId("messageid2");
+        mail2.setExtractedSubject("foo");
+
+        auto mail3 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail3.setExtractedMessageId("messageid2");
+        mail3.setExtractedSubject("foo");
+
+        store.startTransaction(Storage::DataStore::ReadWrite);
+        store.add("mail", mail, false);
+        store.add("mail", mail2, false);
+        store.add("mail", mail3, false);
+
+        mail.setExtractedSubject("foo");
+
+        store.modify("mail", mail, QByteArrayList{}, false);
+
+        {
+            const auto ids = store.fullScan("mail");
+
+            QCOMPARE(ids.size(), 3);
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail.identifier())));
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail2.identifier())));
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail3.identifier())));
+        }
+
+        store.remove("mail", mail3, false);
+        store.commitTransaction();
+
+        {
+            const auto ids = store.fullScan("mail");
+
+            QCOMPARE(ids.size(), 2);
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail.identifier())));
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail2.identifier())));
+        }
+    }
+
     void readAll()
     {
         using namespace Sink;