An automated test that keeps memory usage in check.

And ensures we scale linearly with the number of entities.

4 files changed, 383 insertions, 79 deletions

diff --git a/tests/databasepopulationandfacadequerybenchmark.cpp b/tests/databasepopulationandfacadequerybenchmark.cpp
new file mode 100644
index 0000000..9a06710
--- /dev/null
+++ b/tests/databasepopulationandfacadequerybenchmark.cpp
@@ -0,0 +1,220 @@
+#include <QtTest>
+
+#include <QString>
+
+#include "testimplementations.h"
+
+#include <common/facade.h>
+#include <common/domainadaptor.h>
+#include <common/resultprovider.h>
+#include <common/synclistresult.h>
+#include <common/definitions.h>
+#include <common/query.h>
+#include <common/clientapi.h>
+
+#include <iostream>
+#include <math.h>
+
+#include "event_generated.h"
+#include "getrssusage.h"
+
+/**
+ * Benchmark read performance of the facade implementation.
+ * 
+ * The memory used should grow linearly with the number of retrieved entities.
+ * The memory used should be independent from the database size, after accounting for the memory mapped db.
+ */
+class DatabasePopulationAndFacadeQueryBenchmark : public QObject
+{
+    Q_OBJECT
+
+    QByteArray identifier;
+    QList<double> mRssGrowthPerEntity;
+    QList<double> mTimePerEntity;
+
+    void populateDatabase(int count)
+    {
+        Akonadi2::Storage(Akonadi2::storageLocation(), "identifier", Akonadi2::Storage::ReadWrite).removeFromDisk();
+        //Setup
+        auto domainTypeAdaptorFactory = QSharedPointer<TestEventAdaptorFactory>::create();
+        {
+            Akonadi2::Storage storage(Akonadi2::storageLocation(), identifier, Akonadi2::Storage::ReadWrite);
+            auto transaction = storage.createTransaction(Akonadi2::Storage::ReadWrite);
+            auto db = transaction.openDatabase("event.main");
+
+            int bufferSizeTotal = 0;
+            int keysSizeTotal = 0;
+            QByteArray attachment;
+            attachment.fill('c', 1000);
+            for (int i = 0; i < count; i++) {
+                auto domainObject = Akonadi2::ApplicationDomain::Event::Ptr::create();
+                domainObject->setProperty("uid", "uid");
+                domainObject->setProperty("summary", "summary");
+                domainObject->setProperty("attachment", attachment);
+                flatbuffers::FlatBufferBuilder fbb;
+                domainTypeAdaptorFactory->createBuffer(*domainObject, fbb);
+                const auto buffer = QByteArray::fromRawData(reinterpret_cast<const char*>(fbb.GetBufferPointer()), fbb.GetSize());
+                const auto key = QString::number(i).toLatin1();
+                db.write(key, buffer);
+                bufferSizeTotal += buffer.size();
+                keysSizeTotal += key.size();
+            }
+            transaction.commit();
+
+            auto dataSizeTotal = count * (QByteArray("uid").size() + QByteArray("summary").size() + attachment.size());
+            auto size = db.getSize();
+            auto onDisk = storage.diskUsage();
+            auto writeAmplification = static_cast<double>(onDisk) / static_cast<double>(bufferSizeTotal);
+            std::cout << "Database size [kb]: " << size/1024 << std::endl;
+            std::cout << "On disk [kb]: " << onDisk/1024 << std::endl;
+            std::cout << "Buffer size total [kb]: " << bufferSizeTotal/1024 << std::endl;
+            std::cout << "Key size total [kb]: " << keysSizeTotal/1024 << std::endl;
+            std::cout << "Data size total [kb]: " << dataSizeTotal/1024 << std::endl;
+            std::cout << "Write amplification: " << writeAmplification << std::endl;
+
+            //The buffer has an overhead, but with a reasonable attachment size it should be relatively small
+            //A write amplification of 2 should be the worst case
+            QVERIFY(writeAmplification < 2);
+        }
+    }
+
+    void testLoad(int count)
+    {
+        const auto startingRss = getCurrentRSS();
+
+        Akonadi2::Query query;
+        query.liveQuery = false;
+        query.requestedProperties << "uid" << "summary";
+
+        //Benchmark
+        QTime time;
+        time.start();
+
+        auto resultSet = QSharedPointer<Akonadi2::ResultProvider<Akonadi2::ApplicationDomain::Event::Ptr> >::create();
+        auto resourceAccess = QSharedPointer<TestResourceAccess>::create();
+        TestResourceFacade facade(identifier, resourceAccess);
+
+        auto ret = facade.load(query);
+        ret.first.exec().waitForFinished();
+        auto emitter = ret.second;
+        QList<Akonadi2::ApplicationDomain::Event::Ptr> list;
+        emitter->onAdded([&list](const Akonadi2::ApplicationDomain::Event::Ptr &event) {
+            list << event;
+        });
+        bool done = false;
+        emitter->onInitialResultSetComplete([&done](const Akonadi2::ApplicationDomain::Event::Ptr &event) {
+            done = true;
+        });
+        emitter->fetch(Akonadi2::ApplicationDomain::Event::Ptr());
+        QTRY_VERIFY(done);
+        QCOMPARE(list.size(), count);
+
+        const auto elapsed = time.elapsed();
+
+        const auto finalRss = getCurrentRSS();
+        const auto rssGrowth = finalRss - startingRss;
+        //Since the database is memory mapped it is attributted to the resident set size.
+        const auto rssWithoutDb = finalRss - Akonadi2::Storage(Akonadi2::storageLocation(), identifier, Akonadi2::Storage::ReadWrite).diskUsage();
+        const auto peakRss =  getPeakRSS();
+        //How much peak deviates from final rss in percent (should be around 0)
+        const auto percentageRssError = static_cast<double>(peakRss - finalRss)*100.0/static_cast<double>(finalRss);
+        auto rssGrowthPerEntity = rssGrowth/count;
+
+        std::cout << "Loaded " << list.size() << "results." << std::endl;
+        std::cout << "The query took [ms]: " << elapsed << std::endl;
+        std::cout << "Current Rss usage [kb]: " << finalRss/1024 << std::endl;
+        std::cout << "Peak Rss usage [kb]: " << peakRss/1024 << std::endl;
+        std::cout << "Rss growth [kb]: " << rssGrowth/1024 << std::endl;
+        std::cout << "Rss growth per entity [byte]: " << rssGrowthPerEntity << std::endl;
+        std::cout << "Rss without db [kb]: " << rssWithoutDb/1024 << std::endl;
+        std::cout << "Percentage error: " << percentageRssError << std::endl;
+
+        mTimePerEntity << static_cast<double>(elapsed)/static_cast<double>(count);
+        mRssGrowthPerEntity << rssGrowthPerEntity;
+
+        QVERIFY(percentageRssError < 10);
+        //This shouldn't include the attachment (but currently does)
+        QEXPECT_FAIL("", "We're loading the attachment", Continue);
+        QVERIFY(rssGrowthPerEntity < 2000);
+
+        // Print memory layout, RSS is what is in memory
+        // std::system("exec pmap -x \"$PPID\"");
+        // std::system("top -p \"$PPID\" -b -n 1");
+    }
+
+private Q_SLOTS:
+
+    void init()
+    {
+        identifier = "identifier";
+    }
+
+    void test1k()
+    {
+        populateDatabase(1000);
+        testLoad(1000);
+    }
+
+    void test2k()
+    {
+        populateDatabase(2000);
+        testLoad(2000);
+    }
+
+    void test5k()
+    {
+        populateDatabase(5000);
+        testLoad(5000);
+    }
+
+    // void test10k()
+    // {
+    //     populateDatabase(10000);
+    //     testLoad(10000);
+    // }
+
+    static double variance(const QList<double> &values)
+    {
+        double mean = 0;
+        for (auto value : values) {
+            mean += value;
+        }
+        mean = mean / static_cast<double>(values.size());
+        double variance = 0;
+        for (auto value : values) {
+            variance += pow(static_cast<double>(value) - mean, 2);
+        }
+        variance = variance / static_cast<double>(values.size() - 1);
+        return variance;
+    }
+
+    static double maxDifference(const QList<double> &values)
+    {
+        auto max = values.first();
+        auto min = values.first();
+        for (auto value : values) {
+            if (value > max) {
+                max = value;
+            }
+            if (value < min) {
+                min = value;
+            }
+        }
+        return max - min;
+    }
+
+    void ensureUsedMemoryRemainsStable()
+    {
+        auto rssStandardDeviation = sqrt(variance(mRssGrowthPerEntity));
+        auto timeStandardDeviation = sqrt(variance(mTimePerEntity));
+        std::cout << "Rss standard deviation " << rssStandardDeviation << std::endl;
+        std::cout << "Rss max difference [byte]" << maxDifference(mRssGrowthPerEntity) << std::endl;
+        std::cout << "Time standard deviation " << timeStandardDeviation << std::endl;
+        std::cout << "Time max difference [ms]" << maxDifference(mTimePerEntity) << std::endl;
+        QVERIFY(rssStandardDeviation < 500);
+        QVERIFY(timeStandardDeviation < 1);
+    }
+};
+
+QTEST_MAIN(DatabasePopulationAndFacadeQueryBenchmark)
+#include "databasepopulationandfacadequerybenchmark.moc"
diff --git a/tests/genericfacadebenchmark.cpp b/tests/genericfacadebenchmark.cpp
deleted file mode 100644
index 703acd1..0000000
--- a/tests/genericfacadebenchmark.cpp
+++ /dev/null
@@ -1,79 +0,0 @@
-#include <QtTest>
-
-#include <QString>
-
-#include "testimplementations.h"
-
-#include <common/facade.h>
-#include <common/domainadaptor.h>
-#include <common/resultprovider.h>
-#include <common/synclistresult.h>
-#include <common/definitions.h>
-
-#include "event_generated.h"
-
-
-
-/**
- * Benchmark read performance of the generic facade implementation.
- */
-class GenericFacadeBenchmark : public QObject
-{
-    Q_OBJECT
-private Q_SLOTS:
-
-    void initTestCase()
-    {
-        Akonadi2::Storage store(Akonadi2::storageLocation(), "identifier", Akonadi2::Storage::ReadWrite);
-        store.removeFromDisk();
-    }
-
-    void testLoad()
-    {
-        const QByteArray identifier = "identifier";
-        const int count = 100000;
-
-        //Setup
-        auto domainTypeAdaptorFactory = QSharedPointer<TestEventAdaptorFactory>::create();
-        {
-            Akonadi2::Storage storage(Akonadi2::storageLocation(), identifier, Akonadi2::Storage::ReadWrite);
-            auto transaction = storage.createTransaction(Akonadi2::Storage::ReadWrite);
-            auto db = transaction.openDatabase();
-            for (int i = 0; i < count; i++) {
-                auto domainObject = Akonadi2::ApplicationDomain::Event::Ptr::create();
-                domainObject->setProperty("uid", "uid");
-                domainObject->setProperty("summary", "summary");
-
-                flatbuffers::FlatBufferBuilder fbb;
-                domainTypeAdaptorFactory->createBuffer(*domainObject, fbb);
-                db.write(QString::number(i).toLatin1(), QByteArray::fromRawData(reinterpret_cast<const char*>(fbb.GetBufferPointer()), fbb.GetSize()));
-            }
-        }
-
-        Akonadi2::Query query;
-        query.liveQuery = false;
-
-        //Benchmark
-        QBENCHMARK {
-            auto resultSet = QSharedPointer<Akonadi2::ResultProvider<Akonadi2::ApplicationDomain::Event::Ptr> >::create();
-            auto resourceAccess = QSharedPointer<TestResourceAccess>::create();
-            TestResourceFacade facade(identifier, resourceAccess);
-
-            async::SyncListResult<Akonadi2::ApplicationDomain::Event::Ptr> result(resultSet->emitter());
-
-            facade.load(query, *resultSet).exec().waitForFinished();
-            resultSet->initialResultSetComplete();
-
-            //We have to wait for the events that deliver the results to be processed by the eventloop
-            result.exec();
-
-            QCOMPARE(result.size(), count);
-        }
-
-        // Print memory layout, RSS is what is in memory
-        // std::system("exec pmap -x \"$PPID\"");
-    }
-};
-
-QTEST_MAIN(GenericFacadeBenchmark)
-#include "genericfacadebenchmark.moc"
diff --git a/tests/getrssusage.cpp b/tests/getrssusage.cpp
new file mode 100644
index 0000000..020c7d6
--- /dev/null
+++ b/tests/getrssusage.cpp
@@ -0,0 +1,124 @@
+/*
+ * Author:  David Robert Nadeau
+ * Site:    http://NadeauSoftware.com/
+ * License: Creative Commons Attribution 3.0 Unported License
+ *          http://creativecommons.org/licenses/by/3.0/deed.en_US
+ *
+ * This file has been adapted to match the coding style of the rest of the codebase.
+ *
+ * On windows link against psapi.lib, for the rest the standard library is sufficient.
+ */
+#include "getrssusage.h"
+
+#if defined(_WIN32)
+#include <windows.h>
+#include <psapi.h>
+
+#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__))
+#include <unistd.h>
+#include <sys/resource.h>
+
+#if defined(__APPLE__) && defined(__MACH__)
+#include <mach/mach.h>
+
+#elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__)))
+#include <fcntl.h>
+#include <procfs.h>
+
+#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
+#include <stdio.h>
+
+#endif
+
+#else
+#error "Cannot define getPeakRSS( ) or getCurrentRSS( ) for an unknown OS."
+#endif
+
+
+/**
+ * Returns the peak (maximum so far) resident set size (physical
+ * memory use) measured in bytes, or zero if the value cannot be
+ * determined on this OS.
+ */
+size_t getPeakRSS( )
+{
+#if defined(_WIN32)
+        /* Windows -------------------------------------------------- */
+        PROCESS_MEMORY_COUNTERS info;
+        GetProcessMemoryInfo( GetCurrentProcess( ), &info, sizeof(info) );
+        return (size_t)info.PeakWorkingSetSize;
+
+#elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__)))
+        /* AIX and Solaris ------------------------------------------ */
+        struct psinfo psinfo;
+        int fd = -1;
+        if ( (fd = open( "/proc/self/psinfo", O_RDONLY )) == -1 )
+                return (size_t)0L;              /* Can't open? */
+        if ( read( fd, &psinfo, sizeof(psinfo) ) != sizeof(psinfo) )
+        {
+                close( fd );
+                return (size_t)0L;              /* Can't read? */
+        }
+        close( fd );
+        return (size_t)(psinfo.pr_rssize * 1024L);
+
+#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__))
+        /* BSD, Linux, and OSX -------------------------------------- */
+        struct rusage rusage;
+        getrusage( RUSAGE_SELF, &rusage );
+#if defined(__APPLE__) && defined(__MACH__)
+        return (size_t)rusage.ru_maxrss;
+#else
+        return (size_t)(rusage.ru_maxrss * 1024L);
+#endif
+
+#else
+        /* Unknown OS ----------------------------------------------- */
+        return (size_t)0L;                      /* Unsupported. */
+#endif
+}
+
+
+
+
+
+/**
+ * Returns the current resident set size (physical memory use) measured
+ * in bytes, or zero if the value cannot be determined on this OS.
+ */
+size_t getCurrentRSS( )
+{
+#if defined(_WIN32)
+        /* Windows -------------------------------------------------- */
+        PROCESS_MEMORY_COUNTERS info;
+        GetProcessMemoryInfo( GetCurrentProcess( ), &info, sizeof(info) );
+        return (size_t)info.WorkingSetSize;
+
+#elif defined(__APPLE__) && defined(__MACH__)
+        /* OSX ------------------------------------------------------ */
+        struct mach_task_basic_info info;
+        mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
+        if ( task_info( mach_task_self( ), MACH_TASK_BASIC_INFO,
+                (task_info_t)&info, &infoCount ) != KERN_SUCCESS )
+                return (size_t)0L;              /* Can't access? */
+        return (size_t)info.resident_size;
+
+#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
+        /* Linux ---------------------------------------------------- */
+        long rss = 0L;
+        FILE* fp = NULL;
+        if ( (fp = fopen( "/proc/self/statm", "r" )) == NULL )
+                return (size_t)0L;              /* Can't open? */
+        if ( fscanf( fp, "%*s%ld", &rss ) != 1 )
+        {
+                fclose( fp );
+                return (size_t)0L;              /* Can't read? */
+        }
+        fclose( fp );
+        return (size_t)rss * (size_t)sysconf( _SC_PAGESIZE);
+
+#else
+        /* AIX, BSD, Solaris, and Unknown OS ------------------------ */
+        return (size_t)0L;                      /* Unsupported. */
+#endif
+}
diff --git a/tests/getrssusage.h b/tests/getrssusage.h
new file mode 100644
index 0000000..e47b14c
--- /dev/null
+++ b/tests/getrssusage.h
@@ -0,0 +1,39 @@
+/*
+ * Author:  David Robert Nadeau
+ * Site:    http://NadeauSoftware.com/
+ * License: Creative Commons Attribution 3.0 Unported License
+ *          http://creativecommons.org/licenses/by/3.0/deed.en_US
+ *
+ * This file has been adapted to match the coding style of the rest of the codebase.
+ *
+ * On windows link against psapi.lib, for the rest the standard library is sufficient.
+ */
+
+#pragma once
+
+#if defined(_WIN32)
+#include <windows.h>
+#include <psapi.h>
+
+#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__))
+#include <unistd.h>
+#include <sys/resource.h>
+
+#if defined(__APPLE__) && defined(__MACH__)
+#include <mach/mach.h>
+
+#elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__)))
+#include <fcntl.h>
+#include <procfs.h>
+
+#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__)
+#include <stdio.h>
+
+#endif
+
+#else
+#error "Cannot define getPeakRSS( ) or getCurrentRSS( ) for an unknown OS."
+#endif
+
+size_t getCurrentRSS();
+size_t getPeakRSS();