From 8e6671fe6d6519f1fecf37338a3263a5b88a00d1 Mon Sep 17 00:00:00 2001 From: Christian Mollekopf Date: Tue, 22 Dec 2015 09:23:27 +0100 Subject: An automated test that keeps memory usage in check. And ensures we scale linearly with the number of entities. --- .../databasepopulationandfacadequerybenchmark.cpp | 220 +++++++++++++++++++++ tests/genericfacadebenchmark.cpp | 79 -------- tests/getrssusage.cpp | 124 ++++++++++++ tests/getrssusage.h | 39 ++++ 4 files changed, 383 insertions(+), 79 deletions(-) create mode 100644 tests/databasepopulationandfacadequerybenchmark.cpp delete mode 100644 tests/genericfacadebenchmark.cpp create mode 100644 tests/getrssusage.cpp create mode 100644 tests/getrssusage.h (limited to 'tests') diff --git a/tests/databasepopulationandfacadequerybenchmark.cpp b/tests/databasepopulationandfacadequerybenchmark.cpp new file mode 100644 index 0000000..9a06710 --- /dev/null +++ b/tests/databasepopulationandfacadequerybenchmark.cpp @@ -0,0 +1,220 @@ +#include + +#include + +#include "testimplementations.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "event_generated.h" +#include "getrssusage.h" + +/** + * Benchmark read performance of the facade implementation. + * + * The memory used should grow linearly with the number of retrieved entities. + * The memory used should be independent from the database size, after accounting for the memory mapped db. + */ +class DatabasePopulationAndFacadeQueryBenchmark : public QObject +{ + Q_OBJECT + + QByteArray identifier; + QList mRssGrowthPerEntity; + QList mTimePerEntity; + + void populateDatabase(int count) + { + Akonadi2::Storage(Akonadi2::storageLocation(), "identifier", Akonadi2::Storage::ReadWrite).removeFromDisk(); + //Setup + auto domainTypeAdaptorFactory = QSharedPointer::create(); + { + Akonadi2::Storage storage(Akonadi2::storageLocation(), identifier, Akonadi2::Storage::ReadWrite); + auto transaction = storage.createTransaction(Akonadi2::Storage::ReadWrite); + auto db = transaction.openDatabase("event.main"); + + int bufferSizeTotal = 0; + int keysSizeTotal = 0; + QByteArray attachment; + attachment.fill('c', 1000); + for (int i = 0; i < count; i++) { + auto domainObject = Akonadi2::ApplicationDomain::Event::Ptr::create(); + domainObject->setProperty("uid", "uid"); + domainObject->setProperty("summary", "summary"); + domainObject->setProperty("attachment", attachment); + flatbuffers::FlatBufferBuilder fbb; + domainTypeAdaptorFactory->createBuffer(*domainObject, fbb); + const auto buffer = QByteArray::fromRawData(reinterpret_cast(fbb.GetBufferPointer()), fbb.GetSize()); + const auto key = QString::number(i).toLatin1(); + db.write(key, buffer); + bufferSizeTotal += buffer.size(); + keysSizeTotal += key.size(); + } + transaction.commit(); + + auto dataSizeTotal = count * (QByteArray("uid").size() + QByteArray("summary").size() + attachment.size()); + auto size = db.getSize(); + auto onDisk = storage.diskUsage(); + auto writeAmplification = static_cast(onDisk) / static_cast(bufferSizeTotal); + std::cout << "Database size [kb]: " << size/1024 << std::endl; + std::cout << "On disk [kb]: " << onDisk/1024 << std::endl; + std::cout << "Buffer size total [kb]: " << bufferSizeTotal/1024 << std::endl; + std::cout << "Key size total [kb]: " << keysSizeTotal/1024 << std::endl; + std::cout << "Data size total [kb]: " << dataSizeTotal/1024 << std::endl; + std::cout << "Write amplification: " << writeAmplification << std::endl; + + //The buffer has an overhead, but with a reasonable attachment size it should be relatively small + //A write amplification of 2 should be the worst case + QVERIFY(writeAmplification < 2); + } + } + + void testLoad(int count) + { + const auto startingRss = getCurrentRSS(); + + Akonadi2::Query query; + query.liveQuery = false; + query.requestedProperties << "uid" << "summary"; + + //Benchmark + QTime time; + time.start(); + + auto resultSet = QSharedPointer >::create(); + auto resourceAccess = QSharedPointer::create(); + TestResourceFacade facade(identifier, resourceAccess); + + auto ret = facade.load(query); + ret.first.exec().waitForFinished(); + auto emitter = ret.second; + QList list; + emitter->onAdded([&list](const Akonadi2::ApplicationDomain::Event::Ptr &event) { + list << event; + }); + bool done = false; + emitter->onInitialResultSetComplete([&done](const Akonadi2::ApplicationDomain::Event::Ptr &event) { + done = true; + }); + emitter->fetch(Akonadi2::ApplicationDomain::Event::Ptr()); + QTRY_VERIFY(done); + QCOMPARE(list.size(), count); + + const auto elapsed = time.elapsed(); + + const auto finalRss = getCurrentRSS(); + const auto rssGrowth = finalRss - startingRss; + //Since the database is memory mapped it is attributted to the resident set size. + const auto rssWithoutDb = finalRss - Akonadi2::Storage(Akonadi2::storageLocation(), identifier, Akonadi2::Storage::ReadWrite).diskUsage(); + const auto peakRss = getPeakRSS(); + //How much peak deviates from final rss in percent (should be around 0) + const auto percentageRssError = static_cast(peakRss - finalRss)*100.0/static_cast(finalRss); + auto rssGrowthPerEntity = rssGrowth/count; + + std::cout << "Loaded " << list.size() << "results." << std::endl; + std::cout << "The query took [ms]: " << elapsed << std::endl; + std::cout << "Current Rss usage [kb]: " << finalRss/1024 << std::endl; + std::cout << "Peak Rss usage [kb]: " << peakRss/1024 << std::endl; + std::cout << "Rss growth [kb]: " << rssGrowth/1024 << std::endl; + std::cout << "Rss growth per entity [byte]: " << rssGrowthPerEntity << std::endl; + std::cout << "Rss without db [kb]: " << rssWithoutDb/1024 << std::endl; + std::cout << "Percentage error: " << percentageRssError << std::endl; + + mTimePerEntity << static_cast(elapsed)/static_cast(count); + mRssGrowthPerEntity << rssGrowthPerEntity; + + QVERIFY(percentageRssError < 10); + //This shouldn't include the attachment (but currently does) + QEXPECT_FAIL("", "We're loading the attachment", Continue); + QVERIFY(rssGrowthPerEntity < 2000); + + // Print memory layout, RSS is what is in memory + // std::system("exec pmap -x \"$PPID\""); + // std::system("top -p \"$PPID\" -b -n 1"); + } + +private Q_SLOTS: + + void init() + { + identifier = "identifier"; + } + + void test1k() + { + populateDatabase(1000); + testLoad(1000); + } + + void test2k() + { + populateDatabase(2000); + testLoad(2000); + } + + void test5k() + { + populateDatabase(5000); + testLoad(5000); + } + + // void test10k() + // { + // populateDatabase(10000); + // testLoad(10000); + // } + + static double variance(const QList &values) + { + double mean = 0; + for (auto value : values) { + mean += value; + } + mean = mean / static_cast(values.size()); + double variance = 0; + for (auto value : values) { + variance += pow(static_cast(value) - mean, 2); + } + variance = variance / static_cast(values.size() - 1); + return variance; + } + + static double maxDifference(const QList &values) + { + auto max = values.first(); + auto min = values.first(); + for (auto value : values) { + if (value > max) { + max = value; + } + if (value < min) { + min = value; + } + } + return max - min; + } + + void ensureUsedMemoryRemainsStable() + { + auto rssStandardDeviation = sqrt(variance(mRssGrowthPerEntity)); + auto timeStandardDeviation = sqrt(variance(mTimePerEntity)); + std::cout << "Rss standard deviation " << rssStandardDeviation << std::endl; + std::cout << "Rss max difference [byte]" << maxDifference(mRssGrowthPerEntity) << std::endl; + std::cout << "Time standard deviation " << timeStandardDeviation << std::endl; + std::cout << "Time max difference [ms]" << maxDifference(mTimePerEntity) << std::endl; + QVERIFY(rssStandardDeviation < 500); + QVERIFY(timeStandardDeviation < 1); + } +}; + +QTEST_MAIN(DatabasePopulationAndFacadeQueryBenchmark) +#include "databasepopulationandfacadequerybenchmark.moc" diff --git a/tests/genericfacadebenchmark.cpp b/tests/genericfacadebenchmark.cpp deleted file mode 100644 index 703acd1..0000000 --- a/tests/genericfacadebenchmark.cpp +++ /dev/null @@ -1,79 +0,0 @@ -#include - -#include - -#include "testimplementations.h" - -#include -#include -#include -#include -#include - -#include "event_generated.h" - - - -/** - * Benchmark read performance of the generic facade implementation. - */ -class GenericFacadeBenchmark : public QObject -{ - Q_OBJECT -private Q_SLOTS: - - void initTestCase() - { - Akonadi2::Storage store(Akonadi2::storageLocation(), "identifier", Akonadi2::Storage::ReadWrite); - store.removeFromDisk(); - } - - void testLoad() - { - const QByteArray identifier = "identifier"; - const int count = 100000; - - //Setup - auto domainTypeAdaptorFactory = QSharedPointer::create(); - { - Akonadi2::Storage storage(Akonadi2::storageLocation(), identifier, Akonadi2::Storage::ReadWrite); - auto transaction = storage.createTransaction(Akonadi2::Storage::ReadWrite); - auto db = transaction.openDatabase(); - for (int i = 0; i < count; i++) { - auto domainObject = Akonadi2::ApplicationDomain::Event::Ptr::create(); - domainObject->setProperty("uid", "uid"); - domainObject->setProperty("summary", "summary"); - - flatbuffers::FlatBufferBuilder fbb; - domainTypeAdaptorFactory->createBuffer(*domainObject, fbb); - db.write(QString::number(i).toLatin1(), QByteArray::fromRawData(reinterpret_cast(fbb.GetBufferPointer()), fbb.GetSize())); - } - } - - Akonadi2::Query query; - query.liveQuery = false; - - //Benchmark - QBENCHMARK { - auto resultSet = QSharedPointer >::create(); - auto resourceAccess = QSharedPointer::create(); - TestResourceFacade facade(identifier, resourceAccess); - - async::SyncListResult result(resultSet->emitter()); - - facade.load(query, *resultSet).exec().waitForFinished(); - resultSet->initialResultSetComplete(); - - //We have to wait for the events that deliver the results to be processed by the eventloop - result.exec(); - - QCOMPARE(result.size(), count); - } - - // Print memory layout, RSS is what is in memory - // std::system("exec pmap -x \"$PPID\""); - } -}; - -QTEST_MAIN(GenericFacadeBenchmark) -#include "genericfacadebenchmark.moc" diff --git a/tests/getrssusage.cpp b/tests/getrssusage.cpp new file mode 100644 index 0000000..020c7d6 --- /dev/null +++ b/tests/getrssusage.cpp @@ -0,0 +1,124 @@ +/* + * Author: David Robert Nadeau + * Site: http://NadeauSoftware.com/ + * License: Creative Commons Attribution 3.0 Unported License + * http://creativecommons.org/licenses/by/3.0/deed.en_US + * + * This file has been adapted to match the coding style of the rest of the codebase. + * + * On windows link against psapi.lib, for the rest the standard library is sufficient. + */ +#include "getrssusage.h" + +#if defined(_WIN32) +#include +#include + +#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) +#include +#include + +#if defined(__APPLE__) && defined(__MACH__) +#include + +#elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__))) +#include +#include + +#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__) +#include + +#endif + +#else +#error "Cannot define getPeakRSS( ) or getCurrentRSS( ) for an unknown OS." +#endif + + +/** + * Returns the peak (maximum so far) resident set size (physical + * memory use) measured in bytes, or zero if the value cannot be + * determined on this OS. + */ +size_t getPeakRSS( ) +{ +#if defined(_WIN32) + /* Windows -------------------------------------------------- */ + PROCESS_MEMORY_COUNTERS info; + GetProcessMemoryInfo( GetCurrentProcess( ), &info, sizeof(info) ); + return (size_t)info.PeakWorkingSetSize; + +#elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__))) + /* AIX and Solaris ------------------------------------------ */ + struct psinfo psinfo; + int fd = -1; + if ( (fd = open( "/proc/self/psinfo", O_RDONLY )) == -1 ) + return (size_t)0L; /* Can't open? */ + if ( read( fd, &psinfo, sizeof(psinfo) ) != sizeof(psinfo) ) + { + close( fd ); + return (size_t)0L; /* Can't read? */ + } + close( fd ); + return (size_t)(psinfo.pr_rssize * 1024L); + +#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) + /* BSD, Linux, and OSX -------------------------------------- */ + struct rusage rusage; + getrusage( RUSAGE_SELF, &rusage ); +#if defined(__APPLE__) && defined(__MACH__) + return (size_t)rusage.ru_maxrss; +#else + return (size_t)(rusage.ru_maxrss * 1024L); +#endif + +#else + /* Unknown OS ----------------------------------------------- */ + return (size_t)0L; /* Unsupported. */ +#endif +} + + + + + +/** + * Returns the current resident set size (physical memory use) measured + * in bytes, or zero if the value cannot be determined on this OS. + */ +size_t getCurrentRSS( ) +{ +#if defined(_WIN32) + /* Windows -------------------------------------------------- */ + PROCESS_MEMORY_COUNTERS info; + GetProcessMemoryInfo( GetCurrentProcess( ), &info, sizeof(info) ); + return (size_t)info.WorkingSetSize; + +#elif defined(__APPLE__) && defined(__MACH__) + /* OSX ------------------------------------------------------ */ + struct mach_task_basic_info info; + mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; + if ( task_info( mach_task_self( ), MACH_TASK_BASIC_INFO, + (task_info_t)&info, &infoCount ) != KERN_SUCCESS ) + return (size_t)0L; /* Can't access? */ + return (size_t)info.resident_size; + +#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__) + /* Linux ---------------------------------------------------- */ + long rss = 0L; + FILE* fp = NULL; + if ( (fp = fopen( "/proc/self/statm", "r" )) == NULL ) + return (size_t)0L; /* Can't open? */ + if ( fscanf( fp, "%*s%ld", &rss ) != 1 ) + { + fclose( fp ); + return (size_t)0L; /* Can't read? */ + } + fclose( fp ); + return (size_t)rss * (size_t)sysconf( _SC_PAGESIZE); + +#else + /* AIX, BSD, Solaris, and Unknown OS ------------------------ */ + return (size_t)0L; /* Unsupported. */ +#endif +} diff --git a/tests/getrssusage.h b/tests/getrssusage.h new file mode 100644 index 0000000..e47b14c --- /dev/null +++ b/tests/getrssusage.h @@ -0,0 +1,39 @@ +/* + * Author: David Robert Nadeau + * Site: http://NadeauSoftware.com/ + * License: Creative Commons Attribution 3.0 Unported License + * http://creativecommons.org/licenses/by/3.0/deed.en_US + * + * This file has been adapted to match the coding style of the rest of the codebase. + * + * On windows link against psapi.lib, for the rest the standard library is sufficient. + */ + +#pragma once + +#if defined(_WIN32) +#include +#include + +#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) +#include +#include + +#if defined(__APPLE__) && defined(__MACH__) +#include + +#elif (defined(_AIX) || defined(__TOS__AIX__)) || (defined(__sun__) || defined(__sun) || defined(sun) && (defined(__SVR4) || defined(__svr4__))) +#include +#include + +#elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__) +#include + +#endif + +#else +#error "Cannot define getPeakRSS( ) or getCurrentRSS( ) for an unknown OS." +#endif + +size_t getCurrentRSS(); +size_t getPeakRSS(); -- cgit v1.2.3