Part 1: JSON Test Harness: Part 1
Part 2: JSON Test Harness: Part 2
Part 3: JSON Test Harness: Part 3
Time to review some test harness code I have written.
https://github.com/Loki-Astari/JsonBenchmark
I wrote a C++ JSON parser/serializer and wanted to compare performance against other JSON parsers. This was originally written by somebody else (credit were credit is due (see README)) but has been highly modified to use better C++ idioms (I hope).
This questions covers the biggest test. These are for the "Performance" tests. Results found here: https://lokiastari.com/Json/Performance.linux.html (Sadly, but not unexpectedly my code is not the fastest :-( )
PerformanceChecker.h
#ifndef THORS_ANVIL_BENCHMARK_PERFORMANCE_CHECKER_H
#define THORS_ANVIL_BENCHMARK_PERFORMANCE_CHECKER_H
#include "CommonReader.h"
#include <map>
#include <chrono>
namespace ThorsAnvil
{
namespace Benchmark
{
class PerformanceChecker: public CommonReader
{
static std::map<std::string, Stat const*> const validator;
static const int loopCount = 10;
public:
using CommonReader::CommonReader;
virtual void executeTest(TestBase const& parser) override;
virtual State executeTest(TestBase const& parser, Test const& test) override;
virtual std::string getDir() const override
{
return "performance";
}
private:
virtual bool useSetUp() const override {return false;}
virtual void generateConPerData(TestBase const& parser, Test const& test, State state) override;
virtual void printResults(TestBase const&, int (&)[3], std::vector<Test const*>&) override{}
private:
void getCodeSize(TestBase const& parser);
void validatePerformance(TestBase const& parser);
class Output
{
Options& options;
TestBase const& parser;
Test const& test;
std::string const name;
std::string const action;
double& minDuration;
bool fail;
public:
Output(Options& options, TestBase const& parser_, Test const& test_, std::string const& name_, std::string const& action_, double& minDuration_);
~Output();
void setPass();
void validateMemory();
void readoutput(int fd, std::ostream& output);
};
template<typename F>
double timeExecution(F&& action);
void executeParse(TestBase const& parser, Test const& test);
void executeStringify(TestBase const& parser, Test const& test);
void executePrettify(TestBase const& parser, Test const& test);
void executeStatistics(TestBase const& parser, Test const& test);
void executeSaxRoundtrip(TestBase const& parser, Test const& test);
void executeSaxStatistics(TestBase const& parser, Test const& test);
};
template<typename F>
double PerformanceChecker::timeExecution(F&& action)
{
auto start = std::chrono::high_resolution_clock::now();
action();
auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> duration = end-start;
return duration.count();
}
}
}
#endif
PerformanceChecker.cpp
#include "PerformanceChecker.h"
#include <memory>
#include <chrono>
#include <iomanip>
#include <algorithm>
#include <unistd.h>
#include <sys/wait.h>
#include <stdlib.h>
using namespace ThorsAnvil::Benchmark;
#pragma vera-pushoff
using namespace std::string_literals;
#pragma vera-pop
static Stat constexpr canada =
{
/*objectCount:*/ 4,
/*arrayCount:*/ 56045,
/*numberCount:*/ 111126,
/*stringCount:*/ 12,
/*trueCount:*/ 0,
/*falseCount:*/ 0,
/*nullCount:*/ 0,
/*memberCount:*/ 8,
/*elementCount:*/ 167170,
/*stringLength:*/ 90
};
static Stat constexpr citm_catalog
{
/*objectCount:*/ 10937,
/*arrayCount:*/ 10451,
/*numberCount:*/ 14392,
/*stringCount:*/ 26604,
/*trueCount:*/ 0,
/*falseCount:*/ 0,
/*nullCount:*/ 1263,
/*memberCou:nt*/ 25869,
/*elementCount:*/ 11908,
/*stringLength:*/ 221379
};
static Stat constexpr twitter =
{
/*objectCount:*/ 1264,
/*arrayCount:*/ 1050,
/*numberCount:*/ 2109,
/*stringCount:*/ 18099,
/*trueCount:*/ 345,
/*falseCount:*/ 2446,
/*nullCount:*/ 1946,
/*memberCou:nt*/ 13345,
/*elementCount:*/ 568,
/*stringLength:*/ 367917
};
std::map<std::string, Stat const*> const PerformanceChecker::validator
{
{"canada.json"s, &canada},
{"citm_catalog.json"s, &citm_catalog},
{"twitter.json"s, &twitter}
};
void PerformanceChecker::executeTest(TestBase const& parser)
{
std::cerr << "\t" << parser.GetName() << "\n";
validatePerformance(parser);
CommonReader::executeTest(parser);
getCodeSize(parser);
}
State PerformanceChecker::executeTest(TestBase const& parser, Test const& test)
{
executeParse(parser, test);
executeStringify(parser, test);
executePrettify(parser, test);
executeStatistics(parser, test);
executeSaxRoundtrip(parser, test);
executeSaxStatistics(parser, test);
return Pass;
}
void PerformanceChecker::generateConPerData(TestBase const& /*parser*/, Test const& /*test*/, State /*state*/)
{}
void PerformanceChecker::getCodeSize(TestBase const& parser)
{
std::string path = QUOTE(THORSANVIL_ROOT) "/build/lib/";
std::string fileName = std::string(parser.GetName());
std::size_t endOfName = fileName.find(' ');
fileName = fileName.substr(0, endOfName);
std::string fullPath = path + "lib" + fileName + "Test" + QUOTE(LIBEXT);
std::ifstream obj(fullPath);
obj.seekg(0, std::ios_base::end);
std::size_t size = obj.tellg();
options.performance << "7. Code size," << parser.GetName() << ",XXX,0,0,0,0,0,0," << size << "\n";
}
void PerformanceChecker::validatePerformance(TestBase const& parser)
{
for (auto const& test: tests)
{
std::string fileName = test.path.str().substr(test.path.str().rfind('/') + 1);
TestSetUp testSetUp(parser, setupName(test), true);
std::unique_ptr<ParseResultBase> dom(parser.Parse(test.input.c_str(), test.input.size()));
Stat stats {};
bool result = parser.Statistics(dom.get(), &stats);
std::cerr << "\t\t" << std::setw(15) << std::left << "Validate:"
<< std::setw(20) << std::left << fileName
<< std::right << std::flush;
if (!result)
{
std::cerr << " Not Implemented\n";
}
else
{
auto find = validator.find(fileName);
if (find == validator.end())
{
std::cerr << " New Test! No comparison\n";
continue;
}
bool ok = stats == *find->second;
std::cerr << (ok ? " Pass" : " Fail") << "\n";
if (!ok)
{
std::cerr << "\t\t\tobjectCount " << std::setw(8) << find->second->objectCount << " : " << std::setw(8) << stats.objectCount << (find->second->objectCount == stats.objectCount ? "" : " BAD") << "\n"
<< "\t\t\tarrayCount " << std::setw(8) << find->second->arrayCount << " : " << std::setw(8) << stats.arrayCount << (find->second->arrayCount == stats.arrayCount ? "" : " BAD") << "\n"
<< "\t\t\tnumberCount " << std::setw(8) << find->second->numberCount << " : " << std::setw(8) << stats.numberCount << (find->second->numberCount == stats.numberCount ? "" : " BAD") << "\n"
<< "\t\t\tstringCount " << std::setw(8) << find->second->stringCount << " : " << std::setw(8) << stats.stringCount << (find->second->stringCount == stats.stringCount ? "" : " BAD") << "\n"
<< "\t\t\ttrueCount " << std::setw(8) << find->second->trueCount << " : " << std::setw(8) << stats.trueCount << (find->second->trueCount == stats.trueCount ? "" : " BAD") << "\n"
<< "\t\t\tfalseCount " << std::setw(8) << find->second->falseCount << " : " << std::setw(8) << stats.falseCount << (find->second->falseCount == stats.falseCount ? "" : " BAD") << "\n"
<< "\t\t\tnullCount " << std::setw(8) << find->second->nullCount << " : " << std::setw(8) << stats.nullCount << (find->second->nullCount == stats.nullCount ? "" : " BAD") << "\n"
<< "\t\t\tmemberCount " << std::setw(8) << find->second->memberCount << " : " << std::setw(8) << stats.memberCount << (find->second->memberCount == stats.memberCount ? "" : " BAD") << "\n"
<< "\t\t\telementCount " << std::setw(8) << find->second->elementCount << " : " << std::setw(8) << stats.elementCount << (find->second->elementCount == stats.elementCount? "" : " BAD") << "\n"
<< "\t\t\tstringLength " << std::setw(8) << find->second->stringLength << " : " << std::setw(8) << stats.stringLength << (find->second->stringLength == stats.stringLength? "" : " BAD") << "\n";
}
}
}
}
void PerformanceChecker::executeParse(TestBase const& parser, Test const& test)
{
double minDuration = std::chrono::duration<double>::max().count();
Output generator(options, parser, test, "Parse", "1", minDuration);
for (int loop = 0; loop < loopCount; ++loop)
{
TestSetUp testSetUp(parser, setupName(test), true);
std::unique_ptr<ParseResultBase> result;
double duration = timeExecution([&parser, &test, &result]()
{ result.reset(parser.Parse(test.input.c_str(), test.input.size()));});
if (!result)
{
return;
}
minDuration = std::min(minDuration, duration);
}
generator.setPass();
}
void PerformanceChecker::executeStringify(TestBase const& parser, Test const& test)
{
double minDuration = std::chrono::duration<double>::max().count();
Output generator(options, parser, test, "Stringify", "2", minDuration);
TestSetUp testSetUp(parser, setupName(test), true);
std::unique_ptr<ParseResultBase> dom(parser.Parse(test.input.c_str(), test.input.size()));
for (int loop = 0; loop < loopCount; ++loop)
{
std::unique_ptr<StringResultBase> result;
double duration = timeExecution([&parser, &result, &dom]()
{ result.reset(parser.Stringify(dom.get()));});
if (!result)
{
return;
}
minDuration = std::min(minDuration, duration);
}
generator.setPass();
}
void PerformanceChecker::executePrettify(TestBase const& parser, Test const& test)
{
double minDuration = std::chrono::duration<double>::max().count();
Output generator(options, parser, test, "Prettify", "3", minDuration);
TestSetUp testSetUp(parser, setupName(test), true);
std::unique_ptr<ParseResultBase> dom(parser.Parse(test.input.c_str(), test.input.size()));
for (int loop = 0; loop < loopCount; ++loop)
{
std::unique_ptr<StringResultBase> result;
double duration = timeExecution([&parser, &result, &dom]()
{ result.reset(parser.Prettify(dom.get()));});
if (!result)
{
return;
}
minDuration = std::min(minDuration, duration);
}
generator.setPass();
}
void PerformanceChecker::executeStatistics(TestBase const& parser, Test const& test)
{
double minDuration = std::chrono::duration<double>::max().count();
Output generator(options, parser, test, "Statistics", "4", minDuration);
TestSetUp testSetUp(parser, setupName(test), true);
std::unique_ptr<ParseResultBase> dom(parser.Parse(test.input.c_str(), test.input.size()));
for (int loop = 0; loop < loopCount; ++loop)
{
Stat stats {};
bool result;
double duration = timeExecution([&parser, &result, &stats, &dom]()
{ result = parser.Statistics(dom.get(), &stats);});
if (!result)
{
return;
}
minDuration = std::min(minDuration, duration);
}
generator.setPass();
}
void PerformanceChecker::executeSaxRoundtrip(TestBase const& parser, Test const& test)
{
double minDuration = std::chrono::duration<double>::max().count();
Output generator(options, parser, test, "SaxRoundtrip", "5", minDuration);
for (int loop = 0; loop < loopCount; ++loop)
{
TestSetUp testSetUp(parser, setupName(test), true);
std::unique_ptr<StringResultBase> result;
double duration = timeExecution([&parser, &test, &result]()
{ result.reset(parser.SaxRoundtrip(test.input.c_str(), test.input.size()));});
if (!result)
{
return;
}
minDuration = std::min(minDuration, duration);
}
generator.setPass();
}
void PerformanceChecker::executeSaxStatistics(TestBase const& parser, Test const& test)
{
double minDuration = std::chrono::duration<double>::max().count();
Output generator(options, parser, test, "SaxStatistics", "6", minDuration);
for (int loop = 0; loop < loopCount; ++loop)
{
TestSetUp testSetUp(parser, setupName(test), true);
Stat stats {};
bool result;
double duration = timeExecution([&parser, &test, &result, &stats]()
{ result = parser.SaxStatistics(test.input.c_str(), test.input.size(), &stats);});
if (!result)
{
return;
}
minDuration = std::min(minDuration, duration);
}
generator.setPass();
}
PerformanceChecker.cpp (PerformanceChecker::Output)
PerformanceChecker::Output::Output(Options& options, TestBase const& parser_, Test const& test_, std::string const& name_, std::string const& action_, double& minDuration_)
: options(options)
, parser(parser_)
, test(test_)
, name(name_)
, action(action_)
, minDuration(minDuration_)
, fail(true)
{
((void)parser);
std::string fileName = test.path.str().substr(test.path.str().rfind('/') + 1);
std::cerr << "\t\t" << std::setw(15) << std::left << name
<< std::setw(20) << std::left << fileName << " ... "
<< std::right << std::flush;
}
PerformanceChecker::Output::~Output()
{
if (!fail)
{
double throughput = test.input.size() / (1024.0 * 1024.0) / minDuration;
std::cerr << std::setw(6) << std::setprecision(3) << (minDuration * 1000) << " ms "
<< std::setw(3) << std::setprecision(3) << throughput << " MB/s\n"
<< std::flush;
options.performance << action << ". "
<< name << ","
<< parser.GetName() << ","
<< test.path.str().substr(test.path.str().rfind('/') + 1) << ","
<< (minDuration * 1000) << ",";
validateMemory();
options.performance << "0\n";
}
else
{
std::cerr << "\t\tNot support\n" << std::flush;
}
}
void PerformanceChecker::Output::setPass() {fail = false;}
void PerformanceChecker::Output::validateMemory()
{
int stdoutPipe[2];
int stderrPipe[2];
if (pipe(stdoutPipe) < 0)
{
std::cerr << "Failed to Create Pipe: Stdout";
exit(1);
}
if (pipe(stderrPipe) < 0)
{
std::cerr << "Failed to Create Pipe: Stderr";
exit(1);
}
pid_t pid = fork();
if (pid < 0)
{
std::cerr << "Failed to Fork Memory App";
exit(1);
}
if (pid == 0)
{
dup2(stdoutPipe[1], fileno(stdout));
dup2(stderrPipe[1], fileno(stderr));
close(stdoutPipe[0]);
close(stdoutPipe[1]);
close(stderrPipe[0]);
close(stderrPipe[1]);
std::string fileName = test.path.str().substr(test.path.str().rfind('/') + 1);
std::string pName = parser.GetName();
std::string memoryApp= QUOTE(THORSANVIL_ROOT) "/build/bin/memory17";
setenv("DYLD_LIBRARY_PATH", QUOTE(THORSANVIL_ROOT) "/build/lib/", 1);
setenv("DYLD_INSERT_LIBRARIES", QUOTE(THORSANVIL_ROOT) "/build/lib/libMemory17.dylib", 1);
setenv("DYLD_FORCE_FLAT_NAMESPACE", "1", 1);
execl(memoryApp.c_str(), memoryApp.c_str(), pName.c_str(), fileName.c_str(), action.c_str(), nullptr);//, envp);
exit(0);
}
close(stdoutPipe[1]);
close(stderrPipe[1]);
readoutput(stdoutPipe[0], std::cout);
readoutput(stderrPipe[0], options.performance);
int stat_loc;
pid_t child = wait4(pid, &stat_loc, 0, nullptr);
if (child != pid)
{
std::cerr << "Processes wait error\n";
exit(1);
}
close(stdoutPipe[0]);
close(stderrPipe[0]);
}
void PerformanceChecker::Output::readoutput(int fd, std::ostream& output)
{
char buffer[4096];
int nbytes;
while ((nbytes = ::read(fd, buffer, sizeof(buffer))) != 0)
{
if (nbytes == -1)
{
switch (errno)
{
case EAGAIN: continue;
case EINTR: continue;
case EBADF: std::cerr << "EBADF\n"; break;
case EFAULT: std::cerr << "EFAULT\n"; break;
case EINVAL: std::cerr << "EINVAL\n"; break;
case EIO: std::cerr << "EIO\n"; break;
case EISDIR: std::cerr << "EISDIR\n"; break;
case ENOBUFS: std::cerr << "ENOBUFS\n"; break;
case ENOMEM: std::cerr << "ENOMEM\n"; break;
case ENXIO: std::cerr << "ENXIO\n"; break;
case ESPIPE: std::cerr << "ESPIPE\n"; break;
case ECONNRESET:std::cerr << "ECONNRESET\n";break;
case ENOTCONN: std::cerr << "ENOTCONN\n"; break;
case ETIMEDOUT: std::cerr << "ETIMEDOUT\n"; break;
default:
std::cerr << "Unknown: " << errno << "\n";
break;
}
std::cerr << "Read from socket failed\n";
exit(1);
}
output.write(buffer, nbytes);
}
}
