JSON Test Harness: Part 1

Question

Part 2: JSON Test Harness: Part 2
Part 3: JSON Test Harness: Part 3
Part 4: JSON Test Harness: Part 4

Time to review some test harness code I have written.

https://github.com/Loki-Astari/JsonBenchmark

I wrote a C++ JSON parser/serializer and wanted to compare performance against other JSON parsers. This was originally written by somebody else (credit were credit is due (see README)) but has been highly modified to use better C++ idioms (I hope).

This review is for the test harness than handles all the different JSON parsers. Each implementation of a test harness (one for each JSON parser) is supposed to:

• include test.h and derive their own class from of TestBase implementing the virtual functions.
• In their own source declare REGISTER_TEST(<YourClass>); to create an instance of an object of their class.
• In test.cpp add REGISTER_TEST_OBJECT(<YourClass>); to link it together.

https://github.com/Loki-Astari/JsonBenchmark/tree/master/src/ThirdParty

TestManager.h

#ifndef THORS_ANVIL_BENCHMARK_TEST_MANAGER_H
#define THORS_ANVIL_BENCHMARK_TEST_MANAGER_H

#include <vector>
#include <string.h>
#include <tuple>
#include <algorithm>

class TestBase;
using ParsrList = std::vector<const TestBase*>;

class TestManager
{
    public:
        static TestManager& instance();
        void addTest(const TestBase* test);
        const ParsrList& getTests() const;

    private:
        ParsrList mTests;
};

#endif

TestManager.cpp

#include "TestManager.h"
#include <memory>

TestManager& TestManager::instance()
{
    static TestManager singleton;
    return singleton;
}

void TestManager::addTest(const TestBase* test)
{
    mTests.push_back(test);
}

const ParsrList& TestManager::getTests() const
{
    return mTests;
}

test.h

#ifndef THORS_ANVIL_BENCHMARK_TEST_H
#define THORS_ANVIL_BENCHMARK_TEST_H

#include <cstddef>
#include <string>

class ParseResultBase
{
    public:
        virtual ~ParseResultBase() {}
};

class StringResultBase
{
    public:
        virtual ~StringResultBase() {}
        virtual const char* c_str() const = 0;
};

struct Stat
{
    std::size_t objectCount;
    std::size_t arrayCount;
    std::size_t numberCount;
    std::size_t stringCount;
    std::size_t trueCount;
    std::size_t falseCount;
    std::size_t nullCount;

    std::size_t memberCount;   // Number of members in all objects
    std::size_t elementCount;  // Number of elements in all arrays
    std::size_t stringLength;  // Number of code units in all strings

    bool operator==(Stat const& rhs) const;
    bool operator!=(Stat const& rhs) const;
};

class TestBase
{
    public:
        virtual ~TestBase() {}
        bool operator<(const TestBase& rhs) const;

        virtual const char* GetName() const = 0;
        virtual const char* GetFilename() const = 0;

        // For each operation, call SetUp() before and TearDown() after.
        // It is mainly for libraries require huge initialize time (e.g. Creating Isolate in V8).
        virtual void SetUp()                                                                          const {}
        virtual void TearDown()                                                                       const {}
        virtual void SetUp(char const*)                                                               const {SetUp();}
        virtual void TearDown(char const*)                                                            const {TearDown();}

        virtual bool ParseDouble(const char* /*json*/, double* /*d*/)                                 const { return false; }
        virtual bool ParseString(const char* /*json*/, std::string& /*s*/)                            const { return false; }
        virtual ParseResultBase* Parse(const char* /*json*/, std::size_t /*length*/)                  const { return nullptr; }
        virtual StringResultBase* Stringify(const ParseResultBase* /*parseResult*/)                   const { return nullptr; }
        virtual StringResultBase* Prettify(const ParseResultBase* /*parseResult*/)                    const { return nullptr; }
        virtual StringResultBase* SaxRoundtrip(const char* /*json*/, std::size_t /*length*/)          const { return nullptr; }
        virtual bool Statistics(const ParseResultBase* /*parseResult*/, Stat* /*stat*/)               const { return false; }
        virtual bool SaxStatistics(const char* /*json*/, std::size_t /*length*/, Stat* /*stat*/)      const { return false; }
        virtual bool SaxStatisticsUTF16(const char* /*json*/, std::size_t /*length*/, Stat* /*stat*/) const { return false; }
};

#define REGISTER_TEST(cls)  std::unique_ptr<TestBase> get ## cls() { return std::make_unique<cls>(); }static_assert(true)

#endif

test.cpp

#include "test.h"
#include "TestManager.h"
#include <memory>

bool Stat::operator!=(Stat const& rhs) const
{
    return !operator==(rhs);
}

bool TestBase::operator<(const TestBase& rhs) const
{
    return strcmp(GetName(), rhs.GetName()) < 0;
}

bool Stat::operator==(Stat const& rhs) const
{
    return std::forward_as_tuple(objectCount, arrayCount, numberCount,
                                 stringCount, trueCount, falseCount,
                                 nullCount, memberCount, elementCount,
                                 stringLength)
        ==
           std::forward_as_tuple(rhs.objectCount, rhs.arrayCount, rhs.numberCount,
                                 rhs.stringCount, rhs.trueCount, rhs.falseCount,
                                 rhs.nullCount, rhs.memberCount, rhs.elementCount,
                                 rhs.stringLength);
}

class TestRunner: public TestBase
{
    std::unique_ptr<TestBase>    pimpl;
    public:
        TestRunner(std::unique_ptr<TestBase>&& src)
            : pimpl(std::move(src))
        {
            TestManager::instance().addTest(this);
        }
        virtual const char* GetName()                                                   const override {return pimpl->GetName();}
        virtual const char* GetFilename()                                               const override {return pimpl->GetFilename();}

        virtual void SetUp(char const* test)                                            const override {return pimpl->SetUp(test);}
        virtual void TearDown(char const* test)                                         const override {return pimpl->TearDown(test);}

        virtual bool ParseDouble(const char* json, double* d)                           const override {return pimpl->ParseDouble(json, d);}
        virtual bool ParseString(const char* json, std::string& s)                      const override {return pimpl->ParseString(json, s);}
        virtual ParseResultBase* Parse(const char* json, size_t length)                 const override {return pimpl->Parse(json, length);}
        virtual StringResultBase* Stringify(const ParseResultBase* parseResult)         const override {return pimpl->Stringify(parseResult);}
        virtual StringResultBase* Prettify(const ParseResultBase* parseResult)          const override {return pimpl->Prettify(parseResult);}
        virtual StringResultBase* SaxRoundtrip(const char* json, size_t length)         const override {return pimpl->SaxRoundtrip(json, length);}
        virtual bool Statistics(const ParseResultBase* parseResult, Stat* stat)         const override {return pimpl->Statistics(parseResult, stat);}
        virtual bool SaxStatistics(const char* json, size_t length, Stat* stat)         const override {return pimpl->SaxStatistics(json, length, stat);}
        virtual bool SaxStatisticsUTF16(const char* json, size_t length, Stat* stat)    const override {return pimpl->SaxStatisticsUTF16(json, length, stat);}
};

#define REGISTER_TEST_OBJECT(cls)       std::unique_ptr<TestBase> get ## cls();TestRunner gRegister ## cls(get ## cls())

REGISTER_TEST_OBJECT(ThorsSerializerTest);
REGISTER_TEST_OBJECT(ArduinojsonTest);
REGISTER_TEST_OBJECT(CajunTest);
REGISTER_TEST_OBJECT(ConfiguruTest);
REGISTER_TEST_OBJECT(FastjsonTest);
REGISTER_TEST_OBJECT(JeayesonTest);
REGISTER_TEST_OBJECT(JsonboxTest);
REGISTER_TEST_OBJECT(JsonconsTest);
REGISTER_TEST_OBJECT(JsoncppTest);
REGISTER_TEST_OBJECT(VoorheesTest);
REGISTER_TEST_OBJECT(JsonxxTest);
REGISTER_TEST_OBJECT(JvarTest);
REGISTER_TEST_OBJECT(JzonTest);
REGISTER_TEST_OBJECT(NlohmannTest);
REGISTER_TEST_OBJECT(PicojsonTest);
REGISTER_TEST_OBJECT(RapidjsonTest);
REGISTER_TEST_OBJECT(RapidjsonFullPrecTest);
REGISTER_TEST_OBJECT(RapidjsonInsituTest);
REGISTER_TEST_OBJECT(RapidjsonIterativeTest);
REGISTER_TEST_OBJECT(RapidjsonAutoUTFTest);
REGISTER_TEST_OBJECT(GasonTest);
REGISTER_TEST_OBJECT(SajsonTest);
REGISTER_TEST_OBJECT(UjsonTest);
REGISTER_TEST_OBJECT(Ujson4cTest);
REGISTER_TEST_OBJECT(PJsonTest);
REGISTER_TEST_OBJECT(UdbTest);
REGISTER_TEST_OBJECT(JusonTest);
REGISTER_TEST_OBJECT(CcanTest);
REGISTER_TEST_OBJECT(CjsonTest);
REGISTER_TEST_OBJECT(VinenthzTest);
REGISTER_TEST_OBJECT(YajlTest);
REGISTER_TEST_OBJECT(JsoncTest);
REGISTER_TEST_OBJECT(JsmnTest);