include/common_bench/benchmark.h

0001 #ifndef BENCHMARK_H
0002 #define BENCHMARK_H
0003
0004 #include "exception.h"
0005 #include <fmt/core.h>
0006 #include <fstream>
0007 #include <iomanip>
0008 #include <iostream>
0009 #include <map>
0010 #include <nlohmann/json.hpp>
0011 #include <string_view>
0012 #include <vector>
0013 #include <string>
0014
0015 namespace common_bench {
0016
0017 /** Exception
0018  */
0019 struct TestDefinitionError : Exception {
0020   TestDefinitionError(std::string_view msg)
0021       : Exception(msg, "test_definition_error") {}
0022 };
0023
0024 /** Benchmark test class.
0025  *  Wrapper for our test data json, with three methods to set the status
0026  *  after test completion (to pass, fail or error). The default value
0027  *  is error.
0028  *  The following fields should be defined in the definitions json
0029  *  for the test to make sense:
0030  *   - name: unique identifier for this test
0031  *   - title: Slightly more verbose identifier for this test
0032  *   - description: Concise description of what is tested
0033  *   - quantity: What quantity is tested? Units of value/target
0034  *   - target: Target value of <quantity> that we want to reach
0035  *   - value: Actual value of <quantity>
0036  *   - weight: Weight for this test (this is defaulted to 1.0 if not specified)
0037  *   - result: pass/fail/error
0038  */
0039 struct Test {
0040   /** Test construction.
0041    * \note Round braces for the json constructor, as else it will pick the wrong
0042    *       initializer-list constructor (it will put everything in an array)
0043    */
0044   Test(const std::map<std::string, std::string> &definition)
0045       : json(definition) {
0046     // std::cout << json.dump() << std::endl;
0047     // initialize with error (as we don't have a value yet)
0048     error();
0049     // Check that all required fields are present
0050     for (const auto &field : {"name", "title", "description", "quantity",
0051                               "target", "value", "result"}) {
0052       if (json.find(field) == json.end()) {
0053         throw TestDefinitionError{
0054             fmt::format("Error in test definition: field '{}' missing", field)};
0055       }
0056     }
0057     // Default "weight" to 1 if not set
0058     if (json.find("weight") == json.end()) {
0059       json["weight"] = 1.0;
0060     }
0061   }
0062
0063   /// Set this test to pass.
0064   void pass(double value) { update_result("pass", value); }
0065   /// Set this test to fail state.
0066   void fail(double value) { update_result("fail", value); }
0067   /// Set this test to error state.
0068   void error(double value = 0) { update_result("error", value); }
0069
0070   nlohmann::json json;
0071
0072 private:
0073   void update_result(std::string_view status, double value) {
0074     json["result"] = status;
0075     json["value"] = value;
0076   }
0077 };
0078
0079 /** Write out a collection of tests to a json file.
0080  *
0081  */
0082 inline void write_test(const std::vector<Test> &data, const std::string &fname) {
0083   nlohmann::json test;
0084   for (auto &entry : data) {
0085     test["tests"].push_back(entry.json);
0086   }
0087   std::cout << fmt::format("Writing test data to {}\n", fname);
0088   std::ofstream output_file(fname);
0089   output_file << std::setw(4) << test << "\n";
0090 }
0091
0092 /** Write a single test to json file.
0093  */
0094 inline void write_test(const Test &data, const std::string &fname) {
0095   std::vector<Test> vtd{data};
0096   write_test(vtd, fname);
0097 }
0098
0099 } // namespace common_bench
0100
0101 #endif