catch2/interfaces/catch_interfaces_reporter.hpp

0001
0002 //              Copyright Catch2 Authors
0003 // Distributed under the Boost Software License, Version 1.0.
0004 //   (See accompanying file LICENSE.txt or copy at
0005 //        https://www.boost.org/LICENSE_1_0.txt)
0006
0007 // SPDX-License-Identifier: BSL-1.0
0008 #ifndef CATCH_INTERFACES_REPORTER_HPP_INCLUDED
0009 #define CATCH_INTERFACES_REPORTER_HPP_INCLUDED
0010
0011 #include <catch2/catch_section_info.hpp>
0012 #include <catch2/catch_totals.hpp>
0013 #include <catch2/catch_assertion_result.hpp>
0014 #include <catch2/internal/catch_message_info.hpp>
0015 #include <catch2/internal/catch_stringref.hpp>
0016 #include <catch2/internal/catch_test_run_info.hpp>
0017 #include <catch2/internal/catch_unique_ptr.hpp>
0018 #include <catch2/benchmark/detail/catch_benchmark_stats.hpp>
0019
0020 #include <map>
0021 #include <string>
0022 #include <vector>
0023 #include <iosfwd>
0024
0025 namespace Catch {
0026
0027     struct ReporterDescription;
0028     struct ListenerDescription;
0029     struct TagInfo;
0030     struct TestCaseInfo;
0031     class TestCaseHandle;
0032     class IConfig;
0033     class IStream;
0034     enum class ColourMode : std::uint8_t;
0035
0036     struct ReporterConfig {
0037         ReporterConfig( IConfig const* _fullConfig,
0038                         Detail::unique_ptr<IStream> _stream,
0039                         ColourMode colourMode,
0040                         std::map<std::string, std::string> customOptions );
0041
0042         ReporterConfig( ReporterConfig&& ) = default;
0043         ReporterConfig& operator=( ReporterConfig&& ) = default;
0044         ~ReporterConfig(); // = default
0045
0046         Detail::unique_ptr<IStream> takeStream() &&;
0047         IConfig const* fullConfig() const;
0048         ColourMode colourMode() const;
0049         std::map<std::string, std::string> const& customOptions() const;
0050
0051     private:
0052         Detail::unique_ptr<IStream> m_stream;
0053         IConfig const* m_fullConfig;
0054         ColourMode m_colourMode;
0055         std::map<std::string, std::string> m_customOptions;
0056     };
0057
0058     struct AssertionStats {
0059         AssertionStats( AssertionResult const& _assertionResult,
0060                         std::vector<MessageInfo> const& _infoMessages,
0061                         Totals const& _totals );
0062
0063         AssertionStats( AssertionStats const& )              = default;
0064         AssertionStats( AssertionStats && )                  = default;
0065         AssertionStats& operator = ( AssertionStats const& ) = delete;
0066         AssertionStats& operator = ( AssertionStats && )     = delete;
0067
0068         AssertionResult assertionResult;
0069         std::vector<MessageInfo> infoMessages;
0070         Totals totals;
0071     };
0072
0073     struct SectionStats {
0074         SectionStats(   SectionInfo&& _sectionInfo,
0075                         Counts const& _assertions,
0076                         double _durationInSeconds,
0077                         bool _missingAssertions );
0078
0079         SectionInfo sectionInfo;
0080         Counts assertions;
0081         double durationInSeconds;
0082         bool missingAssertions;
0083     };
0084
0085     struct TestCaseStats {
0086         TestCaseStats(  TestCaseInfo const& _testInfo,
0087                         Totals const& _totals,
0088                         std::string&& _stdOut,
0089                         std::string&& _stdErr,
0090                         bool _aborting );
0091
0092         TestCaseInfo const * testInfo;
0093         Totals totals;
0094         std::string stdOut;
0095         std::string stdErr;
0096         bool aborting;
0097     };
0098
0099     struct TestRunStats {
0100         TestRunStats(   TestRunInfo const& _runInfo,
0101                         Totals const& _totals,
0102                         bool _aborting );
0103
0104         TestRunInfo runInfo;
0105         Totals totals;
0106         bool aborting;
0107     };
0108
0109     //! By setting up its preferences, a reporter can modify Catch2's behaviour
0110     //! in some regards, e.g. it can request Catch2 to capture writes to
0111     //! stdout/stderr during test execution, and pass them to the reporter.
0112     struct ReporterPreferences {
0113         //! Catch2 should redirect writes to stdout and pass them to the
0114         //! reporter
0115         bool shouldRedirectStdOut = false;
0116         //! Catch2 should call `Reporter::assertionEnded` even for passing
0117         //! assertions
0118         bool shouldReportAllAssertions = false;
0119     };
0120
0121     /**
0122      * The common base for all reporters and event listeners
0123      *
0124      * Implementing classes must also implement:
0125      *
0126      *     //! User-friendly description of the reporter/listener type
0127      *     static std::string getDescription()
0128      *
0129      * Generally shouldn't be derived from by users of Catch2 directly,
0130      * instead they should derive from one of the utility bases that
0131      * derive from this class.
0132      */
0133     class IEventListener {
0134     protected:
0135         //! Derived classes can set up their preferences here
0136         ReporterPreferences m_preferences;
0137         //! The test run's config as filled in from CLI and defaults
0138         IConfig const* m_config;
0139
0140     public:
0141         IEventListener( IConfig const* config ): m_config( config ) {}
0142
0143         virtual ~IEventListener(); // = default;
0144
0145         // Implementing class must also provide the following static methods:
0146         // static std::string getDescription();
0147
0148         ReporterPreferences const& getPreferences() const {
0149             return m_preferences;
0150         }
0151
0152         //! Called when no test cases match provided test spec
0153         virtual void noMatchingTestCases( StringRef unmatchedSpec ) = 0;
0154         //! Called for all invalid test specs from the cli
0155         virtual void reportInvalidTestSpec( StringRef invalidArgument ) = 0;
0156
0157         /**
0158          * Called once in a testing run before tests are started
0159          *
0160          * Not called if tests won't be run (e.g. only listing will happen)
0161          */
0162         virtual void testRunStarting( TestRunInfo const& testRunInfo ) = 0;
0163
0164         //! Called _once_ for each TEST_CASE, no matter how many times it is entered
0165         virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
0166         //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
0167         virtual void testCasePartialStarting( TestCaseInfo const& testInfo, uint64_t partNumber ) = 0;
0168         //! Called when a `SECTION` is being entered. Not called for skipped sections
0169         virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;
0170
0171         //! Called when user-code is being probed before the actual benchmark runs
0172         virtual void benchmarkPreparing( StringRef benchmarkName ) = 0;
0173         //! Called after probe but before the user-code is being benchmarked
0174         virtual void benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) = 0;
0175         //! Called with the benchmark results if benchmark successfully finishes
0176         virtual void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) = 0;
0177         //! Called if running the benchmarks fails for any reason
0178         virtual void benchmarkFailed( StringRef benchmarkName ) = 0;
0179
0180         //! Called before assertion success/failure is evaluated
0181         virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0;
0182
0183         //! Called after assertion was fully evaluated
0184         virtual void assertionEnded( AssertionStats const& assertionStats ) = 0;
0185
0186         //! Called after a `SECTION` has finished running
0187         virtual void sectionEnded( SectionStats const& sectionStats ) = 0;
0188         //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
0189         virtual void testCasePartialEnded(TestCaseStats const& testCaseStats, uint64_t partNumber ) = 0;
0190         //! Called _once_ for each TEST_CASE, no matter how many times it is entered
0191         virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0;
0192         /**
0193          * Called once after all tests in a testing run are finished
0194          *
0195          * Not called if tests weren't run (e.g. only listings happened)
0196          */
0197         virtual void testRunEnded( TestRunStats const& testRunStats ) = 0;
0198
0199         /**
0200          * Called with test cases that are skipped due to the test run aborting.
0201          * NOT called for test cases that are explicitly skipped using the `SKIP` macro.
0202          *
0203          * Deprecated - will be removed in the next major release.
0204          */
0205         virtual void skipTest( TestCaseInfo const& testInfo ) = 0;
0206
0207         //! Called if a fatal error (signal/structured exception) occurred
0208         virtual void fatalErrorEncountered( StringRef error ) = 0;
0209
0210         //! Writes out information about provided reporters using reporter-specific format
0211         virtual void listReporters(std::vector<ReporterDescription> const& descriptions) = 0;
0212         //! Writes out the provided listeners descriptions using reporter-specific format
0213         virtual void listListeners(std::vector<ListenerDescription> const& descriptions) = 0;
0214         //! Writes out information about provided tests using reporter-specific format
0215         virtual void listTests(std::vector<TestCaseHandle> const& tests) = 0;
0216         //! Writes out information about the provided tags using reporter-specific format
0217         virtual void listTags(std::vector<TagInfo> const& tags) = 0;
0218     };
0219     using IEventListenerPtr = Detail::unique_ptr<IEventListener>;
0220
0221 } // end namespace Catch
0222
0223 #endif // CATCH_INTERFACES_REPORTER_HPP_INCLUDED