catch2/interfaces/catch_interfaces_reporter.hpp

0001
0002 //              Copyright Catch2 Authors
0003 // Distributed under the Boost Software License, Version 1.0.
0004 //   (See accompanying file LICENSE.txt or copy at
0005 //        https://www.boost.org/LICENSE_1_0.txt)
0006
0007 // SPDX-License-Identifier: BSL-1.0
0008 #ifndef CATCH_INTERFACES_REPORTER_HPP_INCLUDED
0009 #define CATCH_INTERFACES_REPORTER_HPP_INCLUDED
0010
0011 #include <catch2/catch_section_info.hpp>
0012 #include <catch2/catch_totals.hpp>
0013 #include <catch2/catch_assertion_result.hpp>
0014 #include <catch2/internal/catch_message_info.hpp>
0015 #include <catch2/internal/catch_stringref.hpp>
0016 #include <catch2/internal/catch_test_run_info.hpp>
0017 #include <catch2/internal/catch_unique_ptr.hpp>
0018 #include <catch2/internal/catch_move_and_forward.hpp>
0019 #include <catch2/benchmark/detail/catch_benchmark_stats.hpp>
0020
0021 #include <map>
0022 #include <string>
0023 #include <vector>
0024 #include <iosfwd>
0025
0026 namespace Catch {
0027
0028     struct ReporterDescription;
0029     struct ListenerDescription;
0030     struct TagInfo;
0031     struct TestCaseInfo;
0032     class TestCaseHandle;
0033     class IConfig;
0034     class IStream;
0035     enum class ColourMode : std::uint8_t;
0036
0037     struct ReporterConfig {
0038         ReporterConfig( IConfig const* _fullConfig,
0039                         Detail::unique_ptr<IStream> _stream,
0040                         ColourMode colourMode,
0041                         std::map<std::string, std::string> customOptions );
0042
0043         ReporterConfig( ReporterConfig&& ) = default;
0044         ReporterConfig& operator=( ReporterConfig&& ) = default;
0045         ~ReporterConfig(); // = default
0046
0047         Detail::unique_ptr<IStream> takeStream() &&;
0048         IConfig const* fullConfig() const;
0049         ColourMode colourMode() const;
0050         std::map<std::string, std::string> const& customOptions() const;
0051
0052     private:
0053         Detail::unique_ptr<IStream> m_stream;
0054         IConfig const* m_fullConfig;
0055         ColourMode m_colourMode;
0056         std::map<std::string, std::string> m_customOptions;
0057     };
0058
0059     struct AssertionStats {
0060         AssertionStats( AssertionResult const& _assertionResult,
0061                         std::vector<MessageInfo> const& _infoMessages,
0062                         Totals const& _totals );
0063
0064         AssertionStats( AssertionStats const& )              = default;
0065         AssertionStats( AssertionStats && )                  = default;
0066         AssertionStats& operator = ( AssertionStats const& ) = delete;
0067         AssertionStats& operator = ( AssertionStats && )     = delete;
0068
0069         AssertionResult assertionResult;
0070         std::vector<MessageInfo> infoMessages;
0071         Totals totals;
0072     };
0073
0074     struct SectionStats {
0075         SectionStats(   SectionInfo&& _sectionInfo,
0076                         Counts const& _assertions,
0077                         double _durationInSeconds,
0078                         bool _missingAssertions );
0079
0080         SectionInfo sectionInfo;
0081         Counts assertions;
0082         double durationInSeconds;
0083         bool missingAssertions;
0084     };
0085
0086     struct TestCaseStats {
0087         TestCaseStats(  TestCaseInfo const& _testInfo,
0088                         Totals const& _totals,
0089                         std::string&& _stdOut,
0090                         std::string&& _stdErr,
0091                         bool _aborting );
0092
0093         TestCaseInfo const * testInfo;
0094         Totals totals;
0095         std::string stdOut;
0096         std::string stdErr;
0097         bool aborting;
0098     };
0099
0100     struct TestRunStats {
0101         TestRunStats(   TestRunInfo const& _runInfo,
0102                         Totals const& _totals,
0103                         bool _aborting );
0104
0105         TestRunInfo runInfo;
0106         Totals totals;
0107         bool aborting;
0108     };
0109
0110     //! By setting up its preferences, a reporter can modify Catch2's behaviour
0111     //! in some regards, e.g. it can request Catch2 to capture writes to
0112     //! stdout/stderr during test execution, and pass them to the reporter.
0113     struct ReporterPreferences {
0114         //! Catch2 should redirect writes to stdout and pass them to the
0115         //! reporter
0116         bool shouldRedirectStdOut = false;
0117         //! Catch2 should call `Reporter::assertionEnded` even for passing
0118         //! assertions
0119         bool shouldReportAllAssertions = false;
0120     };
0121
0122     /**
0123      * The common base for all reporters and event listeners
0124      *
0125      * Implementing classes must also implement:
0126      *
0127      *     //! User-friendly description of the reporter/listener type
0128      *     static std::string getDescription()
0129      *
0130      * Generally shouldn't be derived from by users of Catch2 directly,
0131      * instead they should derive from one of the utility bases that
0132      * derive from this class.
0133      */
0134     class IEventListener {
0135     protected:
0136         //! Derived classes can set up their preferences here
0137         ReporterPreferences m_preferences;
0138         //! The test run's config as filled in from CLI and defaults
0139         IConfig const* m_config;
0140
0141     public:
0142         IEventListener( IConfig const* config ): m_config( config ) {}
0143
0144         virtual ~IEventListener(); // = default;
0145
0146         // Implementing class must also provide the following static methods:
0147         // static std::string getDescription();
0148
0149         ReporterPreferences const& getPreferences() const {
0150             return m_preferences;
0151         }
0152
0153         //! Called when no test cases match provided test spec
0154         virtual void noMatchingTestCases( StringRef unmatchedSpec ) = 0;
0155         //! Called for all invalid test specs from the cli
0156         virtual void reportInvalidTestSpec( StringRef invalidArgument ) = 0;
0157
0158         /**
0159          * Called once in a testing run before tests are started
0160          *
0161          * Not called if tests won't be run (e.g. only listing will happen)
0162          */
0163         virtual void testRunStarting( TestRunInfo const& testRunInfo ) = 0;
0164
0165         //! Called _once_ for each TEST_CASE, no matter how many times it is entered
0166         virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
0167         //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
0168         virtual void testCasePartialStarting( TestCaseInfo const& testInfo, uint64_t partNumber ) = 0;
0169         //! Called when a `SECTION` is being entered. Not called for skipped sections
0170         virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;
0171
0172         //! Called when user-code is being probed before the actual benchmark runs
0173         virtual void benchmarkPreparing( StringRef benchmarkName ) = 0;
0174         //! Called after probe but before the user-code is being benchmarked
0175         virtual void benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) = 0;
0176         //! Called with the benchmark results if benchmark successfully finishes
0177         virtual void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) = 0;
0178         //! Called if running the benchmarks fails for any reason
0179         virtual void benchmarkFailed( StringRef benchmarkName ) = 0;
0180
0181         //! Called before assertion success/failure is evaluated
0182         virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0;
0183
0184         //! Called after assertion was fully evaluated
0185         virtual void assertionEnded( AssertionStats const& assertionStats ) = 0;
0186
0187         //! Called after a `SECTION` has finished running
0188         virtual void sectionEnded( SectionStats const& sectionStats ) = 0;
0189         //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
0190         virtual void testCasePartialEnded(TestCaseStats const& testCaseStats, uint64_t partNumber ) = 0;
0191         //! Called _once_ for each TEST_CASE, no matter how many times it is entered
0192         virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0;
0193         /**
0194          * Called once after all tests in a testing run are finished
0195          *
0196          * Not called if tests weren't run (e.g. only listings happened)
0197          */
0198         virtual void testRunEnded( TestRunStats const& testRunStats ) = 0;
0199
0200         /**
0201          * Called with test cases that are skipped due to the test run aborting.
0202          * NOT called for test cases that are explicitly skipped using the `SKIP` macro.
0203          *
0204          * Deprecated - will be removed in the next major release.
0205          */
0206         virtual void skipTest( TestCaseInfo const& testInfo ) = 0;
0207
0208         //! Called if a fatal error (signal/structured exception) occurred
0209         virtual void fatalErrorEncountered( StringRef error ) = 0;
0210
0211         //! Writes out information about provided reporters using reporter-specific format
0212         virtual void listReporters(std::vector<ReporterDescription> const& descriptions) = 0;
0213         //! Writes out the provided listeners descriptions using reporter-specific format
0214         virtual void listListeners(std::vector<ListenerDescription> const& descriptions) = 0;
0215         //! Writes out information about provided tests using reporter-specific format
0216         virtual void listTests(std::vector<TestCaseHandle> const& tests) = 0;
0217         //! Writes out information about the provided tags using reporter-specific format
0218         virtual void listTags(std::vector<TagInfo> const& tags) = 0;
0219     };
0220     using IEventListenerPtr = Detail::unique_ptr<IEventListener>;
0221
0222 } // end namespace Catch
0223
0224 #endif // CATCH_INTERFACES_REPORTER_HPP_INCLUDED