test skipping, CLI extension support

Author: Tsche

example/CMakeLists.txt

@@ -3,6 +3,7 @@ target_sources(example_test PRIVATE
     tparams.cpp
     params.cpp
     fixtures.cpp
+    conditional.cpp
 )
 
 add_executable(params_debug params_debug.cpp)

example/conditional.cpp

@@ -0,0 +1,40 @@
+#include <rsl/test>
+#include <rsl/config>
+
+namespace demo::conditional {
+
+[[=rsl::test, =rsl::skip]] 
+void never_run() {
+  ASSERT(false);
+}
+
+static constexpr bool skip = true;
+[[=rsl::test, =rsl::skip_if(skip)]] 
+void skip_constant() {
+  ASSERT(false);
+}
+
+bool should_skip() {
+  // determine at runtime whether to skip or not
+  return true;
+}
+
+[[=rsl::test, =rsl::skip_if(should_skip)]] 
+void skip_runtime() {
+  ASSERT(false);
+}
+
+struct CLI : rsl::cli_extension<CLI> {
+  [[=option, =flag]] bool skip_cli = false;
+};
+
+[[=rsl::test, =rsl::skip_if([] { return CLI::value.skip_cli; })]] 
+void skip_cli() {
+  ASSERT(false, "Pass --skip-cli to disable this test.");
+}
+
+[[=rsl::test, =rsl::skip_if(&CLI::skip_cli)]] 
+void skip_cli2() {
+  ASSERT(false, "Pass --skip-cli to disable this test.");
+}
+}  // namespace

example/tparams.cpp

@@ -12,7 +12,7 @@ consteval std::vector<rsl::testing::ParamSet> make_tparams() {
 [[=rsl::tparams({{^^int, 10}, {^^float, 21}})]]
 [[=rsl::params({{42, 'c'}, {12, 'b'}})]]
 constexpr inline auto tparam_gt_5 = []<typename T, unsigned I>(int a, char b) static {
-    ASSERT(I < 5);
+    ASSERT(I > 5);
 };
 
 struct TestAggregate{ unsigned value; };

include/rsl/testing/all.hpp

@@ -10,12 +10,19 @@
 #include <rsl/testing/_testing_impl/discovery.hpp>
 
 namespace rsl {
-using testing::expect_failure;
 using testing::fixture;
-using testing::params;
+using testing::fuzz;
+using testing::property_test;
 using testing::test;
+
+using testing::params;
 using testing::tparams;
 
+using testing::expect_failure;
+using testing::rename;
+using testing::skip;
+using testing::skip_if;
+
 }  // namespace rsl
 
 #ifndef RSLTEST_IMPL_USED

include/rsl/testing/annotations.hpp

@@ -1,4 +1,5 @@
 #pragma once
+#include <type_traits>
 #include <vector>
 #include <initializer_list>
 #include <meta>
@@ -29,15 +30,32 @@ struct Skip {
   bool (*value)() = &_testing_impl::constant_predicate<true>;
 };
 
-struct SkipIf {
-  static consteval Skip operator()(auto condition) {
-    // TODO wrap arbitrary conditions
-    return Skip();
-  }
-};
 
-struct Rename {
-  rsl::string_view value;
+namespace _impl { 
+template <typename T, bool T::*condition>
+bool wrap_cli_nsdm() {
+  return T::get().*condition;
+}
+}
+
+struct SkipIf {
+  static consteval Skip operator()(bool condition) {
+    return {extract<bool (*)()>(
+      substitute(^^_testing_impl::constant_predicate, {std::meta::reflect_constant(condition)}))};
+    }
+    
+    static consteval Skip operator()(bool (*condition)()) { return {condition}; }
+    
+    template <typename T>
+    requires requires { T::get(); }
+    static consteval Skip operator()(bool T::* condition)  {
+      return {extract<bool (*)()>(
+        substitute(^^_impl::wrap_cli_nsdm, {^^T, std::meta::reflect_constant(condition)}))};
+      }
+    };
+    
+    struct Rename {
+      rsl::string_view value;
 
   static consteval Rename operator()(std::string_view new_name) {
     return Rename(define_static_string(new_name));

include/rsl/testing/test_case.hpp

@@ -5,10 +5,17 @@
 
 namespace rsl::testing {
 
+enum class TestOutcome: uint8_t {
+  FAIL,
+  PASS,
+  SKIP
+};
+
 struct TestResult {
   class Test const* test;
   std::string name;
-  bool passed;
+
+  TestOutcome outcome;
   double duration_ms;
 
   std::string failure;

src/CMakeLists.txt

@@ -1,4 +1,4 @@
-target_sources(rsltest PRIVATE 
+target_sources(rsltest PUBLIC 
     capture.cpp
     test.cpp
 )

src/main/reporters/catch2xml.cpp

@@ -198,11 +198,17 @@ class[[= rename("xml")]] Catch2XmlReporter : public Reporter::Registrar<Catch2Xm
       section->stderr = {.value = result.stderr};
     }
 
-    if (result.passed) {
-      ++section->results.successes;
-    } else {
-      section->failure = {.value = result.failure};
-      ++section->results.failures;
+    switch (result.outcome) {
+      using enum TestOutcome;
+      case PASS: ++section->results.successes; break;
+      case FAIL:
+        section->failure = {.value = result.failure};
+        ++section->results.failures;
+        break;
+      case SKIP:
+        section->results.skipped = true;
+        ++tc.result.skips;
+        break;
     }
     section->results.durationInSeconds += result.duration_ms / 1000.;
 

src/main/reporters/terminal.cpp

@@ -14,7 +14,7 @@ class [[=rename("plain")]] ConsoleReporter : public Reporter::Registrar<ConsoleR
     auto const color = std::array{must_colorize ? "\033[32m" : "", must_colorize ? "\033[31m" : ""};
     const char* const reset = must_colorize ? "\033[0m" : "";
 
-    if (result.passed) {
+    if (result.outcome == TestOutcome::PASS) {
       std::print("[{}       OK {}] {} ({:.3f} ms)\n",
                  color[0],
                  reset,
@@ -32,7 +32,7 @@ class [[=rename("plain")]] ConsoleReporter : public Reporter::Registrar<ConsoleR
     }
   }
   void after_run(std::span<TestResult> results) override {
-    auto passed = std::ranges::count_if(results, [](auto& r) { return r.passed; });
+    auto passed = std::ranges::count_if(results, [](auto& r) { return r.outcome == TestOutcome::PASS; });
     std::print("=== Summary ===\n{} / {} tests passed.\n", passed, results.size());
   }
 

src/main/reporters/xml.cpp

@@ -22,7 +22,7 @@ class [[=rename("junit")]] JUnitXmlReporter : public Reporter::Registrar<JUnitXm
   void before_test(TestCase const& test) override {}
   void after_test(TestResult const& result) override {
     auto node = testcase{.name=std::string(result.name), .time=result.duration_ms / 1000.};
-    if (!result.passed) {
+    if (result.outcome == TestOutcome::FAIL) {
       node.failure = result.failure + "\n";
     }
     suite.tests.push_back(node);