Supporting running a test suite against more than one schema (#768)

Signed-off-by: Juan Cruz Viotti <jv@jviotti.com>

Author: jviotti

src/test/include/sourcemeta/blaze/test.h

@@ -90,17 +90,17 @@ struct SOURCEMETA_BLAZE_TEST_EXPORT TestSuite {
 #if defined(_MSC_VER)
 #pragma warning(disable : 4251)
 #endif
-  /// The target schema URI or file path
-  sourcemeta::core::JSON::String target;
+  /// The target schema URIs or file paths
+  std::vector<sourcemeta::core::JSON::String> targets;
   /// The list of test cases in the suite
   std::vector<TestCase> tests;
+  /// The compiled schema templates for fast validation
+  std::vector<Template> schemas_fast;
+  /// The compiled schema templates for exhaustive validation
+  std::vector<Template> schemas_exhaustive;
 #if defined(_MSC_VER)
 #pragma warning(default : 4251)
 #endif
-  /// The compiled schema template for fast validation
-  Template schema_fast;
-  /// The compiled schema template for exhaustive validation
-  Template schema_exhaustive;
   /// The evaluator instance used for validation
   Evaluator evaluator;
 
@@ -193,7 +193,9 @@ struct SOURCEMETA_BLAZE_TEST_EXPORT TestSuite {
   ///     sourcemeta::core::schema_walker,
   ///     sourcemeta::blaze::default_schema_compiler)};
   ///
-  /// assert(suite.target == "https://json-schema.org/draft/2020-12/schema");
+  /// assert(suite.targets.size() == 1);
+  /// assert(suite.targets.front() ==
+  ///   "https://json-schema.org/draft/2020-12/schema");
   /// assert(suite.tests.size() == 2);
   /// ```
   static auto

src/test/test_parser.cc

@@ -101,9 +101,11 @@ auto TestSuite::parse(const sourcemeta::core::JSON &document,
   TEST_ERROR_IF(!document.defines("target"), tracker,
                 sourcemeta::core::empty_pointer,
                 "The test document must contain a `target` property");
-  TEST_ERROR_IF(!document.at("target").is_string(), tracker,
-                sourcemeta::core::Pointer{"target"},
-                "The test document `target` property must be a URI");
+  TEST_ERROR_IF(!document.at("target").is_string() &&
+                    !document.at("target").is_array(),
+                tracker, sourcemeta::core::Pointer{"target"},
+                "The test document `target` property must be a URI or an "
+                "array of URIs");
   TEST_ERROR_IF(!document.defines("tests"), tracker,
                 sourcemeta::core::empty_pointer,
                 "The test document must contain a `tests` property");
@@ -113,12 +115,33 @@ auto TestSuite::parse(const sourcemeta::core::JSON &document,
 
   const auto base_path_uri{
       sourcemeta::core::URI::from_path(base_path / "test.json")};
-  sourcemeta::core::URI schema_uri{document.at("target").to_string()};
-  schema_uri.resolve_from(base_path_uri);
-  schema_uri.canonicalize();
 
   TestSuite test_suite;
-  test_suite.target = schema_uri.recompose();
+
+  if (document.at("target").is_string()) {
+    sourcemeta::core::URI schema_uri{document.at("target").to_string()};
+    schema_uri.resolve_from(base_path_uri);
+    schema_uri.canonicalize();
+    test_suite.targets.push_back(schema_uri.recompose());
+  } else {
+    TEST_ERROR_IF(document.at("target").empty(), tracker,
+                  sourcemeta::core::Pointer{"target"},
+                  "The test document `target` array must contain at least "
+                  "one URI");
+    // TODO(C++23): Use std::views::enumerate when available in libc++
+    std::size_t target_index{0};
+    for (const auto &target_entry : document.at("target").as_array()) {
+      const sourcemeta::core::Pointer target_location{"target", target_index};
+      TEST_ERROR_IF(!target_entry.is_string(), tracker, target_location,
+                    "Each entry in the test document `target` array must be "
+                    "a URI");
+      sourcemeta::core::URI schema_uri{target_entry.to_string()};
+      schema_uri.resolve_from(base_path_uri);
+      schema_uri.canonicalize();
+      test_suite.targets.push_back(schema_uri.recompose());
+      target_index += 1;
+    }
+  }
 
   // TODO(C++23): Use std::views::enumerate when available in libc++
   std::size_t index{0};
@@ -131,23 +154,28 @@ auto TestSuite::parse(const sourcemeta::core::JSON &document,
     index += 1;
   }
 
-  const auto target_schema{sourcemeta::core::wrap(test_suite.target)};
-
-  try {
-    test_suite.schema_fast =
-        compile(target_schema, walker, schema_resolver, compiler,
-                Mode::FastValidation, default_dialect, default_id, "", tweaks);
-    test_suite.schema_exhaustive =
-        compile(target_schema, walker, schema_resolver, compiler,
-                Mode::Exhaustive, default_dialect, default_id, "", tweaks);
-  } catch (const sourcemeta::core::SchemaReferenceError &error) {
-    if (error.location() == sourcemeta::core::Pointer{"$ref"} &&
-        error.identifier() == test_suite.target) {
-      throw sourcemeta::core::SchemaResolutionError{
-          test_suite.target, "Could not resolve schema under test"};
+  test_suite.schemas_fast.reserve(test_suite.targets.size());
+  test_suite.schemas_exhaustive.reserve(test_suite.targets.size());
+
+  for (const auto &target : test_suite.targets) {
+    const auto target_schema{sourcemeta::core::wrap(target)};
+
+    try {
+      test_suite.schemas_fast.push_back(compile(
+          target_schema, walker, schema_resolver, compiler,
+          Mode::FastValidation, default_dialect, default_id, "", tweaks));
+      test_suite.schemas_exhaustive.push_back(
+          compile(target_schema, walker, schema_resolver, compiler,
+                  Mode::Exhaustive, default_dialect, default_id, "", tweaks));
+    } catch (const sourcemeta::core::SchemaReferenceError &error) {
+      if (error.location() == sourcemeta::core::Pointer{"$ref"} &&
+          error.identifier() == target) {
+        throw sourcemeta::core::SchemaResolutionError{
+            target, "Could not resolve schema under test"};
+      }
+
+      throw;
     }
-
-    throw;
   }
 
   return test_suite;

src/test/test_runner.cc

@@ -5,21 +5,26 @@
 namespace sourcemeta::blaze {
 
 auto TestSuite::run(const Callback &callback) -> Result {
-  Result result{.total = this->tests.size(),
+  const auto total{this->targets.size() * this->tests.size()};
+  Result result{.total = total,
                 .passed = 0,
                 .start = std::chrono::steady_clock::now(),
                 .end = {}};
-  // TODO(C++23): Use std::views::enumerate when available in libc++
-  for (std::size_t index = 0; index < this->tests.size(); ++index) {
-    const auto &test_case = this->tests[index];
-    const auto start{std::chrono::steady_clock::now()};
-    const auto actual{
-        this->evaluator.validate(this->schema_fast, test_case.data)};
-    const auto end{std::chrono::steady_clock::now()};
-    callback(this->target, index + 1, this->tests.size(), test_case, actual,
-             start, end);
-    if (test_case.valid == actual) {
-      result.passed += 1;
+
+  std::size_t step{0};
+  for (std::size_t target_index = 0; target_index < this->targets.size();
+       ++target_index) {
+    const auto &target = this->targets[target_index];
+    const auto &schema_fast = this->schemas_fast[target_index];
+    for (const auto &test_case : this->tests) {
+      const auto start{std::chrono::steady_clock::now()};
+      const auto actual{this->evaluator.validate(schema_fast, test_case.data)};
+      const auto end{std::chrono::steady_clock::now()};
+      step += 1;
+      callback(target, step, total, test_case, actual, start, end);
+      if (test_case.valid == actual) {
+        result.passed += 1;
+      }
     }
   }