catch_discover_tests uses tempfile to retrieve JSON from the binary

This allows it to deal with badly behaved code, where 3rd party
dependencies write into stdout during global construction.

Closes #3162
Closes #3166

Author: horenmar

extras/Catch.cmake

@@ -44,7 +44,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
 
   ``catch_discover_tests`` sets up a post-build command on the test executable
   that generates the list of tests by parsing the output from running the test
-  with the ``--list-test-names-only`` argument.  This ensures that the full
+  with the ``--list-tests --reporter json`` argument.  This ensures that the full
   list of tests is obtained.  Since test discovery occurs at build time, it is
   not necessary to re-run CMake when the list of tests changes.
   However, it requires that :prop_tgt:`CROSSCOMPILING_EMULATOR` is properly set
@@ -67,7 +67,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
 
   ``TEST_SPEC arg1...``
     Specifies test cases, wildcarded test cases, tags and tag expressions to
-    pass to the Catch executable with the ``--list-test-names-only`` argument.
+    pass to the Catch executable when listing the tests.
 
   ``EXTRA_ARGS arg1...``
     Any extra arguments to pass on the command line to each test case.

extras/CatchAddTests.cmake

@@ -16,6 +16,42 @@ function(add_command NAME)
   set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE)
 endfunction()
 
+# Generates random filename in the temp folder.
+# Temp folder is retrieved by checking env vars from various platforms.
+function(make_temp_file_path OUT_VARIABLE FALLBACK_PATH)
+  set(TEMP_DIR "")
+  set(ENV_VARS
+    # From XDG base dir specification
+    XDG_RUNTIME_DIR
+    # From POSIX standard
+    TMPDIR
+    # From Windows
+    TMP
+    TEMP
+  )
+
+  foreach(var ${ENV_VARS})
+    message(NOTICE "Checking ${var} env var")
+    if(DEFINED ENV{${var}} AND NOT "$ENV{${var}}" STREQUAL "")
+        set(TEMP_DIR "$ENV{${var}}")
+        break()
+    endif()
+  endforeach()
+
+  # If all checks fail, we use the fallback path
+  if(TEMP_DIR STREQUAL "")
+    set(TEMP_DIR "${FALLBACK_PATH}")
+  endif()
+
+  file(TO_CMAKE_PATH "${TEMP_DIR}" TEMP_DIR)
+
+  # Generate the random file name
+  string(RANDOM LENGTH 8 RAND_ID)
+  set(FINAL_TEMP_PATH "${TEMP_DIR}/Catch2-test-listing.${RAND_ID}.json")
+
+  set(${OUT_VARIABLE} "${FINAL_TEMP_PATH}" PARENT_SCOPE)
+endfunction()
+
 function(catch_discover_tests_impl)
 
   cmake_parse_arguments(
@@ -72,8 +108,13 @@ function(catch_discover_tests_impl)
     set(ENV{DYLD_FRAMEWORK_PATH} "${paths}")
   endif()
 
+  make_temp_file_path(listing_output_path "${_TEST_WORKING_DIR}")
+
   execute_process(
-    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --reporter json
+    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec}
+      --list-tests
+      --reporter json
+      --out "${listing_output_path}"
     OUTPUT_VARIABLE listing_output
     RESULT_VARIABLE result
     WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
@@ -86,6 +127,10 @@ function(catch_discover_tests_impl)
     )
   endif()
 
+  # Read the JSON output back from the output file (and then get rid of the file)
+  file(READ ${listing_output_path} listing_output)
+  file(REMOVE ${listing_output_path})
+
   # Prepare reporter
   if(reporter)
     set(reporter_arg "--reporter ${reporter}")

tests/TestScripts/DiscoverTests/VerifyRegistration.py

@@ -12,6 +12,7 @@
 import sys
 import re
 import json
+import tempfile
 from collections import namedtuple
 from typing import List
 
@@ -72,14 +73,19 @@ def get_test_names(build_path: str) -> List[TestInfo]:
     config_path = "Debug" if os.name == 'nt' else ""
     full_path = os.path.join(build_path, config_path, 'tests')
 
-
-    cmd = [full_path, '--reporter', 'json', '--list-tests']
-    result = subprocess.run(cmd,
-                            capture_output = True,
-                            check = True,
-                            text = True)
-
-    test_listing = json.loads(result.stdout)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        fname = f'{tmpdir}/listing-output.json'
+        cmd = [full_path,
+          '--list-tests',
+          '--reporter', 'json',
+          '--out', fname
+        ]
+        result = subprocess.run(cmd,
+                                capture_output = False,
+                                check = True,
+                                text = True)
+        with open(fname, mode='r', encoding='utf-8') as file:
+            test_listing = json.load(file)
 
     assert test_listing['version'] == 1
 
@@ -96,10 +102,18 @@ def get_ctest_listing(build_path):
     os.chdir(build_path)
 
     cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
-    result = subprocess.run(cmd,
-                            capture_output = True,
-                            check = True,
-                            text = True)
+    try:
+        result = subprocess.run(cmd,
+                                capture_output = True,
+                                check = True,
+                                text = True)
+    except subprocess.CalledProcessError as err:
+        print('Error when getting output from CTest')
+        print(f'cmd: {err.cmd}')
+        print(f'stderr: {err.stderr}')
+        print(f'stdout: {err.stdout}')
+        exit(4)
+
     os.chdir(old_path)
     return result.stdout
 

tests/TestScripts/DiscoverTests/register-tests.cpp

@@ -8,6 +8,24 @@
 
 #include <catch2/catch_test_macros.hpp>
 
+#include <cstdio>
+#include <iostream>
+
+namespace {
+
+struct PrintsWhenConstructed {
+    PrintsWhenConstructed() {
+        std::cout << "Hello\n";
+        std::cerr << "Holla\n";
+        std::fprintf(stdout, "Hullo\n");
+        std::fprintf(stderr, "Hillo\n");
+    }
+};
+
+static PrintsWhenConstructed instance;
+
+}
+
 TEST_CASE("@Script[C:\\EPM1A]=x;\"SCALA_ZERO:\"", "[script regressions]"){}
 TEST_CASE("Some test") {}
 TEST_CASE( "Let's have a test case with a long name. Longer. No, even longer. "