diff --git a/.devcontainer/toolshed-amd64/devcontainer.json b/.devcontainer/toolshed-amd64/devcontainer.json index b7f0fc92..0abb8d44 100644 --- a/.devcontainer/toolshed-amd64/devcontainer.json +++ b/.devcontainer/toolshed-amd64/devcontainer.json @@ -19,7 +19,6 @@ "streetsidesoftware.code-spell-checker", "xaver.clang-format", "vadimcn.vscode-lldb", - "matepek.vscode-catch2-test-adapter", "ms-python.python", "ms-python.mypy-type-checker", "ms-python.black-formatter", diff --git a/.devcontainer/toolshed-arm64/devcontainer.json b/.devcontainer/toolshed-arm64/devcontainer.json index 751399a5..b162f09a 100644 --- a/.devcontainer/toolshed-arm64/devcontainer.json +++ b/.devcontainer/toolshed-arm64/devcontainer.json @@ -19,7 +19,6 @@ "streetsidesoftware.code-spell-checker", "xaver.clang-format", "vadimcn.vscode-lldb", - "matepek.vscode-catch2-test-adapter", "ms-python.python", "ms-python.mypy-type-checker", "ms-python.black-formatter", diff --git a/.devcontainer/toolshed-cov-amd64/devcontainer.json b/.devcontainer/toolshed-cov-amd64/devcontainer.json deleted file mode 100644 index 42b924ad..00000000 --- a/.devcontainer/toolshed-cov-amd64/devcontainer.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "C/C++ verification environment with lcov (amd64)", - "image": "ghcr.io/opencyphal/toolshed:ts22.4.10", - "workspaceFolder": "/repo", - "workspaceMount": "source=${localWorkspaceFolder},target=/repo,type=bind,consistency=delegated", - "mounts": [ - "source=profile-nunavut-c,target=/root,type=volume", - "target=/root/.vscode-server,type=volume" - ], - "customizations": { - "vscode": { - "extensions": [ - "uavcan.dsdl", - "wholroyd.jinja", - "ms-vscode.cpptools", - "ms-vscode.cpptools-themes", - "ms-vscode.cmake-tools", - "josetr.cmake-language-support-vscode", - "streetsidesoftware.code-spell-checker", - "xaver.clang-format", - "vadimcn.vscode-lldb", - "matepek.vscode-catch2-test-adapter", - "ms-python.python", - "ms-python.mypy-type-checker", - "ms-python.black-formatter", - "ms-python.pylint" - ] - } - }, - "runArgs": ["--platform=linux/amd64", "-i"], - "postCreateCommand": "git submodule update --init --recursive" -} diff --git a/.devcontainer/toolshed-cov-arm64/devcontainer.json b/.devcontainer/toolshed-cov-arm64/devcontainer.json deleted file mode 100644 index b3fbfe70..00000000 --- a/.devcontainer/toolshed-cov-arm64/devcontainer.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "C/C++ verification environment with lcov (arm64)", - "image": "ghcr.io/opencyphal/toolshed:ts22.4.10", - "workspaceFolder": "/repo", - "workspaceMount": "source=${localWorkspaceFolder},target=/repo,type=bind,consistency=delegated", - "mounts": [ - "source=profile-nunavut-c,target=/root,type=volume", - "target=/root/.vscode-server,type=volume" - ], - "customizations": { - "vscode": { - "extensions": [ - "uavcan.dsdl", - "wholroyd.jinja", - "ms-vscode.cpptools", - "ms-vscode.cpptools-themes", - "ms-vscode.cmake-tools", - "josetr.cmake-language-support-vscode", - "streetsidesoftware.code-spell-checker", - "xaver.clang-format", - "vadimcn.vscode-lldb", - "matepek.vscode-catch2-test-adapter", - "ms-python.python", - "ms-python.mypy-type-checker", - "ms-python.black-formatter", - "ms-python.pylint" - ] - } - }, - "runArgs": ["--platform=linux/arm64", "-i"], - "postCreateCommand": "git submodule update --init --recursive" -} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac1df76e..2a4f7e67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -172,12 +172,14 @@ jobs: uses: actions/upload-artifact@v4 with: name: verification-c-coverage-reports - path: verification/build/DebugCov/coverage/* + path: | + verification/build/DebugCov/coverage/html/ + verification/build/DebugCov/coverage/lcov/ language-verification-cpp-clang-native-extra: runs-on: ubuntu-latest needs: test - container: ghcr.io/opencyphal/toolshed:ts22.4.10 + container: ghcr.io/opencyphal/toolshed:ts24.4.3 steps: - uses: actions/checkout@v4 with: @@ -191,7 +193,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: verification-cpp-coverage-reports - path: verification/build/DebugCov/coverage/* + path: | + verification/build/DebugCov/coverage/html/ + verification/build/DebugCov/coverage/lcov/ language-verification-python: runs-on: ubuntu-latest diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 2d18712c..38025f7e 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -13,6 +13,5 @@ "streetsidesoftware.code-spell-checker", "xaver.clang-format", "vadimcn.vscode-lldb", - "matepek.vscode-catch2-test-adapter" ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index ff575c9c..8455ebed 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -138,5 +138,9 @@ "addWords": true } }, - "cmake.sourceDirectory": "${workspaceFolder}/verification" + "cmake.sourceDirectory": "${workspaceFolder}/verification", + "sonarlint.connectedMode.project": { + "connectionId": "opencyphal", + "projectKey": "OpenCyphal_nunavut" + } } diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 783d6c08..6fc5c2aa 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -285,9 +285,27 @@ After configuring you can also use Ninja directly:: ninja -t targets To obtain coverage information for the verification suite (not the Python code), -build the `cov_all` target and inspect the output under the `coverage` directory:: +build the ``cov_all`` target and inspect the output under the ``coverage`` directory. +Coverage is generated using LLVM source-based code coverage and includes HTML, text, +and lcov format reports. Only generated Nunavut code is included in coverage; vendor +code (submodules, googletest, Unity), build artifacts, system headers, and test source +files are excluded:: + + cmake --build --preset build-DebugCov-clang-native-c-11 --target cov_all + +This will: + 1. Build all test executables with coverage instrumentation + 2. Run all tests (generating .profraw files) + 3. Merge coverage data into tests.profdata + 4. Generate coverage reports in: + - ``build/DebugCov/coverage/html/index.html`` - Interactive HTML report + - ``build/DebugCov/coverage/text/summary.txt`` - Text summary + - ``build/DebugCov/coverage/lcov/coverage.lcov`` - lcov format for SonarCloud - cmake --build --preset build-clang-native-c-11-debug --target cov_all +.. note:: + + The ``DebugCov`` configuration is specifically for coverage builds. Use a preset + that includes ``DebugCov`` in its name (e.g., ``build-DebugCov-clang-native-c-11``) .. warning:: @@ -295,6 +313,7 @@ build the `cov_all` target and inspect the output under the `coverage` directory build. For example ``git clean -xdf`` or clone a new repo in a different folder. Each compiler suite leaves different byproducts that may interfere with the coverage tools in the other suite. + While we strongly encourage you to use the cmake presets, the CMakeLists.txt for the verification suite is driven by three variables you can set in your environment or pass into cmake if using cmake directly: diff --git a/verification/CMakeLists.txt b/verification/CMakeLists.txt index 9dc7081c..8a5d5251 100644 --- a/verification/CMakeLists.txt +++ b/verification/CMakeLists.txt @@ -51,6 +51,12 @@ set(CMAKE_PREFIX_PATH ${LOCAL_PROJECT_ROOT}) # set(NUNAVUT_VERIFICATIONS_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/$") +# +# Enable CTest for test discovery and execution +# +include(CTest) +enable_testing() + # +-[DEPENDENCIES]------------------------------------------------------------+ # # Load the Nunavut cmake integration. @@ -61,7 +67,6 @@ find_package(Nunavut 3.0 REQUIRED) # We generate coverage reports. Please look at them (It wasn't easy to get this to work). # find_package(verification-coverage REQUIRED) -find_package(genhtml REQUIRED) # # We require googletest to run the verification suite. @@ -382,30 +387,26 @@ endif() # +---------------------------------------------------------------------------+ # | VERIFICATION SUITE # +---------------------------------------------------------------------------+ -# We generate individual test binaires so we can record which test generated -# what coverage. We also allow test authors to generate coverage reports for -# just one test allowing for faster iteration. -set(ALL_TEST_RUNS "") -set(ALL_TEST_INFO_FILES "") -# Don't allow tests to run in parallel -set_property(GLOBAL PROPERTY JOB_POOLS coverage_test_runs=1) +# Append llvm source coverage outputs to this list as you define each test. +set(LOCAL_PROFRAW_FILES) +set(LOCAL_TEST_EXECUTABLES) -function(runTestCpp) +function(compileTestCpp) set(options "") set(oneValueArgs TEST_FILE) set(multiValueArgs LINK LANGUAGE_FLAVORS) - cmake_parse_arguments(runTestCpp "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments(compileTestCpp "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) # Skip tests not relevant to the specified language standard - list(FIND runTestCpp_LANGUAGE_FLAVORS "${LOCAL_VERIFICATION_LANGUAGE_STANDARD_CPP}" FIND_INDEX) + list(FIND compileTestCpp_LANGUAGE_FLAVORS "${LOCAL_VERIFICATION_LANGUAGE_STANDARD_CPP}" FIND_INDEX) if(${FIND_INDEX} EQUAL -1) - message(STATUS "Skipping ${runTestCpp_TEST_FILE}") + message(STATUS "Skipping ${compileTestCpp_TEST_FILE}") return() endif() - set(NATIVE_TEST "${CMAKE_CURRENT_SOURCE_DIR}/cpp/suite/${runTestCpp_TEST_FILE}") + set(NATIVE_TEST "${CMAKE_CURRENT_SOURCE_DIR}/cpp/suite/${compileTestCpp_TEST_FILE}") get_filename_component(NATIVE_TEST_NAME ${NATIVE_TEST} NAME_WE) define_native_unit_test( @@ -414,30 +415,33 @@ function(runTestCpp) SOURCE ${NATIVE_TEST} OUTDIR ${NUNAVUT_VERIFICATIONS_BINARY_DIR} DSDL_TARGETS - ${runTestCpp_LINK} + ${compileTestCpp_LINK} ) # Some tests use deprecated DSDLs target_compile_options(${NATIVE_TEST_NAME} PRIVATE "-Wno-deprecated-declarations") target_link_libraries(${NATIVE_TEST_NAME} PUBLIC o1heap) target_include_directories(${NATIVE_TEST_NAME} PUBLIC "${NUNAVUT_SUBMODULES_DIR}/CETL/include") - define_native_test_run( - NAME ${NATIVE_TEST_NAME} - OUTDIR ${NUNAVUT_VERIFICATIONS_BINARY_DIR} - OUT_CUSTOM_TARGET LOCAL_TEST_RUN_TARGET - ) - define_coverage_native_test_run( - NAME ${NATIVE_TEST_NAME} - JOB_POOL coverage_test_runs - OUTDIR ${NUNAVUT_VERIFICATIONS_BINARY_DIR} - SOURCE_FILTER_DIR ${CMAKE_CURRENT_SOURCE_DIR}/\\* - OUT_INFO_FILE LOCAL_TEST_INFO_FILE + + # Register test with CTest for automatic discovery and execution + add_test(NAME ${NATIVE_TEST_NAME} + COMMAND $ + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + + # Track profraw file locations for coverage merge + list(APPEND LOCAL_PROFRAW_FILES "${CMAKE_CURRENT_BINARY_DIR}/$/${NATIVE_TEST_NAME}.profraw") + set(LOCAL_PROFRAW_FILES ${LOCAL_PROFRAW_FILES} PARENT_SCOPE) + + list(APPEND LOCAL_TEST_EXECUTABLES ${NATIVE_TEST_NAME}) + set(LOCAL_TEST_EXECUTABLES ${LOCAL_TEST_EXECUTABLES} PARENT_SCOPE) + + # Set test properties - use generator expression in ENVIRONMENT for proper config resolution + set_tests_properties(${NATIVE_TEST_NAME} PROPERTIES + TIMEOUT 300 # 5 minute timeout for each test + LABELS "cpp;unittest" + ENVIRONMENT "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/$/${NATIVE_TEST_NAME}.profraw" ) - define_native_test_coverage(${NATIVE_TEST_NAME} ${NUNAVUT_VERIFICATIONS_BINARY_DIR}) - list(APPEND ALL_TEST_RUNS "${LOCAL_TEST_RUN_TARGET}") - list(APPEND ALL_TEST_INFO_FILES "${LOCAL_TEST_INFO_FILE}") - set(ALL_TEST_RUNS ${ALL_TEST_RUNS} PARENT_SCOPE) - set(ALL_TEST_INFO_FILES ${ALL_TEST_INFO_FILES} PARENT_SCOPE) + endfunction() @@ -506,107 +510,255 @@ endfunction() if(LOCAL_NUNAVUT_VERIFICATION_TARGET_LANG STREQUAL "cpp") if(NOT ${CMAKE_CROSSCOMPILING}) - runTestCpp(TEST_FILE test_serialization.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++17-pmr c++20 c++20-pmr) - runTestCpp(TEST_FILE test_array_c++17-pmr.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++17-pmr c++20-pmr) - runTestCpp(TEST_FILE test_array_cetl++14-17.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS cetl++14-17 ) - runTestCpp(TEST_FILE test_bitarray.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++17-pmr c++20 c++20-pmr) - runTestCpp(TEST_FILE test_compiles.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++20 ) - runTestCpp(TEST_FILE test_large_bitset.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++17-pmr c++20 c++20-pmr) - runTestCpp(TEST_FILE test_unionant.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 cetl++14-17 c++17 c++17-pmr c++20 c++20-pmr) - runTestCpp(TEST_FILE test_unionant_c++14.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 ) - runTestCpp(TEST_FILE test_unionant_pmr.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS cetl++14-17 c++17-pmr c++20-pmr) - runTestCpp(TEST_FILE test_constant.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++17-pmr c++20 c++20-pmr) + compileTestCpp(TEST_FILE test_serialization.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++17-pmr c++20 c++20-pmr) + compileTestCpp(TEST_FILE test_array_c++17-pmr.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++17-pmr c++20-pmr) + compileTestCpp(TEST_FILE test_array_cetl++14-17.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS cetl++14-17 ) + compileTestCpp(TEST_FILE test_bitarray.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++17-pmr c++20 c++20-pmr) + compileTestCpp(TEST_FILE test_compiles.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++20 ) + compileTestCpp(TEST_FILE test_large_bitset.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++17-pmr c++20 c++20-pmr) + compileTestCpp(TEST_FILE test_unionant.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 cetl++14-17 c++17 c++17-pmr c++20 c++20-pmr) + compileTestCpp(TEST_FILE test_unionant_c++14.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 ) + compileTestCpp(TEST_FILE test_unionant_pmr.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS cetl++14-17 c++17-pmr c++20-pmr) + compileTestCpp(TEST_FILE test_constant.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 c++17 c++17-pmr c++20 c++20-pmr) else() compileTestMcu(TEST_FILE test_armv7m.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} ${LOCAL_TEST_TYPES_CPP_LIBRARY} LANGUAGE_FLAVORS c++14 cetl++14-17 c++17 c++17-pmr c++20 c++20-pmr) endif() endif() -function(runTestC) +function(compileTestC) set(options "") set(oneValueArgs TEST_FILE FRAMEWORK) set(multiValueArgs LINK LANGUAGE_FLAVORS) - cmake_parse_arguments(runTestC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments(compileTestC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) # Skip tests not relevant to the specified language standard - list(FIND runTestC_LANGUAGE_FLAVORS "${LOCAL_VERIFICATION_LANGUAGE_STANDARD_C}" FIND_INDEX) + list(FIND compileTestC_LANGUAGE_FLAVORS "${LOCAL_VERIFICATION_LANGUAGE_STANDARD_C}" FIND_INDEX) if(${FIND_INDEX} EQUAL -1) - message(STATUS "Skipping ${runTestC_TEST_FILE}") + message(STATUS "Skipping ${compileTestC_TEST_FILE}") return() endif() - set(NATIVE_TEST "${CMAKE_CURRENT_SOURCE_DIR}/c/suite/${runTestC_TEST_FILE}") + set(NATIVE_TEST "${CMAKE_CURRENT_SOURCE_DIR}/c/suite/${compileTestC_TEST_FILE}") get_filename_component(NATIVE_TEST_NAME ${NATIVE_TEST} NAME_WE) define_native_unit_test( - FRAMEWORK ${runTestC_FRAMEWORK} + FRAMEWORK ${compileTestC_FRAMEWORK} NAME ${NATIVE_TEST_NAME} SOURCE ${NATIVE_TEST} OUTDIR ${NUNAVUT_VERIFICATIONS_BINARY_DIR} DSDL_TARGETS - ${runTestC_LINK} - ) - define_native_test_run( - NAME ${NATIVE_TEST_NAME} - OUTDIR ${NUNAVUT_VERIFICATIONS_BINARY_DIR} - OUT_CUSTOM_TARGET LOCAL_TEST_RUN_TARGET + ${compileTestC_LINK} ) - define_coverage_native_test_run( - NAME ${NATIVE_TEST_NAME} - JOB_POOL coverage_test_runs - OUTDIR ${NUNAVUT_VERIFICATIONS_BINARY_DIR} - SOURCE_FILTER_DIR ${CMAKE_CURRENT_SOURCE_DIR}/\\* - OUT_INFO_FILE LOCAL_TEST_INFO_FILE + + # Register test with CTest for automatic discovery and execution + add_test(NAME ${NATIVE_TEST_NAME} + COMMAND $ + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + + # Track profraw file locations for coverage merge + list(APPEND LOCAL_PROFRAW_FILES "${CMAKE_CURRENT_BINARY_DIR}/$/${NATIVE_TEST_NAME}.profraw") + set(LOCAL_PROFRAW_FILES ${LOCAL_PROFRAW_FILES} PARENT_SCOPE) + + list(APPEND LOCAL_TEST_EXECUTABLES ${NATIVE_TEST_NAME}) + set(LOCAL_TEST_EXECUTABLES ${LOCAL_TEST_EXECUTABLES} PARENT_SCOPE) + + # Set test properties - use generator expression in ENVIRONMENT for proper config resolution + set_tests_properties(${NATIVE_TEST_NAME} PROPERTIES + TIMEOUT 300 # 5 minute timeout for each test + LABELS "c;unittest" + ENVIRONMENT "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/$/${NATIVE_TEST_NAME}.profraw" ) - define_native_test_coverage(${NATIVE_TEST_NAME} ${NUNAVUT_VERIFICATIONS_BINARY_DIR}) - list(APPEND ALL_TEST_RUNS "${LOCAL_TEST_RUN_TARGET}") - list(APPEND ALL_TEST_INFO_FILES "${LOCAL_TEST_INFO_FILE}") - set(ALL_TEST_RUNS ${ALL_TEST_RUNS} PARENT_SCOPE) - set(ALL_TEST_INFO_FILES ${ALL_TEST_INFO_FILES} PARENT_SCOPE) + endfunction() if(LOCAL_NUNAVUT_VERIFICATION_TARGET_LANG STREQUAL "c") if(NOT ${CMAKE_CROSSCOMPILING}) - runTestCpp(TEST_FILE test_canard.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23) - runTestCpp(TEST_FILE test_support_assert.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23) - runTestC( TEST_FILE test_constant.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "unity") - runTestC( TEST_FILE test_override_variable_array_capacity.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "unity") - runTestC( TEST_FILE test_serialization.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "unity") - runTestC( TEST_FILE test_support.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "unity") - runTestC( TEST_FILE test_simple.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "none") + compileTestCpp(TEST_FILE test_canard.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23) + compileTestCpp(TEST_FILE test_support_assert.cpp LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23) + compileTestC( TEST_FILE test_constant.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "unity") + compileTestC( TEST_FILE test_override_variable_array_capacity.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "unity") + compileTestC( TEST_FILE test_serialization.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "unity") + compileTestC( TEST_FILE test_support.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "unity") + compileTestC( TEST_FILE test_simple.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23 FRAMEWORK "none") else() compileTestMcu(TEST_FILE test_armv7m.c LINK ${LOCAL_TEST_TYPES_C_LIBRARY} LANGUAGE_FLAVORS c11 c17 c23) endif() endif() -# +---------------------------------------------------------------------------+ -# Finally, we setup an overall report. the coverage.info should be uploaded -# to a coverage reporting service as part of the CI pipeline. -define_coverage_summary( - INFO_FILES ${ALL_TEST_INFO_FILES} - OUTDIR ${NUNAVUT_VERIFICATIONS_BINARY_DIR} - OUT_INFO_FILE LOCAL_INFO_SUMMARY +# +-[LLVM SOURCE-BASED CODE COVERAGE]----------------------------------------+ +# +# Create comprehensive coverage reporting using LLVM source-based coverage. +# This generates HTML, text, and lcov format reports with vendor code excluded. +# See: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html +# +if(CMAKE_C_COMPILER_ID MATCHES "Clang|AppleClang" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang") + message(STATUS "LLVM profdata merge and coverage reporting configured") + message(STATUS " Test executables: ${LOCAL_TEST_EXECUTABLES}") + message(STATUS " Profraw files will be: ${LOCAL_PROFRAW_FILES}") + + # Create the merged profdata file path + set(MERGED_PROFDATA_FILE "${CMAKE_CURRENT_BINARY_DIR}/$/tests.profdata") + + # Create coverage output directories + set(COVERAGE_BASE_DIR "${CMAKE_CURRENT_BINARY_DIR}/$/coverage") + set(COVERAGE_HTML_DIR "${COVERAGE_BASE_DIR}/html") + set(COVERAGE_LCOV_DIR "${COVERAGE_BASE_DIR}/lcov") + set(COVERAGE_TEXT_DIR "${COVERAGE_BASE_DIR}/text") + + # Build the list of test executable files for llvm-cov + # We need to convert target names to actual file paths at build time + set(TEST_BINARIES_FOR_COV) + foreach(TEST_TARGET ${LOCAL_TEST_EXECUTABLES}) + list(APPEND TEST_BINARIES_FOR_COV "$") + endforeach() + + # Create a file to store test binary paths (needed because generator expressions + # don't work well in custom commands with complex arguments) + file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/test_binaries_$.txt" + CONTENT "$") + + # Exclusion filter for vendor code, system headers, and test source files + # We INCLUDE: All generated code in build/.*/generated/ + # We EXCLUDE: submodules/, _deps/, system headers, test suite source files, CMakeFiles + # Note: Paths in llvm-cov reports are relative, so patterns match anywhere in the path + set(COVERAGE_IGNORE_REGEX "(submodules/|_deps/|/usr/|/Applications/|/Library/|/suite/test_|CMakeFiles/)") + + # Create a script that runs tests before merging profdata + # This ensures .profraw files exist before attempting to merge + # We use file(GENERATE) to properly handle $ generator expressions + file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/run_tests_and_merge_$.cmake" + CONTENT "execute_process( + COMMAND \"${CMAKE_CTEST_COMMAND}\" --build-config \"$\" --output-on-failure + WORKING_DIRECTORY \"${CMAKE_CURRENT_BINARY_DIR}\" + RESULT_VARIABLE test_result ) - -# This just gives us a utility to clean things up manually. You don't need to hook it up to anything. -define_coverage_zero_all(OUTDIR ${NUNAVUT_VERIFICATIONS_BINARY_DIR}) - -add_custom_target( - cov_all - ${GENHTML} - --title "${PROJECT_NAME} native test coverage" - --output-directory ${NUNAVUT_VERIFICATIONS_BINARY_DIR}/coverage/all - --demangle-cpp - --sort - --num-spaces 4 - --function-coverage - --branch-coverage - --legend - --highlight - --show-details - ${LOCAL_INFO_SUMMARY} - DEPENDS ${LOCAL_INFO_SUMMARY} - COMMENT "Build and run all tests and generate an overall html coverage report." +if(NOT test_result EQUAL 0) + message(WARNING \"Some tests failed, but continuing with coverage generation\") +endif() +file(GLOB PROFRAW_FILES \"${CMAKE_CURRENT_BINARY_DIR}/$/*.profraw\") +if(NOT PROFRAW_FILES) + message(FATAL_ERROR \"No .profraw files found. Tests may not have run successfully.\") +endif() +execute_process( + COMMAND \"${LLVM_PROFDATA}\" merge -sparse \${PROFRAW_FILES} -o \"${CMAKE_CURRENT_BINARY_DIR}/$/tests.profdata\" + RESULT_VARIABLE merge_result ) - -add_custom_target(test_all DEPENDS ${ALL_TEST_RUNS}) +if(NOT merge_result EQUAL 0) + message(FATAL_ERROR \"Failed to merge profraw files\") +endif() +" + ) + + # Target to run tests and merge profdata + add_custom_target(run_tests_and_merge + COMMAND ${CMAKE_COMMAND} + -P "${CMAKE_CURRENT_BINARY_DIR}/run_tests_and_merge_$.cmake" + COMMENT "Running tests and merging LLVM profile data files" + VERBATIM + ) + + # Make sure test executables are built before running tests + foreach(TEST_TARGET ${LOCAL_TEST_EXECUTABLES}) + add_dependencies(run_tests_and_merge ${TEST_TARGET}) + endforeach() + + # Create a script to generate coverage reports + # Using file(GENERATE) to properly handle $ generator expressions + file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/generate_coverage_reports_$.cmake" + CONTENT "file(READ \"${CMAKE_CURRENT_BINARY_DIR}/test_binaries_$.txt\" TEST_BINARIES) +string(REPLACE \"\\n\" \";\" TEST_BINARIES_LIST \"\${TEST_BINARIES}\") +set(LLVM_COV \"${LLVM_COV}\") +set(COVERAGE_HTML_DIR \"${CMAKE_CURRENT_BINARY_DIR}/$/coverage/html\") +set(COVERAGE_LCOV_DIR \"${CMAKE_CURRENT_BINARY_DIR}/$/coverage/lcov\") +set(COVERAGE_TEXT_DIR \"${CMAKE_CURRENT_BINARY_DIR}/$/coverage/text\") +set(PROFDATA_FILE \"${CMAKE_CURRENT_BINARY_DIR}/$/tests.profdata\") +set(IGNORE_REGEX \"${COVERAGE_IGNORE_REGEX}\") +file(MAKE_DIRECTORY \"\${COVERAGE_HTML_DIR}\") +file(MAKE_DIRECTORY \"\${COVERAGE_LCOV_DIR}\") +file(MAKE_DIRECTORY \"\${COVERAGE_TEXT_DIR}\") +# Build list of -object arguments for llvm-cov (one per test binary) +set(OBJECT_ARGS) +foreach(BINARY IN LISTS TEST_BINARIES_LIST) + list(APPEND OBJECT_ARGS -object \${BINARY}) +endforeach() +# Generate HTML report +execute_process( + COMMAND \${LLVM_COV} show + -format=html + -output-dir=\${COVERAGE_HTML_DIR} + -instr-profile=\${PROFDATA_FILE} + -ignore-filename-regex=\${IGNORE_REGEX} + \${OBJECT_ARGS} + RESULT_VARIABLE html_result +) +if(NOT html_result EQUAL 0) + message(WARNING \"Failed to generate HTML coverage report\") +endif() +# Generate text summary report +execute_process( + COMMAND \${LLVM_COV} report + -instr-profile=\${PROFDATA_FILE} + -ignore-filename-regex=\${IGNORE_REGEX} + \${OBJECT_ARGS} + OUTPUT_FILE \"\${COVERAGE_TEXT_DIR}/summary.txt\" + RESULT_VARIABLE report_result +) +if(NOT report_result EQUAL 0) + message(WARNING \"Failed to generate text coverage report\") +endif() +# Generate detailed text report +execute_process( + COMMAND \${LLVM_COV} show + -instr-profile=\${PROFDATA_FILE} + -ignore-filename-regex=\${IGNORE_REGEX} + \${OBJECT_ARGS} + OUTPUT_FILE \"\${COVERAGE_TEXT_DIR}/detailed.txt\" + RESULT_VARIABLE show_result +) +if(NOT show_result EQUAL 0) + message(WARNING \"Failed to generate detailed text coverage report\") +endif() +# Generate lcov format for SonarCloud +execute_process( + COMMAND \${LLVM_COV} export + -format=lcov + -instr-profile=\${PROFDATA_FILE} + -ignore-filename-regex=\${IGNORE_REGEX} + \${OBJECT_ARGS} + OUTPUT_FILE \"\${COVERAGE_LCOV_DIR}/coverage.lcov\" + RESULT_VARIABLE lcov_result +) +if(NOT lcov_result EQUAL 0) + message(WARNING \"Failed to generate lcov coverage report\") +endif() +message(STATUS \"Coverage reports generated:\") +message(STATUS \" HTML: \${COVERAGE_HTML_DIR}/index.html\") +message(STATUS \" Text: \${COVERAGE_TEXT_DIR}/summary.txt\") +message(STATUS \" Lcov: \${COVERAGE_LCOV_DIR}/coverage.lcov\") +" + ) + + # Target to generate all coverage reports + add_custom_target(generate_coverage_reports + COMMAND ${CMAKE_COMMAND} + -P "${CMAKE_CURRENT_BINARY_DIR}/generate_coverage_reports_$.cmake" + DEPENDS run_tests_and_merge + COMMENT "Generating coverage reports (HTML, text, lcov)" + VERBATIM + ) + + # Master target: cov_all - runs tests and generates all coverage reports + add_custom_target(cov_all + DEPENDS generate_coverage_reports + COMMENT "Complete coverage pipeline: run tests → merge profdata → generate reports" + ) + + message(STATUS "Coverage targets configured:") + message(STATUS " - cmake --build . --target cov_all # Complete coverage pipeline") + message(STATUS " - cmake --build . --target run_tests_and_merge # Run tests and merge profdata") + message(STATUS " - cmake --build . --target generate_coverage_reports # Generate reports from existing profdata") + message(STATUS " Output directory: ${COVERAGE_BASE_DIR}") + message(STATUS " Exclusion filter: ${COVERAGE_IGNORE_REGEX}") + +endif() diff --git a/verification/CMakePresets.json b/verification/CMakePresets.json index 1639cb4b..313e5074 100644 --- a/verification/CMakePresets.json +++ b/verification/CMakePresets.json @@ -553,7 +553,7 @@ "configurePreset": "configure-gcc-native-c-11", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -561,7 +561,7 @@ "configurePreset": "configure-gcc-native-c-11-arr-override", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -569,7 +569,7 @@ "configurePreset": "configure-gcc-native-c-17", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -577,7 +577,7 @@ "configurePreset": "configure-gcc-native-c-23", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -585,7 +585,7 @@ "configurePreset": "configure-gcc-native-cpp-14", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -593,7 +593,7 @@ "configurePreset": "configure-gcc-native-cetl-14-17", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -601,7 +601,7 @@ "configurePreset": "configure-gcc-native-cpp-17", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -609,7 +609,7 @@ "configurePreset": "configure-gcc-native-cpp-17-pmr", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -617,7 +617,7 @@ "configurePreset": "configure-gcc-native-cpp-20", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -625,7 +625,7 @@ "configurePreset": "configure-gcc-native-cpp-20-pmr", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -633,7 +633,7 @@ "configurePreset": "configure-gcc-native32-c-11", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -641,7 +641,7 @@ "configurePreset": "configure-gcc-native32-c-11-arr-override", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -649,7 +649,7 @@ "configurePreset": "configure-gcc-native32-c-17", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -657,7 +657,7 @@ "configurePreset": "configure-gcc-native32-c-23", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -665,7 +665,7 @@ "configurePreset": "configure-gcc-native32-cpp-14", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -673,7 +673,7 @@ "configurePreset": "configure-gcc-native32-cetl-14-17", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -681,7 +681,7 @@ "configurePreset": "configure-gcc-native32-cpp-17", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -689,7 +689,7 @@ "configurePreset": "configure-gcc-native32-cpp-17-pmr", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -697,7 +697,7 @@ "configurePreset": "configure-gcc-native32-cpp-20", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -705,7 +705,7 @@ "configurePreset": "configure-gcc-native32-cpp-20-pmr", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -713,7 +713,7 @@ "configurePreset": "configure-clang-native-c-11", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -721,7 +721,7 @@ "configurePreset": "configure-clang-native-c-11-arr-override", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -729,7 +729,7 @@ "configurePreset": "configure-clang-native-c-17", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -737,7 +737,7 @@ "configurePreset": "configure-clang-native-c-23", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -745,7 +745,7 @@ "configurePreset": "configure-clang-native-cpp-14", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -753,7 +753,7 @@ "configurePreset": "configure-clang-native-cetl-14-17", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -761,7 +761,7 @@ "configurePreset": "configure-clang-native-cpp-17", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -769,7 +769,7 @@ "configurePreset": "configure-clang-native-cpp-17-pmr", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -777,7 +777,7 @@ "configurePreset": "configure-clang-native-cpp-20", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -785,7 +785,7 @@ "configurePreset": "configure-clang-native-cpp-20-pmr", "configuration": "Release", "targets": [ - "test_all" + "all" ] }, { @@ -873,7 +873,7 @@ "configurePreset": "configure-gcc-native-c-11", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -881,7 +881,7 @@ "configurePreset": "configure-gcc-native-c-11-arr-override", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -889,7 +889,7 @@ "configurePreset": "configure-gcc-native-c-17", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -897,7 +897,7 @@ "configurePreset": "configure-gcc-native-c-23", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -905,7 +905,7 @@ "configurePreset": "configure-gcc-native-cpp-14", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -913,7 +913,7 @@ "configurePreset": "configure-gcc-native-cetl-14-17", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -921,7 +921,7 @@ "configurePreset": "configure-gcc-native-cpp-17", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -929,7 +929,7 @@ "configurePreset": "configure-gcc-native-cpp-17-pmr", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -937,7 +937,7 @@ "configurePreset": "configure-gcc-native-cpp-20", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -945,7 +945,7 @@ "configurePreset": "configure-gcc-native-cpp-20-pmr", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -953,7 +953,7 @@ "configurePreset": "configure-gcc-native32-c-11", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -961,7 +961,7 @@ "configurePreset": "configure-gcc-native32-c-11-arr-override", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -969,7 +969,7 @@ "configurePreset": "configure-gcc-native32-c-17", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -977,7 +977,7 @@ "configurePreset": "configure-gcc-native32-c-23", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -985,7 +985,7 @@ "configurePreset": "configure-gcc-native32-cpp-14", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -993,7 +993,7 @@ "configurePreset": "configure-gcc-native32-cetl-14-17", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1001,7 +1001,7 @@ "configurePreset": "configure-gcc-native32-cpp-17", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1009,7 +1009,7 @@ "configurePreset": "configure-gcc-native32-cpp-17-pmr", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1017,7 +1017,7 @@ "configurePreset": "configure-gcc-native32-cpp-20", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1025,7 +1025,7 @@ "configurePreset": "configure-gcc-native32-cpp-20-pmr", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1033,7 +1033,7 @@ "configurePreset": "configure-clang-native-c-11", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1041,7 +1041,7 @@ "configurePreset": "configure-clang-native-c-11-arr-override", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1049,7 +1049,7 @@ "configurePreset": "configure-clang-native-c-17", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1057,7 +1057,7 @@ "configurePreset": "configure-clang-native-c-23", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1065,7 +1065,7 @@ "configurePreset": "configure-clang-native-cpp-14", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1073,7 +1073,7 @@ "configurePreset": "configure-clang-native-cetl-14-17", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1081,7 +1081,7 @@ "configurePreset": "configure-clang-native-cpp-17", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1089,7 +1089,7 @@ "configurePreset": "configure-clang-native-cpp-17-pmr", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1097,7 +1097,7 @@ "configurePreset": "configure-clang-native-cpp-20", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1105,7 +1105,7 @@ "configurePreset": "configure-clang-native-cpp-20-pmr", "configuration": "Debug", "targets": [ - "test_all" + "all" ] }, { @@ -1193,7 +1193,7 @@ "configurePreset": "configure-gcc-native-c-11", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1201,7 +1201,7 @@ "configurePreset": "configure-gcc-native-c-11-arr-override", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1209,7 +1209,7 @@ "configurePreset": "configure-gcc-native-c-17", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1217,7 +1217,7 @@ "configurePreset": "configure-gcc-native-c-23", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1225,7 +1225,7 @@ "configurePreset": "configure-gcc-native-cpp-14", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1233,7 +1233,7 @@ "configurePreset": "configure-gcc-native-cetl-14-17", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1241,7 +1241,7 @@ "configurePreset": "configure-gcc-native-cpp-17", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1249,7 +1249,7 @@ "configurePreset": "configure-gcc-native-cpp-17-pmr", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1257,7 +1257,7 @@ "configurePreset": "configure-gcc-native-cpp-20", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1265,7 +1265,7 @@ "configurePreset": "configure-gcc-native-cpp-20-pmr", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1273,7 +1273,7 @@ "configurePreset": "configure-gcc-native32-c-11", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1281,7 +1281,7 @@ "configurePreset": "configure-gcc-native32-c-11-arr-override", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1289,7 +1289,7 @@ "configurePreset": "configure-gcc-native32-c-17", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1297,7 +1297,7 @@ "configurePreset": "configure-gcc-native32-c-23", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1305,7 +1305,7 @@ "configurePreset": "configure-gcc-native32-cpp-14", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1313,7 +1313,7 @@ "configurePreset": "configure-gcc-native32-cetl-14-17", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1321,7 +1321,7 @@ "configurePreset": "configure-gcc-native32-cpp-17", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1329,7 +1329,7 @@ "configurePreset": "configure-gcc-native32-cpp-17-pmr", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1337,7 +1337,7 @@ "configurePreset": "configure-gcc-native32-cpp-20", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1345,7 +1345,7 @@ "configurePreset": "configure-gcc-native32-cpp-20-pmr", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1353,7 +1353,7 @@ "configurePreset": "configure-clang-native-c-11", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1361,7 +1361,7 @@ "configurePreset": "configure-clang-native-c-11-arr-override", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1369,7 +1369,7 @@ "configurePreset": "configure-clang-native-c-17", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1377,7 +1377,7 @@ "configurePreset": "configure-clang-native-c-23", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1385,7 +1385,7 @@ "configurePreset": "configure-clang-native-cpp-14", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1393,7 +1393,7 @@ "configurePreset": "configure-clang-native-cetl-14-17", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1401,7 +1401,7 @@ "configurePreset": "configure-clang-native-cpp-17", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1409,7 +1409,7 @@ "configurePreset": "configure-clang-native-cpp-17-pmr", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1417,7 +1417,7 @@ "configurePreset": "configure-clang-native-cpp-20", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1425,7 +1425,7 @@ "configurePreset": "configure-clang-native-cpp-20-pmr", "configuration": "DebugAsan", "targets": [ - "test_all" + "all" ] }, { @@ -1513,7 +1513,7 @@ "configurePreset": "configure-gcc-native-c-11", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1521,7 +1521,7 @@ "configurePreset": "configure-gcc-native-c-11-arr-override", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1529,7 +1529,7 @@ "configurePreset": "configure-gcc-native-c-17", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1537,7 +1537,7 @@ "configurePreset": "configure-gcc-native-c-23", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1545,7 +1545,7 @@ "configurePreset": "configure-gcc-native-cpp-14", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1553,7 +1553,7 @@ "configurePreset": "configure-gcc-native-cetl-14-17", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1561,7 +1561,7 @@ "configurePreset": "configure-gcc-native-cpp-17", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1569,7 +1569,7 @@ "configurePreset": "configure-gcc-native-cpp-17-pmr", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1577,7 +1577,7 @@ "configurePreset": "configure-gcc-native-cpp-20", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1585,7 +1585,7 @@ "configurePreset": "configure-gcc-native-cpp-20-pmr", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1593,7 +1593,7 @@ "configurePreset": "configure-gcc-native32-c-11", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1601,7 +1601,7 @@ "configurePreset": "configure-gcc-native32-c-11-arr-override", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1609,7 +1609,7 @@ "configurePreset": "configure-gcc-native32-c-17", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1617,7 +1617,7 @@ "configurePreset": "configure-gcc-native32-c-23", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1625,7 +1625,7 @@ "configurePreset": "configure-gcc-native32-cpp-14", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1633,7 +1633,7 @@ "configurePreset": "configure-gcc-native32-cetl-14-17", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1641,7 +1641,7 @@ "configurePreset": "configure-gcc-native32-cpp-17", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1649,7 +1649,7 @@ "configurePreset": "configure-gcc-native32-cpp-17-pmr", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1657,7 +1657,7 @@ "configurePreset": "configure-gcc-native32-cpp-20", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1665,7 +1665,7 @@ "configurePreset": "configure-gcc-native32-cpp-20-pmr", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1673,7 +1673,7 @@ "configurePreset": "configure-clang-native-c-11", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1681,7 +1681,7 @@ "configurePreset": "configure-clang-native-c-11-arr-override", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1689,7 +1689,7 @@ "configurePreset": "configure-clang-native-c-17", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1697,7 +1697,7 @@ "configurePreset": "configure-clang-native-c-23", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1705,7 +1705,7 @@ "configurePreset": "configure-clang-native-cpp-14", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1713,7 +1713,7 @@ "configurePreset": "configure-clang-native-cetl-14-17", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1721,7 +1721,7 @@ "configurePreset": "configure-clang-native-cpp-17", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1729,7 +1729,7 @@ "configurePreset": "configure-clang-native-cpp-17-pmr", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1737,7 +1737,7 @@ "configurePreset": "configure-clang-native-cpp-20", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] }, { @@ -1745,15 +1745,13 @@ "configurePreset": "configure-clang-native-cpp-20-pmr", "configuration": "DebugCov", "targets": [ - "test_all" + "all" ] } ], "workflowPresets": [ { "name": "workflow-gcc-arm-none-eabi-c-11", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1771,8 +1769,6 @@ }, { "name": "workflow-gcc-arm-none-eabi-c-11-arr-override", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1790,8 +1786,6 @@ }, { "name": "workflow-gcc-arm-none-eabi-c-17", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1809,8 +1803,6 @@ }, { "name": "workflow-gcc-arm-none-eabi-c-23", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1828,8 +1820,6 @@ }, { "name": "workflow-gcc-arm-none-eabi-cpp-14", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1847,8 +1837,6 @@ }, { "name": "workflow-gcc-arm-none-eabi-cetl-14-17", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1866,8 +1854,6 @@ }, { "name": "workflow-gcc-arm-none-eabi-cpp-17", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1885,8 +1871,6 @@ }, { "name": "workflow-gcc-arm-none-eabi-cpp-17-pmr", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1904,8 +1888,6 @@ }, { "name": "workflow-gcc-arm-none-eabi-cpp-20", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1923,8 +1905,6 @@ }, { "name": "workflow-gcc-arm-none-eabi-cpp-20-pmr", - "displayName": "Configure, build, and install on arm cores (gcc).", - "description": "TCPM, auto-generated workflow: Configure, build, and install on arm cores (gcc).", "steps": [ { "type": "configure", @@ -1942,8 +1922,6 @@ }, { "name": "workflow-gcc-native-c-11", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -1965,8 +1943,6 @@ }, { "name": "workflow-gcc-native-c-11-arr-override", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -1988,8 +1964,6 @@ }, { "name": "workflow-gcc-native-c-17", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2011,8 +1985,6 @@ }, { "name": "workflow-gcc-native-c-23", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2034,8 +2006,6 @@ }, { "name": "workflow-gcc-native-cpp-14", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2057,8 +2027,6 @@ }, { "name": "workflow-gcc-native-cetl-14-17", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2080,8 +2048,6 @@ }, { "name": "workflow-gcc-native-cpp-17", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2103,8 +2069,6 @@ }, { "name": "workflow-gcc-native-cpp-17-pmr", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2126,8 +2090,6 @@ }, { "name": "workflow-gcc-native-cpp-20", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2149,8 +2111,6 @@ }, { "name": "workflow-gcc-native-cpp-20-pmr", - "displayName": "Native configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2172,8 +2132,6 @@ }, { "name": "workflow-gcc-native32-c-11", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2195,8 +2153,6 @@ }, { "name": "workflow-gcc-native32-c-11-arr-override", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2218,8 +2174,6 @@ }, { "name": "workflow-gcc-native32-c-17", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2241,8 +2195,6 @@ }, { "name": "workflow-gcc-native32-c-23", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2264,8 +2216,6 @@ }, { "name": "workflow-gcc-native32-cpp-14", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2287,8 +2237,6 @@ }, { "name": "workflow-gcc-native32-cetl-14-17", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2310,8 +2258,6 @@ }, { "name": "workflow-gcc-native32-cpp-17", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2333,8 +2279,6 @@ }, { "name": "workflow-gcc-native32-cpp-17-pmr", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2356,8 +2300,6 @@ }, { "name": "workflow-gcc-native32-cpp-20", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2379,8 +2321,6 @@ }, { "name": "workflow-gcc-native32-cpp-20-pmr", - "displayName": "32-bit linux configure, build, and test (gcc).", - "description": "TCPM, auto-generated workflow: 32-bit linux configure, build, and test (gcc).", "steps": [ { "type": "configure", @@ -2402,8 +2342,6 @@ }, { "name": "workflow-clang-native-c-11", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2429,8 +2367,6 @@ }, { "name": "workflow-clang-native-c-11-arr-override", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2456,8 +2392,6 @@ }, { "name": "workflow-clang-native-c-17", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2483,8 +2417,6 @@ }, { "name": "workflow-clang-native-c-23", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2510,8 +2442,6 @@ }, { "name": "workflow-clang-native-cpp-14", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2537,8 +2467,6 @@ }, { "name": "workflow-clang-native-cetl-14-17", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2564,8 +2492,6 @@ }, { "name": "workflow-clang-native-cpp-17", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2591,8 +2517,6 @@ }, { "name": "workflow-clang-native-cpp-17-pmr", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2618,8 +2542,6 @@ }, { "name": "workflow-clang-native-cpp-20", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2645,8 +2567,6 @@ }, { "name": "workflow-clang-native-cpp-20-pmr", - "displayName": "Native configure, build, and test (clang).", - "description": "TCPM, auto-generated workflow: Native configure, build, and test (clang).", "steps": [ { "type": "configure", @@ -2672,7 +2592,80 @@ } ], "packagePresets": [], - "testPresets": [], + "testPresets": [ + { + "name": "test-all", + "displayName": "Run All Tests", + "description": "Run all CTests", + "configurePreset": "configure-clang-native-cpp-20", + "configuration": "Debug", + "output": { + "outputOnFailure": true + } + }, + { + "name": "test-debug", + "displayName": "Run Tests (Debug)", + "description": "Run all CTests in Debug configuration", + "configurePreset": "configure-clang-native-cpp-20", + "configuration": "Debug", + "output": { + "outputOnFailure": true, + "verbosity": "default" + } + }, + { + "name": "test-debug-cov", + "displayName": "Run Tests (Debug with Coverage)", + "description": "Run all CTests in DebugCov configuration for coverage analysis", + "configurePreset": "configure-clang-native-cpp-20", + "configuration": "DebugCov", + "output": { + "outputOnFailure": true, + "verbosity": "default" + } + }, + { + "name": "test-release", + "displayName": "Run Tests (Release)", + "description": "Run all CTests in Release configuration", + "configurePreset": "configure-clang-native-cpp-20", + "configuration": "Release", + "output": { + "outputOnFailure": true + } + }, + { + "name": "test-cpp-only", + "displayName": "Run C++ Tests Only", + "description": "Run only C++ unit tests", + "configurePreset": "configure-clang-native-cpp-20", + "configuration": "Debug", + "filter": { + "include": { + "label": "cpp" + } + }, + "output": { + "outputOnFailure": true + } + }, + { + "name": "test-c-only", + "displayName": "Run C Tests Only", + "description": "Run only C unit tests", + "configurePreset": "configure-clang-native-c-17", + "configuration": "Debug", + "filter": { + "include": { + "label": "c" + } + }, + "output": { + "outputOnFailure": true + } + } + ], "vendor": { "tcpm": { "version": 1, @@ -2694,13 +2687,13 @@ "workflow-displayName-gcc-arm-none-eabi": "Configure, build, and install on arm cores (gcc).", "workflow-description-gcc-arm-none-eabi": "Configure, compile, and flash arm cores using a gcc toolchain.", "gcc-native-targets": [ - "test_all" + "all" ], "gcc-native32-targets": [ - "test_all" + "all" ], "clang-native-targets": [ - "test_all" + "all" ], "gcc-arm-none-eabi-targets": [ "test_armv7m" @@ -2896,8 +2889,6 @@ ], "shape": { "toolchain": { - "displayName": "{{{pq}(this).text({pq}('vendor tcpm static workflow-displayName-{parameter}').text())}}", - "description": "TCPM, auto-generated workflow: {{{pq}(this).text({pq}('vendor tcpm static workflow-displayName-{parameter}').text())}}", "steps": [ { "type": "configure", diff --git a/verification/CMakePresetsVendorTemplate.json b/verification/CMakePresetsVendorTemplate.json index d59d961a..b12f3c73 100644 --- a/verification/CMakePresetsVendorTemplate.json +++ b/verification/CMakePresetsVendorTemplate.json @@ -31,13 +31,13 @@ "workflow-displayName-gcc-arm-none-eabi": "Configure, build, and install on arm cores (gcc).", "workflow-description-gcc-arm-none-eabi": "Configure, compile, and flash arm cores using a gcc toolchain.", "gcc-native-targets": [ - "test_all" + "all" ], "gcc-native32-targets": [ - "test_all" + "all" ], "clang-native-targets": [ - "test_all" + "all" ], "gcc-arm-none-eabi-targets": [ "test_armv7m" @@ -182,8 +182,6 @@ ], "shape": { "toolchain": { - "displayName": "{{{pq}(this).text({pq}('vendor tcpm static workflow-displayName-{parameter}').text())}}", - "description": "TCPM, auto-generated workflow: {{{pq}(this).text({pq}('vendor tcpm static workflow-displayName-{parameter}').text())}}", "steps": [ { "type": "configure", diff --git a/verification/cmake/compiler_flag_sets/common.cmake b/verification/cmake/compiler_flag_sets/common.cmake index 80cc036b..301147fb 100644 --- a/verification/cmake/compiler_flag_sets/common.cmake +++ b/verification/cmake/compiler_flag_sets/common.cmake @@ -115,12 +115,13 @@ add_compile_options( "$<$,$>:--coverage>" "$<$,$>:-fno-elide-constructors>" "$<$,$>:-fprofile-instr-generate>" - "$<$,$>:-ftest-coverage>" - "$<$,$>:-fprofile-arcs>" "$<$,$>:-fcoverage-mapping>" ) add_link_options(${EXE_LINKER_FLAG_SET}) add_link_options("$<$:${LOCAL_SANITIZER_OPTIONS}>") -add_link_options("$<$:--coverage>") +add_link_options("$<$,$>:--coverage>") +add_link_options( + "$<$,$>:-fprofile-instr-generate>" +) set(CMAKE_C_EXTENSIONS OFF) set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/verification/cmake/modules/Findgenhtml.cmake b/verification/cmake/modules/Findgenhtml.cmake deleted file mode 100644 index 84df68b8..00000000 --- a/verification/cmake/modules/Findgenhtml.cmake +++ /dev/null @@ -1,42 +0,0 @@ -# -# Find genhtml and create coverage report target. -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# - -find_program(GENHTML genhtml) - -if(GENHTML) - # - # function: define_native_test_coverage - creates makefile targets to generate coverage - # data for an individual test. - # - # param: ARG_TEST_NAME string - The name of the test to generate coverage data for. - # param: ARG_OUTDIR path - The path where the test binaries live and where the coverage - # data will be stored. - # - function(define_native_test_coverage ARG_TEST_NAME ARG_OUTDIR) - - add_custom_target( - cov_${ARG_TEST_NAME} - COMMAND - ${GENHTML} --title "${ARG_TEST_NAME} coverage" - --output-directory ${ARG_OUTDIR}/coverage/${ARG_TEST_NAME} - --demangle-cpp - --sort - --num-spaces 4 - --function-coverage - --branch-coverage - --legend - --highlight - --show-details - ${ARG_OUTDIR}/coverage.${ARG_TEST_NAME}.filtered.info - DEPENDS run_${ARG_TEST_NAME}_with_lcov - ) - endfunction() -endif() - -include(FindPackageHandleStandardArgs) - -find_package_handle_standard_args(genhtml - GENHTML_FOUND -) diff --git a/verification/cmake/modules/Findverification-coverage.cmake b/verification/cmake/modules/Findverification-coverage.cmake index c4031e39..d6b45af5 100644 --- a/verification/cmake/modules/Findverification-coverage.cmake +++ b/verification/cmake/modules/Findverification-coverage.cmake @@ -3,302 +3,25 @@ # Copyright Amazon.com Inc. or its affiliates. # SPDX-License-Identifier: MIT # +# This module finds the LLVM coverage tools required for source-based code coverage. +# It provides llvm-cov and llvm-profdata for generating coverage reports. +# +# See: https://clang.llvm.org/docs/SourceBasedCodeCoverage.html +# -find_program(LCOV lcov) - -if(LCOV) - - # +---------------------------------------------------------------------------+ - # What follows are some gymnastics to allow coverage reports to be generated - # using either gcc or clang but resulting in the same .info format. The - # consistent output is needed to ensure we can merge and compare coverage data - # regardless of the compiler used to create the tests. - - set(VERIFICATION_COVERAGE_GOV_TOOL_ARG ) - - if (VERIFICATION_COVERAGE_USE_LLVM_COV) - # Try to find llvm coverage. If we don't find it - # we'll simply omit the tool arg and hope that lcov - # can figure it out. - # We also add some hints to help on osx. You may need to install llvm from - # homebrew since it doesn't look like it comes with xcode. - find_program(LLVM_COV - NAMES - llvm-cov - llvm-cov-6.0 - HINTS - /usr/local/opt/llvm/bin - ) - - if (LLVM_COV) - message(STATUS "Generating an llvm-cov wrapper to enable lcov report generation from clang output.") - # Thanks to http://logan.tw/posts/2015/04/28/check-code-coverage-with-clang-and-lcov/ - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/gcov_tool.sh "#!/usr/bin/env bash\nexec ${LLVM_COV} gcov \"$@\"\n") - file(COPY ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/gcov_tool.sh - DESTINATION ${CMAKE_CURRENT_BINARY_DIR} - NO_SOURCE_PERMISSIONS - FILE_PERMISSIONS OWNER_READ - OWNER_WRITE - OWNER_EXECUTE - GROUP_READ - GROUP_EXECUTE - WORLD_READ - WORLD_EXECUTE) - set(VERIFICATION_COVERAGE_GOV_TOOL_ARG "--gcov-tool" "${CMAKE_CURRENT_BINARY_DIR}/gcov_tool.sh") - else() - message(WARNING "llvm-cov was not found but we are compiling using clang. The coverage report build step may fail.") - endif() - endif() - # +---------------------------------------------------------------------------+ - - # - # function: define_coverage_native_test_run - creates a makefile target that will build and - # run individual unit tests. This also properly sets up the coverage counters. - # - # param: NAME string - The name of the test to run. - # param: JOB_POOL optional[string] - The name of a Ninja job pool to add the custom command to. - # param: BASE_DIR optional[path] (default: CMAKE_CURRENT_BINARY_DIR) - # - The root path under which object files for the test can be found. As these are - # normally found under ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles its best to use - # CMAKE_CURRENT_BINARY_DIR as this value which is the default. This can, however - # cause problems when using Ninja Multi-Config. - # param: OUTDIR path - The path where the test binaries live and under which the info files will be - # generated. - # param: SOURCE_FILTER_DIR pattern - pattern for paths to include (exclusively) in the coverage - # data. For example, test/foo/* - # param: OUT_CUSTOM_TARGET - If set, this is the name of a local variable set in the calling (parent) scope - # that contains the name of the custom target (i.e. add_custom_target) defined by - # this method that will run the test and lcov to generate an info file. - # param: OUT_INFO_FILE - If set, this is the name of a local variable set in the calling (parent) scope - # that will contain the info file generated with coverage data from the test run. - # - function(define_coverage_native_test_run) - - # +-[input]----------------------------------------------------------------+ - set(options) - set(singleValueArgs - NAME - JOB_POOL - OUTDIR - SOURCE_FILTER_DIR - BASE_DIR - OUT_CUSTOM_TARGET - OUT_INFO_FILE - ) - set(multiValueArgs) - cmake_parse_arguments(PARSE_ARGV 0 ARG "${options}" "${singleValueArgs}" "${multiValueArgs}") - - if (NOT ARG_BASE_DIR) - set(ARG_BASE_DIR ${CMAKE_CURRENT_BINARY_DIR}) - endif() - - # +-[body]-----------------------------------------------------------------+ - message(STATUS "Adding test ${ARG_NAME} for source ${ARG_SOURCE_FILTER_DIR} (${ARG_OUTDIR}/${ARG_NAME})") - - set(LOCAL_INFO_FILE "${ARG_OUTDIR}/coverage.${ARG_NAME}.filtered.info") - set(LOCAL_RUN_TARGET "run_${ARG_NAME}_with_lcov") - if (ARG_JOB_POOL) - list(APPEND LOCAL_JOB_POOL_ARG "JOB_POOL" ${ARG_JOB_POOL}) - else() - set(LOCAL_JOB_POOL_ARG) - endif() - - add_custom_command( - WORKING_DIRECTORY ${ARG_BASE_DIR} - ${LOCAL_JOB_POOL_ARG} - COMMAND # Reset coverage data - ${LCOV} - ${VERIFICATION_COVERAGE_GOV_TOOL_ARG} - --zerocounters - --directory ${ARG_BASE_DIR} - COMMAND # Generate initial "zero coverage" data. - ${LCOV} - ${VERIFICATION_COVERAGE_GOV_TOOL_ARG} - --rc branch_coverage=1 - --capture - --initial - --directory ${ARG_BASE_DIR} - --output-file ${ARG_OUTDIR}/coverage.baseline.info - COMMAND - ${ARG_OUTDIR}/${ARG_NAME} - COMMAND # Generate coverage from tests. - ${LCOV} - ${VERIFICATION_COVERAGE_GOV_TOOL_ARG} - --rc branch_coverage=1 - --capture - --directory ${ARG_BASE_DIR} - --test-name ${ARG_NAME} - --output-file ${ARG_OUTDIR}/coverage.${ARG_NAME}.test.info - COMMAND # Combine all the test runs with the baseline - ${LCOV} - ${VERIFICATION_COVERAGE_GOV_TOOL_ARG} - --rc branch_coverage=1 - --add-tracefile ${ARG_OUTDIR}/coverage.baseline.info - --add-tracefile ${ARG_OUTDIR}/coverage.${ARG_NAME}.test.info - --output-file ${ARG_OUTDIR}/coverage.${ARG_NAME}.info - COMMAND # Filter only the interesting data - ${LCOV} - ${VERIFICATION_COVERAGE_GOV_TOOL_ARG} - --rc branch_coverage=1 - --extract ${ARG_OUTDIR}/coverage.${ARG_NAME}.info - ${ARG_SOURCE_FILTER_DIR} - --output-file ${LOCAL_INFO_FILE} - OUTPUT ${LOCAL_INFO_FILE} - DEPENDS ${ARG_NAME} - ) - - add_custom_target(${LOCAL_RUN_TARGET} DEPENDS ${LOCAL_INFO_FILE}) - - # +-[OUT]---------------------------------------------------------------------+ - - if (ARG_OUT_CUSTOM_TARGET) - set(${ARG_OUT_CUSTOM_TARGET} "${LOCAL_RUN_TARGET}" PARENT_SCOPE) - endif() - - if (ARG_OUT_INFO_FILE) - set(${ARG_OUT_INFO_FILE} ${LOCAL_INFO_FILE} PARENT_SCOPE) - endif() - - endfunction() - - # - # function: define_coverage_summary - Runs lcov over info files generated by test runs defined by calls to - # define_coverage_native_test_run and creates a single, summarized coverage info file. - # - # Example Usage:: - # - # define_coverage_native_test_run( - # NAME my_test - # OUTDIR ${CMAKE_CURRENT_BINARY_DIR}/$ - # SOURCE_FILTER_DIR ${CMAKE_CURRENT_SOURCE_DIR}/\\* - # OUT_INFO_FILE LOCAL_INFO_FILE - # ) - # - # list(APPEND ALL_INFO_FILES ${LOCAL_INFO_FILE}) - # - # # add other tests and append to LOCAL_INFO_FILE list. - # - # define_coverage_summary( - # INFO_FILES ${LOCAL_INFO_FILE} - # OUTDIR ${CMAKE_CURRENT_BINARY_DIR}/$ - # OUT_INFO_FILE LOCAL_SUMMARY_INFO_FILE - # ) - # # ${LOCAL_SUMMARY_INFO_FILE} can be used as input to genhtml or uploaded to a coverage index service like - # # coveralls.io - # - # param: INFO_FILES list[path] - A list of info files to include in the summary. - # param: OUTDIR path - The path where the info files live. - # param: OUT_CUSTOM_TARGET - If set, this is the name of a local variable set in the calling (parent) scope - # that contains the name of a custom target (i.e. add_custom_target) defined by - # this method that run the info summary rule. - # param: OUT_INFO_FILE - If set, this is the name of a local variable set in the calling (parent) scope - # that will contain the info file generated with coverage data from the test run. - # - function(define_coverage_summary) - - # +-[inputs]---------------------------------------------------------------+ - set(options) - set(singleValueArgs - OUTDIR - OUT_INFO_FILE - OUT_CUSTOM_TARGET - ) - set(multiValueArgs - INFO_FILES - ) - cmake_parse_arguments(PARSE_ARGV 0 ARG "${options}" "${singleValueArgs}" "${multiValueArgs}") - - # +-[body]-----------------------------------------------------------------+ - - set(LOCAL_ALL_INFO_FILE "${ARG_OUTDIR}/coverage.all.info") - set(LOCAL_FILTERED_INFO_FILE "${ARG_OUTDIR}/coverage.info") - list(REMOVE_DUPLICATES ARG_INFO_FILES) - set(LOCAL_ADD_TRACEFILE_ARGS) - foreach(LOCAL_INFO_FILE ${ARG_INFO_FILES}) - list(APPEND LOCAL_ADD_TRACEFILE_ARGS --add-tracefile ${LOCAL_INFO_FILE}) - endforeach() - - add_custom_command( - OUTPUT ${LOCAL_ALL_INFO_FILE} - COMMAND - ${LCOV} - ${VERIFICATION_COVERAGE_GOV_TOOL_ARG} - --rc branch_coverage=1 - ${LOCAL_ADD_TRACEFILE_ARGS} - --output-file ${LOCAL_ALL_INFO_FILE} - DEPENDS ${ARG_INFO_FILES} - ) - - add_custom_command( - OUTPUT ${LOCAL_FILTERED_INFO_FILE} - COMMAND - ${LCOV} - ${VERIFICATION_COVERAGE_GOV_TOOL_ARG} - --rc branch_coverage=1 - --extract ${LOCAL_ALL_INFO_FILE} - ${LOCAL_PROJECT_ROOT}/\\* - --output-file ${LOCAL_FILTERED_INFO_FILE} - DEPENDS ${LOCAL_ALL_INFO_FILE} - ) - - add_custom_target( - cov_info - DEPENDS ${LOCAL_FILTERED_INFO_FILE} - ) - - # +-[outputs]--------------------------------------------------------------+ - if (ARG_OUT_CUSTOM_TARGET) - set(${ARG_OUT_CUSTOM_TARGET} "cov_info" PARENT_SCOPE) - endif() - - if (ARG_OUT_INFO_FILE) - set(${ARG_OUT_INFO_FILE} ${LOCAL_FILTERED_INFO_FILE} PARENT_SCOPE) - endif() - - endfunction() - - - - # - # function: define_coverage_zero_all - Defines a custom rule to zero out all counter under a given directory. - # - # param: OUTDIR path - The path where the info files live. - # param: OUT_CUSTOM_TARGET - If set, this is the name of a local variable set in the calling (parent) scope - # that contains the name of a custom target (i.e. add_custom_target) defined by - # this method that run the info summary rule. - # - function(define_coverage_zero_all) - # +-[inputs]---------------------------------------------------------------+ - set(options) - set(singleValueArgs - OUTDIR - OUT_CUSTOM_TARGET - ) - set(multiValueArgs) - cmake_parse_arguments(PARSE_ARGV 0 ARG "${options}" "${singleValueArgs}" "${multiValueArgs}") - - # +-[body]-----------------------------------------------------------------+ - - add_custom_target( - cov_zero - ${LCOV} - ${VERIFICATION_COVERAGE_GOV_TOOL_ARG} - --zerocounters - --directory ${ARG_OUTDIR} - COMMENT "Resetting coverage counters under ${ARG_OUTDIR}" - ) - - # +-[outputs]--------------------------------------------------------------+ - if (ARG_OUT_CUSTOM_TARGET) - set(${ARG_OUT_CUSTOM_TARGET} "cov_zero" PARENT_SCOPE) - endif() - - endfunction() +find_program(LLVM_COV llvm-cov) +find_program(LLVM_PROFDATA llvm-profdata) +if(LLVM_COV AND LLVM_PROFDATA) + message(STATUS "Found LLVM coverage tools:") + message(STATUS " llvm-cov: ${LLVM_COV}") + message(STATUS " llvm-profdata: ${LLVM_PROFDATA}") endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args(verification-coverage - LCOV_FOUND + REQUIRED_VARS + LLVM_COV + LLVM_PROFDATA ) diff --git a/verification/cmake/utils.cmake b/verification/cmake/utils.cmake index 9c182085..311bc9e4 100644 --- a/verification/cmake/utils.cmake +++ b/verification/cmake/utils.cmake @@ -75,45 +75,6 @@ function(define_native_unit_test) endfunction() - -# -# function: define_native_test_run - creates a makefile target that will build and -# run individual unit tests. -# -# param: NAME string - The name of the test to run. -# param: OUTDIR path - The path where the test binaries live. -# param: OUT_CUSTOM_TARGET - A variable to set in the parent scope with the name of the custom target -# defined by this function. -# -function(define_native_test_run) - # +--[ INPUTS ]-----------------------------------------------------------+ - set(options "") - set(monoValues NAME OUTDIR OUT_CUSTOM_TARGET) - set(multiValues "") - - cmake_parse_arguments( - ARG - "${options}" - "${monoValues}" - "${multiValues}" - ${ARGN} - ) - - # +--[ BODY ]-------------------------------------------------------------+ - add_custom_target( - run_${ARG_NAME} - COMMAND - ${ARG_OUTDIR}/${ARG_NAME} - DEPENDS - ${ARG_NAME} - ) - - # +--[ OUTPUTS ]----------------------------------------------------------+ - - set(${ARG_OUT_CUSTOM_TARGET} "run_${ARG_NAME}" PARENT_SCOPE) - -endfunction() - # # function: handle_nunavut_verification_language_and_standard - Parse NUNAVUT_VERIFICATION_LANG, # NUNAVUT_VERIFICATION_LANG_STANDARD, and NUNAVUT_VERIFICATION_TARGET_PLATFORM applying defaults,