feat(runner): --fail-on-risky to fail tests with no assertions (#629)

Author: Chemaclass

.env.example

@@ -23,6 +23,7 @@ BASHUNIT_SHOW_EXECUTION_TIME=       # Default: true
 BASHUNIT_SHOW_SKIPPED=              # Default: false (show skipped test details)
 BASHUNIT_SHOW_INCOMPLETE=           # Default: false (show incomplete test details)
 BASHUNIT_FAILURES_ONLY=             # Default: false (only show failures)
+BASHUNIT_FAIL_ON_RISKY=             # Default: false (treat no-assertion tests as failed)
 BASHUNIT_NO_COLOR=                  # Default: false (disable colors)
 
 #───────────────────────────────────────────────────────────────────────────────

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Added
 - Allow `bashunit::spy` to accept an optional exit code (e.g. `bashunit::spy thing 1`) or custom implementation function (e.g. `bashunit::spy thing mock_thing`) (#600)
 - Allow most assert functions to accept an optional trailing label parameter to override the failure message title (e.g. `assert_same "a" "$b" "checking user name"`) (#77)
+- Add `--fail-on-risky` flag and `BASHUNIT_FAIL_ON_RISKY` env var to treat risky tests (no assertions) as failures (#115)
 
 ### Fixed
 - Fix `--stop-on-failure` not stopping when a test errors with a runtime error (e.g. `command not found`, `illegal option`) (#383)

docs/command-line.md

@@ -76,6 +76,7 @@ bashunit test tests/ --parallel --simple
 | `--debug [file]`               | Enable shell debug mode                          |
 | `--no-output`                  | Suppress all output                              |
 | `--failures-only`              | Only show failures                               |
+| `--fail-on-risky`              | Treat risky tests (no assertions) as failures    |
 | `--no-progress`                | Suppress real-time progress, show only summary   |
 | `--show-output`                | Show test output on failure (default)            |
 | `--no-output-on-failure`       | Hide test output on failure                      |

docs/configuration.md

@@ -353,6 +353,22 @@ BASHUNIT_NO_OUTPUT=true
 ```
 :::
 
+## Fail on risky
+
+> `BASHUNIT_FAIL_ON_RISKY=true|false`
+
+Treat risky tests (tests with zero assertions) as failures instead of warnings. `false` by default.
+
+When enabled, a test that finishes without running any assertion is reported as failed, and the run exits with a non-zero status.
+
+Similar as using `--fail-on-risky` option on the command line.
+
+::: code-group
+```bash [Example]
+BASHUNIT_FAIL_ON_RISKY=true
+```
+:::
+
 ## Failures only
 
 > `BASHUNIT_FAILURES_ONLY=true|false`

src/console_header.sh

@@ -122,6 +122,7 @@ Options:
   --debug [file]              Enable shell debug mode
   --no-output                 Suppress all output
   --failures-only             Only show failures (suppress passed/skipped/incomplete)
+  --fail-on-risky             Treat risky tests (no assertions) as failures
   --no-progress               Suppress real-time progress, show only final results
   --show-output               Show test output on failure (default: enabled)
   --no-output-on-failure      Hide test output on failure

src/env.sh

@@ -65,6 +65,7 @@ _BASHUNIT_DEFAULT_NO_COLOR="false"
 _BASHUNIT_DEFAULT_SHOW_OUTPUT_ON_FAILURE="true"
 _BASHUNIT_DEFAULT_NO_PROGRESS="false"
 _BASHUNIT_DEFAULT_OUTPUT_FORMAT=""
+_BASHUNIT_DEFAULT_FAIL_ON_RISKY="false"
 
 : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}"
 : "${BASHUNIT_PARALLEL_JOBS:=0}"
@@ -87,6 +88,7 @@ _BASHUNIT_DEFAULT_OUTPUT_FORMAT=""
 : "${BASHUNIT_SHOW_OUTPUT_ON_FAILURE:=${SHOW_OUTPUT_ON_FAILURE:=$_BASHUNIT_DEFAULT_SHOW_OUTPUT_ON_FAILURE}}"
 : "${BASHUNIT_NO_PROGRESS:=${NO_PROGRESS:=$_BASHUNIT_DEFAULT_NO_PROGRESS}}"
 : "${BASHUNIT_OUTPUT_FORMAT:=${OUTPUT_FORMAT:=$_BASHUNIT_DEFAULT_OUTPUT_FORMAT}}"
+: "${BASHUNIT_FAIL_ON_RISKY:=${FAIL_ON_RISKY:=$_BASHUNIT_DEFAULT_FAIL_ON_RISKY}}"
 # Support NO_COLOR standard (https://no-color.org)
 if [ -n "${NO_COLOR:-}" ]; then
   BASHUNIT_NO_COLOR="true"
@@ -186,6 +188,10 @@ function bashunit::env::is_tap_output_enabled() {
   [ "$BASHUNIT_OUTPUT_FORMAT" = "tap" ]
 }
 
+function bashunit::env::is_fail_on_risky_enabled() {
+  [ "$BASHUNIT_FAIL_ON_RISKY" = "true" ]
+}
+
 function bashunit::env::active_internet_connection() {
   if [ "${BASHUNIT_NO_NETWORK:-}" = "true" ]; then
     return 1

src/main.sh

@@ -119,6 +119,9 @@ function bashunit::main::cmd_test() {
     --failures-only)
       export BASHUNIT_FAILURES_ONLY=true
       ;;
+    --fail-on-risky)
+      export BASHUNIT_FAIL_ON_RISKY=true
+      ;;
     --show-output)
       export BASHUNIT_SHOW_OUTPUT_ON_FAILURE=true
       ;;

src/parallel.sh

@@ -90,7 +90,11 @@ function bashunit::parallel::aggregate_test_results() {
       # Check for risky test (zero assertions, no error)
       local total_for_test=$((failed + passed + skipped + incomplete + snapshot))
       if [ "$total_for_test" -eq 0 ] && [ "${exit_code:-0}" -eq 0 ]; then
-        bashunit::state::add_tests_risky
+        if bashunit::env::is_fail_on_risky_enabled; then
+          bashunit::state::add_tests_failed
+        else
+          bashunit::state::add_tests_risky
+        fi
         continue
       fi
 

src/runner.sh

@@ -857,6 +857,22 @@ function bashunit::runner::run_test() {
 
   # Check for risky test (zero assertions)
   if [ "$total_assertions" -eq 0 ]; then
+    if bashunit::env::is_fail_on_risky_enabled; then
+      local risky_msg="Test has no assertions (risky)"
+      bashunit::state::add_tests_failed
+      bashunit::console_results::print_error_test "$fn_name" "$risky_msg"
+      bashunit::reports::add_test_failed "$test_file" "$label" "$duration" "$total_assertions" "$risky_msg"
+      bashunit::runner::write_failure_result_output "$test_file" "$fn_name" "$risky_msg"
+      bashunit::internal_log "Test failed (risky)" "$label"
+      if bashunit::env::is_stop_on_failure_enabled; then
+        if bashunit::parallel::is_enabled; then
+          bashunit::parallel::mark_stop_on_failure
+        else
+          exit "$EXIT_CODE_STOP_ON_FAILURE"
+        fi
+      fi
+      return
+    fi
     bashunit::state::add_tests_risky
     if ! bashunit::env::is_failures_only_enabled; then
       bashunit::console_results::print_risky_test "${label}" "$duration"

tests/acceptance/bashunit_risky_test.sh

@@ -36,3 +36,51 @@ function test_bashunit_risky_test_does_not_fail() {
   assert_contains "risky" "$actual"
   assert_not_contains "failed" "$actual"
 }
+
+function test_bashunit_fail_on_risky_flag_makes_risky_fail() {
+  local test_file=./tests/acceptance/fixtures/test_bashunit_risky_no_assertions.sh
+
+  local actual_raw
+  set +e
+  actual_raw="$(BASHUNIT_STRICT_MODE=false ./bashunit \
+    --no-parallel --fail-on-risky --skip-env-file --env "$TEST_ENV_FILE" "$test_file")"
+  set -e
+
+  local actual
+  actual="$(printf "%s" "$actual_raw" | strip_ansi)"
+
+  assert_contains "1 failed" "$actual"
+  assert_not_contains "1 risky" "$actual"
+  assert_general_error "$(BASHUNIT_STRICT_MODE=false ./bashunit \
+    --no-parallel --fail-on-risky --skip-env-file --env "$TEST_ENV_FILE" "$test_file")"
+}
+
+function test_bashunit_fail_on_risky_env_var_makes_risky_fail() {
+  local test_file=./tests/acceptance/fixtures/test_bashunit_risky_no_assertions.sh
+
+  local actual_raw
+  set +e
+  actual_raw="$(BASHUNIT_STRICT_MODE=false BASHUNIT_FAIL_ON_RISKY=true ./bashunit \
+    --no-parallel --skip-env-file --env "$TEST_ENV_FILE" "$test_file")"
+  set -e
+
+  local actual
+  actual="$(printf "%s" "$actual_raw" | strip_ansi)"
+
+  assert_contains "1 failed" "$actual"
+}
+
+function test_bashunit_fail_on_risky_works_in_parallel() {
+  local test_file=./tests/acceptance/fixtures/test_bashunit_risky_no_assertions.sh
+
+  local actual_raw
+  set +e
+  actual_raw="$(BASHUNIT_STRICT_MODE=false ./bashunit \
+    --parallel --fail-on-risky --skip-env-file --env "$TEST_ENV_FILE" "$test_file")"
+  set -e
+
+  local actual
+  actual="$(printf "%s" "$actual_raw" | strip_ansi)"
+
+  assert_contains "1 failed" "$actual"
+}