Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ BASHUNIT_SHOW_EXECUTION_TIME= # Default: true
BASHUNIT_SHOW_SKIPPED= # Default: false (show skipped test details)
BASHUNIT_SHOW_INCOMPLETE= # Default: false (show incomplete test details)
BASHUNIT_FAILURES_ONLY= # Default: false (only show failures)
BASHUNIT_FAIL_ON_RISKY= # Default: false (treat no-assertion tests as failed)
BASHUNIT_NO_COLOR= # Default: false (disable colors)

#───────────────────────────────────────────────────────────────────────────────
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
### Added
- Allow `bashunit::spy` to accept an optional exit code (e.g. `bashunit::spy thing 1`) or custom implementation function (e.g. `bashunit::spy thing mock_thing`) (#600)
- Allow most assert functions to accept an optional trailing label parameter to override the failure message title (e.g. `assert_same "a" "$b" "checking user name"`) (#77)
- Add `--fail-on-risky` flag and `BASHUNIT_FAIL_ON_RISKY` env var to treat risky tests (no assertions) as failures (#115)

### Fixed
- Fix `--stop-on-failure` not stopping when a test errors with a runtime error (e.g. `command not found`, `illegal option`) (#383)
Expand Down
1 change: 1 addition & 0 deletions docs/command-line.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ bashunit test tests/ --parallel --simple
| `--debug [file]` | Enable shell debug mode |
| `--no-output` | Suppress all output |
| `--failures-only` | Only show failures |
| `--fail-on-risky` | Treat risky tests (no assertions) as failures |
| `--no-progress` | Suppress real-time progress, show only summary |
| `--show-output` | Show test output on failure (default) |
| `--no-output-on-failure` | Hide test output on failure |
Expand Down
16 changes: 16 additions & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,22 @@ BASHUNIT_NO_OUTPUT=true
```
:::

## Fail on risky

> `BASHUNIT_FAIL_ON_RISKY=true|false`

Treat risky tests (tests with zero assertions) as failures instead of warnings. `false` by default.

When enabled, a test that finishes without running any assertion is reported as failed, and the run exits with a non-zero status.

Similar as using `--fail-on-risky` option on the command line.

::: code-group
```bash [Example]
BASHUNIT_FAIL_ON_RISKY=true
```
:::

## Failures only

> `BASHUNIT_FAILURES_ONLY=true|false`
Expand Down
1 change: 1 addition & 0 deletions src/console_header.sh
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ Options:
--debug [file] Enable shell debug mode
--no-output Suppress all output
--failures-only Only show failures (suppress passed/skipped/incomplete)
--fail-on-risky Treat risky tests (no assertions) as failures
--no-progress Suppress real-time progress, show only final results
--show-output Show test output on failure (default: enabled)
--no-output-on-failure Hide test output on failure
Expand Down
6 changes: 6 additions & 0 deletions src/env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ _BASHUNIT_DEFAULT_NO_COLOR="false"
_BASHUNIT_DEFAULT_SHOW_OUTPUT_ON_FAILURE="true"
_BASHUNIT_DEFAULT_NO_PROGRESS="false"
_BASHUNIT_DEFAULT_OUTPUT_FORMAT=""
_BASHUNIT_DEFAULT_FAIL_ON_RISKY="false"

: "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_BASHUNIT_DEFAULT_PARALLEL_RUN}}"
: "${BASHUNIT_PARALLEL_JOBS:=0}"
Expand All @@ -87,6 +88,7 @@ _BASHUNIT_DEFAULT_OUTPUT_FORMAT=""
: "${BASHUNIT_SHOW_OUTPUT_ON_FAILURE:=${SHOW_OUTPUT_ON_FAILURE:=$_BASHUNIT_DEFAULT_SHOW_OUTPUT_ON_FAILURE}}"
: "${BASHUNIT_NO_PROGRESS:=${NO_PROGRESS:=$_BASHUNIT_DEFAULT_NO_PROGRESS}}"
: "${BASHUNIT_OUTPUT_FORMAT:=${OUTPUT_FORMAT:=$_BASHUNIT_DEFAULT_OUTPUT_FORMAT}}"
: "${BASHUNIT_FAIL_ON_RISKY:=${FAIL_ON_RISKY:=$_BASHUNIT_DEFAULT_FAIL_ON_RISKY}}"
# Support NO_COLOR standard (https://no-color.org)
if [ -n "${NO_COLOR:-}" ]; then
BASHUNIT_NO_COLOR="true"
Expand Down Expand Up @@ -186,6 +188,10 @@ function bashunit::env::is_tap_output_enabled() {
[ "$BASHUNIT_OUTPUT_FORMAT" = "tap" ]
}

function bashunit::env::is_fail_on_risky_enabled() {
[ "$BASHUNIT_FAIL_ON_RISKY" = "true" ]
}

function bashunit::env::active_internet_connection() {
if [ "${BASHUNIT_NO_NETWORK:-}" = "true" ]; then
return 1
Expand Down
3 changes: 3 additions & 0 deletions src/main.sh
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,9 @@ function bashunit::main::cmd_test() {
--failures-only)
export BASHUNIT_FAILURES_ONLY=true
;;
--fail-on-risky)
export BASHUNIT_FAIL_ON_RISKY=true
;;
--show-output)
export BASHUNIT_SHOW_OUTPUT_ON_FAILURE=true
;;
Expand Down
6 changes: 5 additions & 1 deletion src/parallel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,11 @@ function bashunit::parallel::aggregate_test_results() {
# Check for risky test (zero assertions, no error)
local total_for_test=$((failed + passed + skipped + incomplete + snapshot))
if [ "$total_for_test" -eq 0 ] && [ "${exit_code:-0}" -eq 0 ]; then
bashunit::state::add_tests_risky
if bashunit::env::is_fail_on_risky_enabled; then
bashunit::state::add_tests_failed
else
bashunit::state::add_tests_risky
fi
continue
fi

Expand Down
16 changes: 16 additions & 0 deletions src/runner.sh
Original file line number Diff line number Diff line change
Expand Up @@ -857,6 +857,22 @@ function bashunit::runner::run_test() {

# Check for risky test (zero assertions)
if [ "$total_assertions" -eq 0 ]; then
if bashunit::env::is_fail_on_risky_enabled; then
local risky_msg="Test has no assertions (risky)"
bashunit::state::add_tests_failed
bashunit::console_results::print_error_test "$fn_name" "$risky_msg"
bashunit::reports::add_test_failed "$test_file" "$label" "$duration" "$total_assertions" "$risky_msg"
bashunit::runner::write_failure_result_output "$test_file" "$fn_name" "$risky_msg"
bashunit::internal_log "Test failed (risky)" "$label"
if bashunit::env::is_stop_on_failure_enabled; then
if bashunit::parallel::is_enabled; then
bashunit::parallel::mark_stop_on_failure
else
exit "$EXIT_CODE_STOP_ON_FAILURE"
fi
fi
return
fi
bashunit::state::add_tests_risky
if ! bashunit::env::is_failures_only_enabled; then
bashunit::console_results::print_risky_test "${label}" "$duration"
Expand Down
48 changes: 48 additions & 0 deletions tests/acceptance/bashunit_risky_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,51 @@ function test_bashunit_risky_test_does_not_fail() {
assert_contains "risky" "$actual"
assert_not_contains "failed" "$actual"
}

function test_bashunit_fail_on_risky_flag_makes_risky_fail() {
local test_file=./tests/acceptance/fixtures/test_bashunit_risky_no_assertions.sh

local actual_raw
set +e
actual_raw="$(BASHUNIT_STRICT_MODE=false ./bashunit \
--no-parallel --fail-on-risky --skip-env-file --env "$TEST_ENV_FILE" "$test_file")"
set -e

local actual
actual="$(printf "%s" "$actual_raw" | strip_ansi)"

assert_contains "1 failed" "$actual"
assert_not_contains "1 risky" "$actual"
assert_general_error "$(BASHUNIT_STRICT_MODE=false ./bashunit \
--no-parallel --fail-on-risky --skip-env-file --env "$TEST_ENV_FILE" "$test_file")"
}

function test_bashunit_fail_on_risky_env_var_makes_risky_fail() {
local test_file=./tests/acceptance/fixtures/test_bashunit_risky_no_assertions.sh

local actual_raw
set +e
actual_raw="$(BASHUNIT_STRICT_MODE=false BASHUNIT_FAIL_ON_RISKY=true ./bashunit \
--no-parallel --skip-env-file --env "$TEST_ENV_FILE" "$test_file")"
set -e

local actual
actual="$(printf "%s" "$actual_raw" | strip_ansi)"

assert_contains "1 failed" "$actual"
}

function test_bashunit_fail_on_risky_works_in_parallel() {
local test_file=./tests/acceptance/fixtures/test_bashunit_risky_no_assertions.sh

local actual_raw
set +e
actual_raw="$(BASHUNIT_STRICT_MODE=false ./bashunit \
--parallel --fail-on-risky --skip-env-file --env "$TEST_ENV_FILE" "$test_file")"
set -e

local actual
actual="$(printf "%s" "$actual_raw" | strip_ansi)"

assert_contains "1 failed" "$actual"
}
6 changes: 5 additions & 1 deletion tests/acceptance/bashunit_upgrade_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,11 @@ function test_upgrade_when_a_new_version_found() {
fi

local output
output="$($TMP_BIN upgrade)"
output="$($TMP_BIN upgrade 2>/dev/null)"

if [[ -z "$output" ]]; then
bashunit::skip "upgrade produced no output (transient network failure)" && return
fi

assert_contains "> Upgrading bashunit to latest version" "$output"
assert_contains "> bashunit upgraded successfully to latest version $LATEST_VERSION" "$output"
Expand Down
1 change: 1 addition & 0 deletions tests/unit/env_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ function provide_boolean_flags_true() {
bashunit::data_set "BASHUNIT_SKIP_ENV_FILE" "bashunit::env::is_skip_env_file_enabled"
bashunit::data_set "BASHUNIT_LOGIN_SHELL" "bashunit::env::is_login_shell_enabled"
bashunit::data_set "BASHUNIT_FAILURES_ONLY" "bashunit::env::is_failures_only_enabled"
bashunit::data_set "BASHUNIT_FAIL_ON_RISKY" "bashunit::env::is_fail_on_risky_enabled"
bashunit::data_set "BASHUNIT_SHOW_OUTPUT_ON_FAILURE" "bashunit::env::is_show_output_on_failure_enabled"
bashunit::data_set "BASHUNIT_NO_PROGRESS" "bashunit::env::is_no_progress_enabled"
bashunit::data_set "BASHUNIT_NO_COLOR" "bashunit::env::is_no_color_enabled"
Expand Down
Loading