diff --git a/CHANGELOG.md b/CHANGELOG.md index 8490f00e..d4feae09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ - `bashunit init [dir]` - initialize project (replaces `--init`) - `bashunit learn` - interactive tutorial (replaces `--learn`) - `bashunit upgrade` - upgrade to latest (replaces `--upgrade`) +- **BREAKING:** Tests now stop at first assertion failure within a test function + - Matches PHPUnit and Jest default behavior + - Subsequent assertions in the same test are skipped after a failure + - Other test functions continue to run normally ### Fixed - Stop executing remaining commands in `set_up`/`tear_down` after first failure diff --git a/docs/configuration.md b/docs/configuration.md index ddafc6f5..fab0fa16 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -85,6 +85,12 @@ Force to stop the runner right after encountering one failing test. `false` by d Similar as using `-S|--stop-on-failure` option on the [command line](/command-line#stop-on-failure). +::: tip Assertion behavior +When an assertion fails within a test, subsequent assertions in the same test are automatically skipped. This matches popular testing libraries default behavior and prevents misleading failures caused by earlier assertion failures. + +The `--stop-on-failure` flag is separate – it stops the entire test runner after a failing **test**, while assertion-level stopping happens automatically within each test. +::: + ## Show header > `BASHUNIT_SHOW_HEADER=true|false` diff --git a/docs/custom-asserts.md b/docs/custom-asserts.md index c0dcf94e..a85e4c83 100644 --- a/docs/custom-asserts.md +++ b/docs/custom-asserts.md @@ -6,6 +6,10 @@ Check the internal functional tests: `tests/functional/custom_asserts_test.sh` ([link](https://github.com/TypedDevs/bashunit/blob/main/tests/functional/custom_asserts_test.sh)) ::: +::: info Assertion behavior +When using the bashunit facade, assertions automatically respect the guard behavior: if a previous assertion in the same test already failed, subsequent assertions are skipped. This matches popular testing libraries default behavior. +::: + ## assertion_failed > `bashunit::assertion_failed ` diff --git a/src/assert.sh b/src/assert.sh index 737969a7..c959faef 100755 --- a/src/assert.sh +++ b/src/assert.sh @@ -1,15 +1,34 @@ #!/usr/bin/env bash +# Guard function: skip assertions if a previous assertion in this test already failed +# This matches PHPUnit/Jest behavior where tests stop at first failure +function assert::guard() { + if state::is_assertion_failed_in_test; then + return 1 + fi + return 0 +} + +# Helper to mark assertion as failed and set the guard flag +function assert::mark_failed() { + state::add_assertions_failed + state::mark_assertion_failed_in_test +} + function fail() { + assert::guard || return 0 + local message="${1:-${FUNCNAME[1]}}" local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failure_message "${label}" "$message" } function assert_true() { + assert::guard || return 0 + local actual="$1" # Check for expected literal values first @@ -30,6 +49,8 @@ function assert_true() { } function assert_false() { + assert::guard || return 0 + local actual="$1" # Check for expected literal values first @@ -68,18 +89,20 @@ function handle_bool_assertion_failure() { local label label="$(helper::normalize_test_function_name "${FUNCNAME[2]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "$label" "$expected" "but got " "$got" } function assert_same() { + assert::guard || return 0 + local expected="$1" local actual="$2" if [[ "$expected" != "$actual" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${expected}" "but got " "${actual}" return fi @@ -88,6 +111,8 @@ function assert_same() { } function assert_equals() { + assert::guard || return 0 + local expected="$1" local actual="$2" @@ -104,7 +129,7 @@ function assert_equals() { if [[ "$expected_cleaned" != "$actual_cleaned" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${expected_cleaned}" "but got " "${actual_cleaned}" return fi @@ -113,6 +138,8 @@ function assert_equals() { } function assert_not_equals() { + assert::guard || return 0 + local expected="$1" local actual="$2" @@ -129,7 +156,7 @@ function assert_not_equals() { if [[ "$expected_cleaned" == "$actual_cleaned" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${expected_cleaned}" "but got " "${actual_cleaned}" return fi @@ -138,12 +165,14 @@ function assert_not_equals() { } function assert_empty() { + assert::guard || return 0 + local expected="$1" if [[ "$expected" != "" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "to be empty" "but got " "${expected}" return fi @@ -152,12 +181,14 @@ function assert_empty() { } function assert_not_empty() { + assert::guard || return 0 + local expected="$1" if [[ "$expected" == "" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "to not be empty" "but got " "${expected}" return fi @@ -166,13 +197,15 @@ function assert_not_empty() { } function assert_not_same() { + assert::guard || return 0 + local expected="$1" local actual="$2" if [[ "$expected" == "$actual" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${expected}" "but got " "${actual}" return fi @@ -181,6 +214,8 @@ function assert_not_same() { } function assert_contains() { + assert::guard || return 0 + local expected="$1" local actual_arr=("${@:2}") local actual @@ -189,7 +224,7 @@ function assert_contains() { if ! [[ $actual == *"$expected"* ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to contain" "${expected}" return fi @@ -198,6 +233,8 @@ function assert_contains() { } function assert_contains_ignore_case() { + assert::guard || return 0 + local expected="$1" local actual="$2" @@ -206,7 +243,7 @@ function assert_contains_ignore_case() { if ! [[ $actual =~ $expected ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to contain" "${expected}" shopt -u nocasematch return @@ -217,6 +254,8 @@ function assert_contains_ignore_case() { } function assert_not_contains() { + assert::guard || return 0 + local expected="$1" local actual_arr=("${@:2}") local actual @@ -225,7 +264,7 @@ function assert_not_contains() { if [[ $actual == *"$expected"* ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to not contain" "${expected}" return fi @@ -234,6 +273,8 @@ function assert_not_contains() { } function assert_matches() { + assert::guard || return 0 + local expected="$1" local actual_arr=("${@:2}") local actual @@ -242,7 +283,7 @@ function assert_matches() { if ! [[ $actual =~ $expected ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to match" "${expected}" return fi @@ -251,6 +292,8 @@ function assert_matches() { } function assert_not_matches() { + assert::guard || return 0 + local expected="$1" local actual_arr=("${@:2}") local actual @@ -259,7 +302,7 @@ function assert_not_matches() { if [[ $actual =~ $expected ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to not match" "${expected}" return fi @@ -268,6 +311,8 @@ function assert_not_matches() { } function assert_exec() { + assert::guard || return 0 + local cmd="$1" shift @@ -340,7 +385,7 @@ function assert_exec() { if [[ $failed -eq 1 ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "$label" "$expected_desc" "but got " "$actual_desc" return fi @@ -349,13 +394,15 @@ function assert_exec() { } function assert_exit_code() { - local actual_exit_code=${3-"$?"} + local actual_exit_code=${3-"$?"} # Capture $? before guard check + assert::guard || return 0 + local expected_exit_code="$1" if [[ "$actual_exit_code" -ne "$expected_exit_code" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual_exit_code}" "to be" "${expected_exit_code}" return fi @@ -364,13 +411,15 @@ function assert_exit_code() { } function assert_successful_code() { - local actual_exit_code=${3-"$?"} + local actual_exit_code=${3-"$?"} # Capture $? before guard check + assert::guard || return 0 + local expected_exit_code=0 if [[ "$actual_exit_code" -ne "$expected_exit_code" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual_exit_code}" "to be exactly" "${expected_exit_code}" return fi @@ -379,12 +428,13 @@ function assert_successful_code() { } function assert_unsuccessful_code() { - local actual_exit_code=${3-"$?"} + local actual_exit_code=${3-"$?"} # Capture $? before guard check + assert::guard || return 0 if [[ "$actual_exit_code" -eq 0 ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual_exit_code}" "to be non-zero" "but was 0" return fi @@ -393,13 +443,15 @@ function assert_unsuccessful_code() { } function assert_general_error() { - local actual_exit_code=${3-"$?"} + local actual_exit_code=${3-"$?"} # Capture $? before guard check + assert::guard || return 0 + local expected_exit_code=1 if [[ "$actual_exit_code" -ne "$expected_exit_code" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual_exit_code}" "to be exactly" "${expected_exit_code}" return fi @@ -408,13 +460,15 @@ function assert_general_error() { } function assert_command_not_found() { - local actual_exit_code=${3-"$?"} + local actual_exit_code=${3-"$?"} # Capture $? before guard check + assert::guard || return 0 + local expected_exit_code=127 if [[ $actual_exit_code -ne "$expected_exit_code" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual_exit_code}" "to be exactly" "${expected_exit_code}" return fi @@ -423,6 +477,8 @@ function assert_command_not_found() { } function assert_string_starts_with() { + assert::guard || return 0 + local expected="$1" local actual_arr=("${@:2}") local actual @@ -431,7 +487,7 @@ function assert_string_starts_with() { if [[ $actual != "$expected"* ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to start with" "${expected}" return fi @@ -440,13 +496,15 @@ function assert_string_starts_with() { } function assert_string_not_starts_with() { + assert::guard || return 0 + local expected="$1" local actual="$2" if [[ $actual == "$expected"* ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to not start with" "${expected}" return fi @@ -455,6 +513,8 @@ function assert_string_not_starts_with() { } function assert_string_ends_with() { + assert::guard || return 0 + local expected="$1" local actual_arr=("${@:2}") local actual @@ -463,7 +523,7 @@ function assert_string_ends_with() { if [[ $actual != *"$expected" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to end with" "${expected}" return fi @@ -472,6 +532,8 @@ function assert_string_ends_with() { } function assert_string_not_ends_with() { + assert::guard || return 0 + local expected="$1" local actual_arr=("${@:2}") local actual @@ -480,7 +542,7 @@ function assert_string_not_ends_with() { if [[ $actual == *"$expected" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to not end with" "${expected}" return fi @@ -489,13 +551,15 @@ function assert_string_not_ends_with() { } function assert_less_than() { + assert::guard || return 0 + local expected="$1" local actual="$2" if ! [[ "$actual" -lt "$expected" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to be less than" "${expected}" return fi @@ -504,13 +568,15 @@ function assert_less_than() { } function assert_less_or_equal_than() { + assert::guard || return 0 + local expected="$1" local actual="$2" if ! [[ "$actual" -le "$expected" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to be less or equal than" "${expected}" return fi @@ -519,13 +585,15 @@ function assert_less_or_equal_than() { } function assert_greater_than() { + assert::guard || return 0 + local expected="$1" local actual="$2" if ! [[ "$actual" -gt "$expected" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to be greater than" "${expected}" return fi @@ -534,13 +602,15 @@ function assert_greater_than() { } function assert_greater_or_equal_than() { + assert::guard || return 0 + local expected="$1" local actual="$2" if ! [[ "$actual" -ge "$expected" ]]; then local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${actual}" "to be greater or equal than" "${expected}" return fi @@ -549,6 +619,8 @@ function assert_greater_or_equal_than() { } function assert_line_count() { + assert::guard || return 0 + local expected="$1" local input_arr=("${@:2}") local input_str @@ -568,7 +640,7 @@ function assert_line_count() { local label label="$(helper::normalize_test_function_name "${FUNCNAME[1]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${input_str}"\ "to contain number of lines equal to" "${expected}"\ "but found" "${actual}" diff --git a/src/bashunit.sh b/src/bashunit.sh index b61fb4cf..57a45b39 100644 --- a/src/bashunit.sh +++ b/src/bashunit.sh @@ -5,17 +5,21 @@ # e.g. adding custom assertions function bashunit::assertion_failed() { + assert::guard || return 0 + local expected=$1 local actual=$2 local failure_condition_message=${3:-"but got "} local label label="$(helper::normalize_test_function_name "${FUNCNAME[2]}")" - state::add_assertions_failed + assert::mark_failed console_results::print_failed_test "${label}" "${expected}" \ "$failure_condition_message" "${actual}" } function bashunit::assertion_passed() { + assert::guard || return 0 + state::add_assertions_passed } diff --git a/src/state.sh b/src/state.sh index 74356839..7550872d 100644 --- a/src/state.sh +++ b/src/state.sh @@ -19,6 +19,7 @@ _TEST_EXIT_CODE=0 _TEST_HOOK_FAILURE="" _TEST_HOOK_MESSAGE="" _CURRENT_TEST_INTERPOLATED_NAME="" +_ASSERTION_FAILED_IN_TEST=false function state::get_tests_passed() { echo "$_TESTS_PASSED" @@ -184,6 +185,14 @@ function state::reset_test_hook_message() { _TEST_HOOK_MESSAGE="" } +function state::is_assertion_failed_in_test() { + [[ "$_ASSERTION_FAILED_IN_TEST" == "true" ]] +} + +function state::mark_assertion_failed_in_test() { + _ASSERTION_FAILED_IN_TEST=true +} + function state::set_duplicated_functions_merged() { state::set_duplicated_test_functions_found state::set_file_with_duplicated_function_names "$1" @@ -200,6 +209,7 @@ function state::initialize_assertions_count() { _TEST_TITLE="" _TEST_HOOK_FAILURE="" _TEST_HOOK_MESSAGE="" + _ASSERTION_FAILED_IN_TEST=false } function state::export_subshell_context() { diff --git a/tests/acceptance/snapshots/bashunit_fail_test_sh.test_bashunit_with_multiple_failing_tests.snapshot b/tests/acceptance/snapshots/bashunit_fail_test_sh.test_bashunit_with_multiple_failing_tests.snapshot index 2a30acb2..b6eba770 100644 --- a/tests/acceptance/snapshots/bashunit_fail_test_sh.test_bashunit_with_multiple_failing_tests.snapshot +++ b/tests/acceptance/snapshots/bashunit_fail_test_sh.test_bashunit_with_multiple_failing_tests.snapshot @@ -3,9 +3,6 @@ ✗ Failed: Assert failing Expected '1' but got  '2' -✗ Failed: Assert failing - Expected '3' - but got  '4' ✒ Incomplete: Assert todo and skip foo ↷ Skipped: Assert todo and skip bar ↷ Skipped: Assert skip and todo baz @@ -17,11 +14,8 @@ |✗ Failed: Assert failing | Expected '1' | but got  '2' -|✗ Failed: Assert failing -| Expected '3' -| but got  '4' Tests:  1 passed, 0 skipped, 2 incomplete, 1 failed, 4 total -Assertions: 1 passed, 2 skipped, 2 incomplete, 2 failed, 7 total +Assertions: 1 passed, 2 skipped, 2 incomplete, 1 failed, 6 total  Some tests failed  diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot index 4d812a09..7f053a1d 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot @@ -13,6 +13,6 @@ | but got  '2' Tests:  1 passed, 1 failed, 2 total -Assertions: 3 passed, 1 failed, 4 total +Assertions: 2 passed, 1 failed, 3 total  Some tests failed  diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env_simple_output.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env_simple_output.snapshot index a0f0ee49..7bb10683 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env_simple_output.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env_simple_output.snapshot @@ -12,6 +12,6 @@ Tests:  1 passed, 1 failed, 2 total -Assertions: 3 passed, 1 failed, 4 total +Assertions: 2 passed, 1 failed, 3 total  Some tests failed  diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot index 4d812a09..7f053a1d 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot @@ -13,6 +13,6 @@ | but got  '2' Tests:  1 passed, 1 failed, 2 total -Assertions: 3 passed, 1 failed, 4 total +Assertions: 2 passed, 1 failed, 3 total  Some tests failed