diff --git a/adrs/adr-005-custom-categories.md b/adrs/adr-005-custom-categories.md new file mode 100644 index 00000000..34087c36 --- /dev/null +++ b/adrs/adr-005-custom-categories.md @@ -0,0 +1,32 @@ +# Title: Support custom test categories + +* Status: accepted +* Authors: @Chemaclass +* Date: 2025-04-24 + +## Context and Problem Statement + +Currently, bashunit only groups tests by file or by matching a filter in the test name. Users would like to run subsets such as "slow" tests without relying on file structure or naming conventions. + +## Considered Options + +* Parse category annotations in comments and filter via a command line option. +* Require categories in function names (e.g. `test_slow_example`). +* Keep relying on the folder structure only. + +## Decision Outcome + +Using comment annotations is the most flexible approach while keeping backwards compatibility. Other testing frameworks (JUnit `@Tag`, pytest markers, NUnit `[Category]`, RSpec metadata) follow similar patterns where categories are declared near the test definition and selected by a flag. Parsing a simple `# @category` comment allows bashunit to mimic this behavior. + +### Positive Consequences + +* Enables running or excluding subsets like slow or integration tests. +* Does not impose naming conventions on test functions. + +### Negative Consequences + +* Slightly more complex parsing logic for test discovery. + +## Links + +* Refers to [Issue #357](https://github.com/TypedDevs/bashunit/issues/357) diff --git a/bashunit b/bashunit index 1c95a2ac..c88fc68f 100755 --- a/bashunit +++ b/bashunit @@ -32,6 +32,7 @@ source "$BASHUNIT_ROOT_DIR/src/main.sh" _ASSERT_FN="" _FILTER="" +_CATEGORY="" _ARGS=() check_os::init @@ -48,6 +49,11 @@ while [[ $# -gt 0 ]]; do _FILTER="$2" shift ;; + -c|--category) + _CATEGORY="$2" + export BASHUNIT_CATEGORY="$2" + shift + ;; -s|--simple) export BASHUNIT_SIMPLE_OUTPUT=true ;; @@ -115,5 +121,5 @@ set +eu if [[ -n "$_ASSERT_FN" ]]; then main::exec_assert "$_ASSERT_FN" "${_ARGS[@]}" else - main::exec_tests "$_FILTER" "${_ARGS[@]}" + main::exec_tests "$_FILTER" "$_CATEGORY" "${_ARGS[@]}" fi diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index cc3252c7..48213b6f 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -75,6 +75,9 @@ export default defineConfig({ }, { text: 'Skipping/incomplete', link: '/skipping-incomplete' + }, { + text: 'Test categories', + link: '/test-categories' }, { text: 'Standalone', link: '/standalone' diff --git a/docs/command-line.md b/docs/command-line.md index 20573d36..e49d185e 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -90,6 +90,19 @@ Filters the tests to be run based on the `test name`. ``` ::: +## Category + +> `bashunit -c|--category ` + +Filters the tests to run based on `@category` comments. + +::: code-group +```bash [Example] +# run only tests tagged as slow +./bashunit ./tests --category slow +``` +::: + ## JUnit Logging > `bashunit -l|--log-junit ` diff --git a/docs/test-categories.md b/docs/test-categories.md new file mode 100644 index 00000000..4d8dfa36 --- /dev/null +++ b/docs/test-categories.md @@ -0,0 +1,31 @@ +# Test categories + +Testing frameworks often allow running subsets of the suite by grouping tests into categories. For example: + +- **JUnit**: uses `@Tag("slow")` annotations so the runner can include or exclude tests by tag. +- **pytest**: provides markers such as `@pytest.mark.slow` that can be selected with `-m slow`. +- **NUnit**: supports categories via the `[Category("slow")]` attribute. +- **RSpec**: allows metadata like `:slow` for filtering. + +These approaches share similar ideas: + +1. A test declares one or more categories. +2. The runner filters tests according to a command line option or configuration. + +## Proposal for bashunit + +1. Allow test functions to declare categories in a comment immediately preceding the function. +2. Syntax example: + +```bash +# @category slow integration +function test_process_big_data() { + ... +} +``` + +3. Introduce a `--category ` option (and `BASHUNIT_CATEGORY` env variable) that filters test functions by the given category. +4. Internally parse the comments when discovering tests and keep a mapping of function -> categories. +5. Running without the option executes all tests; running with `--category slow` executes only those marked as `slow`. + +This approach maintains backwards compatibility and mirrors the tagging mechanisms of other testing libraries while remaining simple to parse with grep/awk. diff --git a/src/console_header.sh b/src/console_header.sh index d6e6622b..4a5835e2 100644 --- a/src/console_header.sh +++ b/src/console_header.sh @@ -2,17 +2,21 @@ function console_header::print_version_with_env() { local filter=${1:-} - local files=("${@:2}") + local category=${2:-} + shift 2 + local files=("$@") if ! env::is_show_header_enabled; then return fi - console_header::print_version "$filter" "${files[@]}" + console_header::print_version "$filter" "$category" "${files[@]}" } function console_header::print_version() { local filter=${1:-} + local category=${2:-} + if [[ -n "$filter" ]]; then shift fi @@ -44,9 +48,16 @@ EOF if [ "$total_tests" -eq 0 ]; then printf "${_COLOR_BOLD}${_COLOR_PASSED}bashunit${_COLOR_DEFAULT} - %s\n" "$BASHUNIT_VERSION" else - printf "${_COLOR_BOLD}${_COLOR_PASSED}bashunit${_COLOR_DEFAULT} - %s | Tests: ~%s\n"\ - "$BASHUNIT_VERSION"\ - "$total_tests" + if [ -n "$category" ]; then + printf "${_COLOR_BOLD}${_COLOR_PASSED}bashunit${_COLOR_DEFAULT} - %s | Tests: ~%s | Category: %s\n"\ + "$BASHUNIT_VERSION"\ + "$total_tests"\ + "$category" + else + printf "${_COLOR_BOLD}${_COLOR_PASSED}bashunit${_COLOR_DEFAULT} - %s | Tests: ~%s\n"\ + "$BASHUNIT_VERSION"\ + "$total_tests" + fi fi } @@ -69,6 +80,9 @@ Options: -f, --filter Filters the tests to run based on the test name. + -c, --category + Filters the tests to run based on @category tags. + -l, --log-junit Create a report JUnit XML file that contains information about the test results. diff --git a/src/env.sh b/src/env.sh index 50cc5165..4d74e427 100644 --- a/src/env.sh +++ b/src/env.sh @@ -12,12 +12,14 @@ _DEFAULT_BOOTSTRAP="tests/bootstrap.sh" _DEFAULT_DEV_LOG="" _DEFAULT_LOG_JUNIT="" _DEFAULT_REPORT_HTML="" +_DEFAULT_CATEGORY="" : "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}" : "${BASHUNIT_DEV_LOG:=${DEV_LOG:=$_DEFAULT_DEV_LOG}}" : "${BASHUNIT_BOOTSTRAP:=${BOOTSTRAP:=$_DEFAULT_BOOTSTRAP}}" : "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}" : "${BASHUNIT_REPORT_HTML:=${REPORT_HTML:=$_DEFAULT_REPORT_HTML}}" +: "${BASHUNIT_CATEGORY:=${CATEGORY:=$_DEFAULT_CATEGORY}}" # Booleans _DEFAULT_PARALLEL_RUN="false" @@ -97,6 +99,7 @@ function env::print_verbose() { "BASHUNIT_BOOTSTRAP" "BASHUNIT_LOG_JUNIT" "BASHUNIT_REPORT_HTML" + "BASHUNIT_CATEGORY" "BASHUNIT_PARALLEL_RUN" "BASHUNIT_SHOW_HEADER" "BASHUNIT_HEADER_ASCII_ART" diff --git a/src/helpers.sh b/src/helpers.sh index 72f6e543..fedfa70a 100755 --- a/src/helpers.sh +++ b/src/helpers.sh @@ -169,6 +169,37 @@ function helper::get_provider_data() { fi } +# +# @param $1 string Function name +# @param $2 string Script path +# +# @return string Categories separated by spaces +# +function helper::get_function_categories() { + local function_name="$1" + local script="$2" + + if [[ ! -f "$script" ]]; then + return + fi + + local line_number + line_number=$( + grep -n -m1 -E "^[[:space:]]*(function[[:space:]]+)?${function_name}[[:space:]]*\(" \ + "$script" | cut -d: -f1 + ) + if [[ -z $line_number ]]; then + return + fi + + local prev_line=$((line_number - 1)) + if [[ $prev_line -le 0 ]]; then + return + fi + + sed -n "${prev_line}p" "$script" | grep -E "# *@category" | sed -E 's/^.*# *@category *//' +} + function helper::trim() { local input_string="$1" local trimmed_string diff --git a/src/main.sh b/src/main.sh index 1203c68c..de440530 100644 --- a/src/main.sh +++ b/src/main.sh @@ -2,7 +2,8 @@ function main::exec_tests() { local filter=$1 - local files=("${@:2}") + local category=$2 + local files=("${@:3}") local test_files=() while IFS= read -r line; do @@ -30,7 +31,7 @@ function main::exec_tests() { parallel::reset fi - console_header::print_version_with_env "$filter" "${test_files[@]}" + console_header::print_version_with_env "$filter" "$category" "${test_files[@]}" if env::is_verbose_enabled; then if env::is_simple_output_enabled; then @@ -38,6 +39,7 @@ function main::exec_tests() { fi printf '%*s\n' "$TERMINAL_WIDTH" '' | tr ' ' '#' printf "%s\n" "Filter: ${filter:-None}" + printf "%s\n" "Category: ${category:-None}" printf "%s\n" "Total files: ${#test_files[@]}" printf "%s\n" "Test files:" printf -- "- %s\n" "${test_files[@]}" @@ -46,7 +48,7 @@ function main::exec_tests() { printf '%*s\n' "$TERMINAL_WIDTH" '' | tr ' ' '#' fi - runner::load_test_files "$filter" "${test_files[@]}" + runner::load_test_files "$filter" "$category" "${test_files[@]}" if parallel::is_enabled; then wait diff --git a/src/runner.sh b/src/runner.sh index a39e06ed..160256f8 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -3,7 +3,8 @@ function runner::load_test_files() { local filter=$1 - shift + local category=$2 + shift 2 local files=("${@}") for test_file in "${files[@]}"; do @@ -14,9 +15,9 @@ function runner::load_test_files() { source "$test_file" runner::run_set_up_before_script if parallel::is_enabled; then - runner::call_test_functions "$test_file" "$filter" 2>/dev/null & + runner::call_test_functions "$test_file" "$filter" "$category" 2>/dev/null & else - runner::call_test_functions "$test_file" "$filter" + runner::call_test_functions "$test_file" "$filter" "$category" fi runner::run_tear_down_after_script runner::clean_set_up_and_tear_down_after_script @@ -63,6 +64,7 @@ function runner::functions_for_script() { function runner::call_test_functions() { local script="$1" local filter="$2" + local category="$3" local prefix="test" # Use declare -F to list all function names local all_fn_names=$(declare -F | awk '{print $3}') @@ -82,6 +84,21 @@ function runner::call_test_functions() { break fi + if [[ -n "$category" ]]; then + local fn_categories + fn_categories="$(helper::get_function_categories "$fn_name" "$script")" + local match=false + for cat in $category; do + if [[ " $fn_categories " == *" $cat "* ]]; then + match=true + break + fi + done + if [[ $match == false ]]; then + continue + fi + fi + local provider_data=() while IFS=" " read -r line; do provider_data+=("$line") diff --git a/tests/acceptance/bashunit_category_test.sh b/tests/acceptance/bashunit_category_test.sh new file mode 100644 index 00000000..a37c079b --- /dev/null +++ b/tests/acceptance/bashunit_category_test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" +} + +function test_bashunit_run_with_category_option() { + local test_file=./tests/acceptance/fixtures/test_bashunit_categories.sh + assert_match_snapshot "$(./bashunit --no-parallel --env "$TEST_ENV_FILE" --category slow "$test_file")" +} diff --git a/tests/acceptance/fixtures/test_bashunit_categories.sh b/tests/acceptance/fixtures/test_bashunit_categories.sh new file mode 100644 index 00000000..41107eb7 --- /dev/null +++ b/tests/acceptance/fixtures/test_bashunit_categories.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# @category slow +function test_slow_example() { + assert_true true +} + +# @category fast +function test_fast_example() { + assert_true true +} diff --git a/tests/acceptance/snapshots/bashunit_category_test_sh.test_bashunit_run_with_category_option.snapshot b/tests/acceptance/snapshots/bashunit_category_test_sh.test_bashunit_run_with_category_option.snapshot new file mode 100644 index 00000000..1755d702 --- /dev/null +++ b/tests/acceptance/snapshots/bashunit_category_test_sh.test_bashunit_run_with_category_option.snapshot @@ -0,0 +1,7 @@ +Running ./tests/acceptance/fixtures/test_bashunit_categories.sh +✓ Passed: Slow example + +Tests:  1 passed, 1 total +Assertions: 1 passed, 1 total + + All tests passed  diff --git a/tests/acceptance/snapshots/bashunit_path_test_sh.test_bashunit_without_path_env_nor_argument.snapshot b/tests/acceptance/snapshots/bashunit_path_test_sh.test_bashunit_without_path_env_nor_argument.snapshot index b43fb469..84b9121d 100644 --- a/tests/acceptance/snapshots/bashunit_path_test_sh.test_bashunit_without_path_env_nor_argument.snapshot +++ b/tests/acceptance/snapshots/bashunit_path_test_sh.test_bashunit_without_path_env_nor_argument.snapshot @@ -16,6 +16,9 @@ Options: -f, --filter Filters the tests to run based on the test name. + -c, --category + Filters the tests to run based on @category tags. + -l, --log-junit Create a report JUnit XML file that contains information about the test results. diff --git a/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_help.snapshot b/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_help.snapshot index 43dd8ec2..b5546968 100644 --- a/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_help.snapshot +++ b/tests/acceptance/snapshots/bashunit_test_sh.test_bashunit_should_display_help.snapshot @@ -15,6 +15,9 @@ Options: -f, --filter Filters the tests to run based on the test name. + -c, --category + Filters the tests to run based on @category tags. + -l, --log-junit Create a report JUnit XML file that contains information about the test results. diff --git a/tests/unit/helpers_test.sh b/tests/unit/helpers_test.sh index 64853bce..3e559d75 100644 --- a/tests/unit/helpers_test.sh +++ b/tests/unit/helpers_test.sh @@ -225,6 +225,18 @@ function test_to_run_with_filter_matching_string_in_function_name() { "$(helper::get_functions_to_run "test" "awesome" "${functions[*]}")" } +function test_get_function_categories() { + # shellcheck disable=SC2317 + # @category slow integration + function fake_function_with_categories() { + return 0 + } + + assert_same \ + "slow integration" \ + "$(helper::get_function_categories "fake_function_with_categories" "${BASH_SOURCE[0]}")" +} + function test_interpolate_fn_name() { local result result="$(helper::interpolate_function_name "test_name_::1::_foo" "bar")"