diff --git a/gator/checks/check_ExecuteFailingCommand.py b/gator/checks/check_ExecuteFailingCommand.py new file mode 100644 index 00000000..67c1f046 --- /dev/null +++ b/gator/checks/check_ExecuteFailingCommand.py @@ -0,0 +1,56 @@ +"""Check that a command executes with an expected error.""" + +import argparse + +from gator import checkers +from gator import invoke + + +def get_parser(): + """Get a parser for the arguments provided on the command-line.""" + # create the parser with the default help formatter + # use a new description since this is a stand-alone check + parser = argparse.ArgumentParser( + prog="ExecuteFailingCommand", + description="Check Provided by GatorGrader: ExecuteFailingCommand", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # Required Named Checker Arguments {{{ + + required_group = parser.add_argument_group("required checker arguments") + + # COMMAND: the command to execute + # REQUIRED? Yes + required_group.add_argument( + "--command", type=str, help="command to execute", required=True + ) + + # }}} + + # Optional Named Checker Arguments {{{ + + # None required for this checker + + # }}} + return parser + + +def parse(args): + """Use the parser on the provided arguments.""" + return checkers.parse(get_parser, args) + + +# pylint: disable=unused-argument +def act(main_parsed_arguments, check_remaining_arguments): + """Perform the action for this check.""" + # extract the two arguments for this check: + # --> command is required to specify the commit count threshold + check_parsed_arguments = parse(check_remaining_arguments) + # Directly run the check since at least one of the argument's for it is mandatory. + # This means that the use of check_ExecuteCommand would have already failed by this + # point since argparse will exit the program if a command-line argument is not provided + command = check_parsed_arguments.command + # Invoke the command with the inverse check so that the command with return code 1 should pass the check + invocations = invoke.invoke_all_command_executes_checks(command, inverse_check=True) + return invocations diff --git a/gator/invoke.py b/gator/invoke.py index 8bede576..e49099e3 100644 --- a/gator/invoke.py +++ b/gator/invoke.py @@ -696,7 +696,7 @@ def invoke_all_command_regex_checks( ) -def invoke_all_command_executes_checks(command): +def invoke_all_command_executes_checks(command, inverse_check=False): """Perform the check for whether or not a command runs without error.""" # pylint: disable=unused-variable # note that the program does not use all of these @@ -706,11 +706,19 @@ def invoke_all_command_executes_checks(command): # this is the opposite of what is used for processes # but, all other GatorGrader checks return 0 on failure and 1 on success command_passed = False - if ( - command_error == constants.markers.Empty - and command_returncode == constants.codes.Success - ): - command_passed = True + # Assume that command passes when it makes error and return code 1 in inverse check mode + if inverse_check is False: + if ( + command_error == constants.markers.Empty + and command_returncode == constants.codes.Success + ): + command_passed = True + else: + if ( + command_error != constants.markers.Empty + or command_returncode != constants.codes.Success + ): + command_passed = True # create the message and diagnostic and report the result message = "The command '" + str(command) + "'" + " executes correctly" diagnostic = "The command returned the error code " + str(command_returncode) diff --git a/tests/checks/test_check_ExecuteFailingCommand.py b/tests/checks/test_check_ExecuteFailingCommand.py new file mode 100644 index 00000000..49126ac7 --- /dev/null +++ b/tests/checks/test_check_ExecuteFailingCommand.py @@ -0,0 +1,73 @@ +"""Tests for ExecuteFailingCommand's input and verification of command-line arguments.""" + +import pytest +import os +import sys + +from unittest.mock import patch + + +from gator import arguments +from gator import report +from gator.exceptions import InvalidCheckArgumentsError +from gator.checks import check_ExecuteFailingCommand + + +@pytest.mark.parametrize( + "commandline_arguments", + [ + ([]), + (["--commandWRONG", "echo"]), + (["--command", "run", "--WRONG"]), + (["--command"]), + ], +) +def test_required_commandline_arguments_cannot_parse(commandline_arguments, capsys): + """Check that incorrect optional command-line arguments check correctly.""" + with pytest.raises(InvalidCheckArgumentsError) as excinfo: + _ = check_ExecuteFailingCommand.parse(commandline_arguments) + captured = capsys.readouterr() + # there is no standard output or error + assert captured.err == "" + assert captured.out == "" + assert excinfo.value.check_name == "ExecuteFailingCommand" + assert excinfo.value.usage + assert excinfo.value.error + + +@pytest.mark.parametrize( + "commandline_arguments", + [(["--command", "run_command_first"]), (["--command", "run_command_second"])], +) +def test_required_commandline_arguments_can_parse(commandline_arguments, not_raises): + """Check that correct optional command-line arguments check correctly.""" + with not_raises(InvalidCheckArgumentsError): + _ = check_ExecuteFailingCommand.parse(commandline_arguments) + + +@pytest.mark.parametrize( + "commandline_arguments, expected_result", + [ + (["ExecuteFailingCommand", "--command", "WrongCommand"], True), + (["ExecuteFailingCommand", "--command", 'echo "CorrectCommand"'], False), + ], +) +def test_act_produces_output(commandline_arguments, expected_result, load_check): + """Check that using the check produces output.""" + testargs = [os.getcwd()] + with patch.object(sys, "argv", testargs): + parsed_arguments, remaining_arguments = arguments.parse(commandline_arguments) + args_verified = arguments.verify(parsed_arguments) + assert args_verified is True + check = load_check(parsed_arguments) + check_result = check.act(parsed_arguments, remaining_arguments) + # check the result + assert check_result is expected_result + # check the contents of the report + assert report.get_result() is not None + assert len(report.get_result()["check"]) > 1 + assert report.get_result()["outcome"] is expected_result + if expected_result: + assert report.get_result()["diagnostic"] == "" + else: + assert report.get_result()["diagnostic"] != "" diff --git a/tests/test_invoke.py b/tests/test_invoke.py index 1c9014ec..87c79298 100644 --- a/tests/test_invoke.py +++ b/tests/test_invoke.py @@ -750,7 +750,7 @@ def test_run_command_grab_output_as_string_count_lines_exact( def test_command_executes_checks_does_not_execute_correctly(): - """Check to see if a command does not run correctly and gets a zero return value.""" + """Check to see if a command does not run correctly and gets a zero return value and fails to pass.""" # note that a zero-code means that the command did not work # this is the opposite of what is used for processes # but, all other GatorGrader checks return 0 on failure and 1 on success @@ -759,7 +759,7 @@ def test_command_executes_checks_does_not_execute_correctly(): def test_command_executes_checks_does_execute_correctly(): - """Check to see if a command does run correctly and gets a non-zero return value.""" + """Check to see if a command does run correctly and gets a non-zero return value and succeeds to pass.""" # note that a zero-code means that the command did not work # this is the opposite of what is used for processes # but, all other GatorGrader checks return 0 on failure and 1 on success @@ -767,6 +767,26 @@ def test_command_executes_checks_does_execute_correctly(): assert status_code is True +def test_command_executes_checks_not_pass_with_correct_command_in_inverse_mode(): + """Check to see if a command does run correctly and gets a non-zero return value but not pass.""" + # note that a zero-code means that the command did not work + # this is the opposite of what is used for processes + # but, all other GatorGrader checks return 0 on failure and 1 on success + status_code = invoke.invoke_all_command_executes_checks("true", inverse_check=True) + assert status_code is False + + +def test_command_executes_checks_pass_with_wrong_command_in_inverse_mode(): + """Check to see if a command does not run correctly and gets a zero return value but pass.""" + # note that a zero-code means that the command did not work + # this is the opposite of what is used for processes + # but, all other GatorGrader checks return 0 on failure and 1 on success + status_code = invoke.invoke_all_command_executes_checks( + "willnotwork", inverse_check=True + ) + assert status_code is True + + # pylint: disable=unused-argument def test_run_command_does_execute_correctly_would_make_output( reset_results_dictionary, tmpdir