Introduce DAN Runner (#129)

DAN (Do Anything Now), inspired from the chatGPT jailbreakers, is a runner that simply does what the user instructs.

We are adding this because we anticipate having hundreds of different scenarios to deal with,
and many different commands. You can think of this as having a custom runner without having to create one.
It is a very power "override all commands" options, with no safety checks.

We will depend heavily on documentation and user-instructing for using this.
The idea is to give the user the power of what will be run when collecting tests and doing something with them.

Author: giovanni-guidini

codecov_cli/runners/__init__.py

@@ -1,5 +1,6 @@
 import logging
 
+from codecov_cli.runners.dan_runner import DoAnythingNowRunner
 from codecov_cli.runners.python_standard_runner import PythonStandardRunner
 from codecov_cli.runners.types import LabelAnalysisRunnerInterface
 
@@ -18,6 +19,9 @@ def get_runner(cli_config, runner_name) -> LabelAnalysisRunnerInterface:
     if runner_name == "python":
         config_params = cli_config.get("runners", {}).get("python", {})
         return PythonStandardRunner(config_params)
+    elif runner_name == "dan":
+        config_params = cli_config.get("runners", {}).get("dan", {})
+        return DoAnythingNowRunner(config_params)
     logger.debug(
         f"Trying to load runner {runner_name}",
         extra=dict(

codecov_cli/runners/dan_runner.py

@@ -0,0 +1,36 @@
+import subprocess
+from typing import List, TypedDict, Union
+
+from codecov_cli.runners.types import (
+    LabelAnalysisRequestResult,
+    LabelAnalysisRunnerInterface,
+)
+
+
+class DoAnythingNowConfigParams(TypedDict):
+    collect_tests_command: Union[List[str], str]
+    process_labelanalysis_result_command: Union[List[str], str]
+
+
+class DoAnythingNowRunner(LabelAnalysisRunnerInterface):
+    def __init__(self, config_params: DoAnythingNowConfigParams = None) -> None:
+        super().__init__()
+        if config_params is None:
+            config_params = DoAnythingNowConfigParams()
+        self.params = config_params
+
+    def collect_tests(self) -> List[str]:
+        command = self.params.get("collect_tests_command", None)
+        if command is None:
+            raise Exception(
+                "DAN runner missing 'collect_tests_command' configuration value"
+            )
+        return subprocess.run(command, check=True, capture_output=True).stdout.decode()
+
+    def process_labelanalysis_result(self, result: LabelAnalysisRequestResult):
+        command = self.params.get("process_labelanalysis_result_command", None)
+        if command is None:
+            raise Exception(
+                "DAN runner missing 'process_labelanalysis_result_command' configuration value"
+            )
+        return subprocess.run(command, check=True, capture_output=True).stdout.decode()

tests/runners/test_dan_runner.py

@@ -0,0 +1,76 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from codecov_cli.runners.dan_runner import DoAnythingNowRunner
+from codecov_cli.runners.python_standard_runner import PythonStandardRunner
+
+
+class TestPythonStandardRunner(object):
+    @patch("codecov_cli.runners.dan_runner.subprocess.run")
+    def test_collect_tests(self, mock_run):
+        test_list = ["test_1", "test_2", "test_3"]
+        mock_stdout = MagicMock()
+        mock_stdout.configure_mock(
+            **{"stdout.decode.return_value": "\n".join(test_list)}
+        )
+        mock_run.return_value = mock_stdout
+
+        config_options = {"collect_tests_command": ["mycommand", "--option"]}
+        runner = DoAnythingNowRunner(config_options)
+        assert runner.params == config_options
+        resp = runner.collect_tests()
+        assert resp.split() == test_list
+        mock_run.assert_called_with(
+            ["mycommand", "--option"],
+            capture_output=True,
+            check=True,
+        )
+
+    def test_collect_test_no_config(self):
+        runner = DoAnythingNowRunner()
+        with pytest.raises(Exception) as exp:
+            runner.collect_tests()
+        assert (
+            str(exp.value)
+            == "DAN runner missing 'collect_tests_command' configuration value"
+        )
+
+    @patch("codecov_cli.runners.dan_runner.subprocess.run")
+    def test_process_labelanalysis_result(self, mock_run):
+        label_analysis_result = {
+            "present_report_labels": ["test_present"],
+            "absent_labels": ["test_absent"],
+            "present_diff_labels": ["test_in_diff"],
+            "global_level_labels": ["test_global"],
+        }
+        cmd_output = "My command output"
+        mock_stdout = MagicMock()
+        mock_stdout.configure_mock(**{"stdout.decode.return_value": cmd_output})
+        mock_run.return_value = mock_stdout
+        config_options = {
+            "process_labelanalysis_result_command": ["mycommand", "--option"]
+        }
+        runner = DoAnythingNowRunner(config_options)
+        runner.process_labelanalysis_result(label_analysis_result)
+        assert runner.params == config_options
+        mock_run.assert_called_with(
+            ["mycommand", "--option"],
+            capture_output=True,
+            check=True,
+        )
+
+    def test_process_labelanalysis_result_no_config(self):
+        label_analysis_result = {
+            "present_report_labels": ["test_present"],
+            "absent_labels": ["test_absent"],
+            "present_diff_labels": ["test_in_diff"],
+            "global_level_labels": ["test_global"],
+        }
+        runner = DoAnythingNowRunner()
+        with pytest.raises(Exception) as exp:
+            runner.process_labelanalysis_result(label_analysis_result)
+        assert (
+            str(exp.value)
+            == "DAN runner missing 'process_labelanalysis_result_command' configuration value"
+        )

tests/runners/test_runners.py

@@ -1,12 +1,14 @@
 from unittest.mock import patch
 
 from codecov_cli.runners import get_runner
+from codecov_cli.runners.dan_runner import DoAnythingNowRunner
 from codecov_cli.runners.python_standard_runner import PythonStandardRunner
 
 
 class TestRunners(object):
     def test_get_standard_runners(self):
         assert isinstance(get_runner({}, "python"), PythonStandardRunner)
+        assert isinstance(get_runner({}, "dan"), DoAnythingNowRunner)
         # TODO: Extend with other standard runners once we create them (e.g. JS)
 
     def test_python_standard_runner_with_options(self):
@@ -17,6 +19,19 @@ def test_python_standard_runner_with_options(self):
         assert isinstance(runner_instance, PythonStandardRunner)
         assert runner_instance.params == {**config_params, "include_curr_dir": False}
 
+    def test_get_dan_runner_with_params(self):
+        config = {
+            "runners": {
+                "dan": {
+                    "collect_tests_command": ["mycommand", "--collect"],
+                    "process_labelanalysis_result_command": ["mycommand", "--process"],
+                }
+            }
+        }
+        runner = get_runner(config, "dan")
+        assert isinstance(runner, DoAnythingNowRunner)
+        assert runner.params == config["runners"]["dan"]
+
     @patch("codecov_cli.runners._load_runner_from_yaml")
     def test_get_runner_from_yaml(self, mock_load_runner):
         config = {"runners": {"my_runner": {"path": "path_to_my_runner"}}}