Add -x/--stop-after-first-failure and --max-failures for early exits. (#45)

Author: tobiasraabe

.conda/meta.yaml

@@ -39,6 +39,7 @@ test:
     - pytest
   source_files:
     - tox.ini
+    - src
     - tests
   commands:
     - pytask --help

docs/changes.rst

@@ -9,6 +9,8 @@ all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask>
 0.0.11 - 2020-xx-xx
 -------------------
 
+- :gh:`45` adds the option to stop execution after a number of tasks has failed. Closes
+  :gh:`44`.
 - :gh:`47` reduce node names in error messages while resolving dependencies.
 - :gh:`50` implements correct usage of singular and plural in collection logs.
 

docs/explanations/pluggy.rst

@@ -3,7 +3,7 @@
 pluggy and the Plugin Architecture
 ==================================
 
-pluggy ([1]_, [2]_) is at the heart of pytask and enables its plugin system. The
+pluggy ([1]_, [2]_, [3]_) is at the heart of pytask and enables its plugin system. The
 mechanism to achieve extensibility is called :term:`hooking`.
 
 At certain points, pytask, or more generally the host, implements entry-points which are
@@ -32,3 +32,6 @@ Thus, it is the host's responsibility to design the entry-points in such a way t
 
 .. [2] `A talk by Floris Bruynooghe about pluggy and pytest
        <https://youtu.be/zZsNPDfOoHU>`_.
+
+.. [3] `An introduction to pluggy by Kracekumar Ramaraju
+       <https://kracekumar.com/post/build_plugins_with_pluggy>`

docs/tutorials/how_to_invoke_pytask.rst

@@ -0,0 +1,43 @@
+How to invoke pytask
+====================
+
+Entry-points
+------------
+
+There are two entry-points to invoke pytask.
+
+1. Use command line interface with
+
+   .. code-block:: console
+
+       $ pytask
+
+   Use the following flags to learn more about pytask and its configuration.
+
+   .. code-block:: console
+
+       $ pytask --version
+       $ pytask -h | --help
+
+2. Invoke pytask programmatically with
+
+   .. code-block:: python
+
+       import pytask
+
+
+       session = pytask.main({"paths": ...})
+
+   Pass command line arguments with their long name and hyphens replaced by underscores
+   as keys of the dictionary.
+
+
+Stopping after the first (or N) failures
+----------------------------------------
+
+To stop the build of the project after the first (N) failures use
+
+.. code-block:: console
+
+    $ pytask -x | --stop-after-first-failure  # Stop after the first failure
+    $ pytask --max-failures 2                 # Stop after the second failure

docs/tutorials/index.rst

@@ -19,3 +19,4 @@ organize and start your own project.
    how_to_collect
    how_to_make_tasks_persist
    how_to_capture
+   how_to_invoke_pytask

src/_pytask/build.py

@@ -90,6 +90,14 @@ def main(config_from_cli):
     default=None,
     help="Trace all function calls in the plugin framework.  [default: False]",
 )
+@click.option(
+    "-x",
+    "--stop-after-first-failure",
+    is_flag=True,
+    default=None,
+    help="Stop after the first failure.",
+)
+@click.option("--max-failures", default=None, help="Stop after some failures.")
 def build(**config_from_cli):
     """Collect and execute tasks and report the results.
 

src/_pytask/config.py

@@ -149,6 +149,24 @@ def pytask_parse_config(config, config_from_cli, config_from_file):
         )
     )
 
+    config["stop_after_first_failure"] = get_first_non_none_value(
+        config_from_cli,
+        config_from_file,
+        key="stop_after_first_failure",
+        default=False,
+        callback=convert_truthy_or_falsy_to_bool,
+    )
+    if config["stop_after_first_failure"]:
+        config["max_failures"] = 1
+    else:
+        config["max_failures"] = get_first_non_none_value(
+            config_from_cli,
+            config_from_file,
+            key="max_failures",
+            default=float("inf"),
+            callback=lambda x: x if x is None else int(x),
+        )
+
 
 @hookimpl
 def pytask_post_parse(config):

src/_pytask/execute.py

@@ -23,7 +23,7 @@ def pytask_execute(session):
     """Execute tasks."""
     session.hook.pytask_execute_log_start(session=session)
     session.scheduler = session.hook.pytask_execute_create_scheduler(session=session)
-    session.execution_reports = session.hook.pytask_execute_build(session=session)
+    session.hook.pytask_execute_build(session=session)
     session.hook.pytask_execute_log_end(
         session=session, reports=session.execution_reports
     )
@@ -49,12 +49,13 @@ def pytask_execute_create_scheduler(session):
 @hookimpl
 def pytask_execute_build(session):
     """Execute tasks."""
-    reports = []
     for task in session.scheduler:
         report = session.hook.pytask_execute_task_protocol(session=session, task=task)
-        reports.append(report)
+        session.execution_reports.append(report)
+        if session.should_stop:
+            return True
 
-    return reports
+    return True
 
 
 @hookimpl
@@ -138,6 +139,11 @@ def pytask_execute_task_process_report(session, report):
                     {"reason": f"Previous task '{task.name}' failed."},
                 )
             )
+
+        session.n_tests_failed += 1
+        if session.n_tests_failed >= session.config["max_failures"]:
+            session.should_stop = True
+
     return True
 
 

src/_pytask/session.py

@@ -35,6 +35,11 @@ class Session:
     execution_start = attr.ib(default=None)
     execution_end = attr.ib(default=None)
 
+    n_tests_failed = attr.ib(default=0)
+    """Optional[int]: Number of tests which have failed."""
+    should_stop = attr.ib(default=False)
+    """Optional[bool]: Indicates whether the session should be stopped."""
+
     @classmethod
     def from_config(cls, config):
         """Construct the class from a config."""

tests/test_execute.py

@@ -198,3 +198,35 @@ def task_dummy(depends_on, produces):
 
     session = main({"paths": tmp_path})
     assert session.exit_code == 0
+
+
+@pytest.mark.parametrize("n_failures", [1, 2, 3])
+def test_execution_stops_after_n_failures(tmp_path, n_failures):
+    source = """
+    def task_1(): raise Exception
+    def task_2(): raise Exception
+    def task_3(): raise Exception
+    """
+    tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source))
+
+    session = main({"paths": tmp_path, "max_failures": n_failures})
+
+    assert len(session.tasks) == 3
+    assert len(session.execution_reports) == n_failures
+
+
+@pytest.mark.parametrize("stop_after_first_failure", [False, True])
+def test_execution_stop_after_first_failure(tmp_path, stop_after_first_failure):
+    source = """
+    def task_1(): raise Exception
+    def task_2(): raise Exception
+    def task_3(): raise Exception
+    """
+    tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source))
+
+    session = main(
+        {"paths": tmp_path, "stop_after_first_failure": stop_after_first_failure}
+    )
+
+    assert len(session.tasks) == 3
+    assert len(session.execution_reports) == 1 if stop_after_first_failure else 3