Add a dynamic spinner to show the progress during collection. (#96)

Author: tobiasraabe

docs/changes.rst

@@ -22,6 +22,7 @@ all releases are available on `PyPI <https://pypi.org/project/pytask>`_ and
   file size of products.
 - :gh:`93` fixes the display of parametrized arguments in the console.
 - :gh:`94` adds ``--show-locals`` which allows to print local variables in tracebacks.
+- :gh:`96` implements a spinner to show the progress during the collection.
 
 
 0.0.14 - 2021-03-23

docs/tutorials/how_to_debug.rst

@@ -1,19 +1,17 @@
 How to debug
 ============
 
-The debug mode is one of pytask's biggest strength. Whenever you encounter an error in
-one of your tasks, jump right into the code and inspect the cause of the exception.
+Whenever you encounter an error in one of your tasks, pytask offers multiple ways which
+help you to gain more information on the root cause.
 
-Quick and easy feedback through a debugger is immensely valuable because it helps you to
-be more productive and gain more confidence in your code.
-
-To facilitate debugging, pytask offers two command-line options.
+Through quick and easy feedback you are more productive and gain more confidence in your
+code.
 
 
 Tracebacks
 ----------
 
-You can enrich the display of tracebacks by show local variables in each stack frame.
+You can enrich the display of tracebacks by showing local variables in each stack frame.
 Just execute pytask with
 
 .. code-block:: console

src/_pytask/cli.py

@@ -43,7 +43,6 @@ def pytask_add_hooks(pm):
     from _pytask import collect
     from _pytask import collect_command
     from _pytask import config
-    from _pytask import console
     from _pytask import database
     from _pytask import debugging
     from _pytask import execute
@@ -62,7 +61,6 @@ def pytask_add_hooks(pm):
     pm.register(collect)
     pm.register(collect_command)
     pm.register(config)
-    pm.register(console)
     pm.register(database)
     pm.register(debugging)
     pm.register(execute)

src/_pytask/collect.py

@@ -11,6 +11,7 @@
 from _pytask.config import hookimpl
 from _pytask.config import IS_FILE_SYSTEM_CASE_SENSITIVE
 from _pytask.console import console
+from _pytask.console import generate_collection_status
 from _pytask.enums import ColorCode
 from _pytask.exceptions import CollectionError
 from _pytask.mark import has_marker
@@ -20,6 +21,7 @@
 from _pytask.nodes import reduce_node_name
 from _pytask.path import find_case_sensitive_path
 from _pytask.report import CollectionReport
+from rich.live import Live
 from rich.traceback import Traceback
 
 
@@ -28,17 +30,14 @@ def pytask_collect(session):
     """Collect tasks."""
     session.collection_start = time.time()
 
-    reports = _collect_from_paths(session)
-    tasks = _extract_successfully_collected_tasks_from_reports(reports)
+    with Live(generate_collection_status(0), console=console, transient=True) as live:
+        _collect_from_paths(session, live)
 
     try:
-        session.hook.pytask_collect_modify_tasks(session=session, tasks=tasks)
+        session.hook.pytask_collect_modify_tasks(session=session, tasks=session.tasks)
     except Exception:
         report = CollectionReport.from_exception(exc_info=sys.exc_info())
-        reports.append(report)
-
-    session.collection_reports = reports
-    session.tasks = tasks
+        session.collection_reports.append(report)
 
     session.hook.pytask_collect_log(
         session=session, reports=session.collection_reports, tasks=session.tasks
@@ -47,21 +46,21 @@ def pytask_collect(session):
     return True
 
 
-def _collect_from_paths(session):
+def _collect_from_paths(session, live=None):
     """Collect tasks from paths.
 
     Go through all paths, check if the path is ignored, and collect the file if not.
 
     """
-    collected_reports = session.collection_reports
     for path in _not_ignored_paths(session.config["paths"], session):
         reports = session.hook.pytask_collect_file_protocol(
-            session=session, path=path, reports=collected_reports
+            session=session, path=path, reports=session.collection_reports
         )
         if reports is not None:
-            collected_reports.extend(reports)
-
-    return collected_reports
+            session.collection_reports.extend(reports)
+            session.tasks.extend(i.node for i in reports if i.successful)
+            if live is not None:
+                live.update(generate_collection_status(len(session.tasks)))
 
 
 @hookimpl
@@ -230,11 +229,6 @@ def _not_ignored_paths(paths: List[Path], session) -> Generator[Path, None, None
                 yield path
 
 
-def _extract_successfully_collected_tasks_from_reports(reports):
-    """Extract successfully collected tasks from reports."""
-    return [i.node for i in reports if i.successful]
-
-
 @hookimpl
 def pytask_collect_log(session, reports, tasks):
     """Log collection."""

src/_pytask/console.py

@@ -3,11 +3,8 @@
 import sys
 from typing import List
 
-import click
-from _pytask.config import hookimpl
-from _pytask.shared import convert_truthy_or_falsy_to_bool
-from _pytask.shared import get_first_non_none_value
 from rich.console import Console
+from rich.status import Status
 from rich.tree import Tree
 
 
@@ -34,28 +31,6 @@
 console = Console(color_system=_COLOR_SYSTEM)
 
 
-@hookimpl
-def pytask_extend_command_line_interface(cli):
-    show_locals_option = click.Option(
-        ["--show-locals"],
-        is_flag=True,
-        default=None,
-        help="Show local variables in tracebacks.",
-    )
-    cli.commands["build"].params.append(show_locals_option)
-
-
-@hookimpl
-def pytask_parse_config(config, config_from_file, config_from_cli):
-    config["show_locals"] = get_first_non_none_value(
-        config_from_cli,
-        config_from_file,
-        key="show_locals",
-        default=False,
-        callback=convert_truthy_or_falsy_to_bool,
-    )
-
-
 def format_strings_as_flat_tree(strings: List[str], title: str, icon: str) -> str:
     """Format list of strings as flat tree."""
     tree = Tree(title)
@@ -84,3 +59,10 @@ def escape_squared_brackets(string: str) -> str:
 
     """
     return string.replace("[", "\\[")
+
+
+def generate_collection_status(n_collected_tasks):
+    """Generate the status object to display the progress during collection."""
+    return Status(
+        f"Collected {n_collected_tasks} tasks.", refresh_per_second=4, spinner="dots"
+    )

src/_pytask/debugging.py

@@ -37,6 +37,12 @@ def pytask_extend_command_line_interface(cli):
             ),
             metavar="module_name:class_name",
         ),
+        click.Option(
+            ["--show-locals"],
+            is_flag=True,
+            default=None,
+            help="Show local variables in tracebacks.",
+        ),
     ]
     cli.commands["build"].params.extend(additional_parameters)
 
@@ -65,6 +71,13 @@ def pytask_parse_config(config, config_from_cli, config_from_file):
         default=None,
         callback=_pdbcls_callback,
     )
+    config["show_locals"] = get_first_non_none_value(
+        config_from_cli,
+        config_from_file,
+        key="show_locals",
+        default=False,
+        callback=convert_truthy_or_falsy_to_bool,
+    )
 
 
 def _pdbcls_callback(x):

tests/test_console.py

@@ -5,6 +5,7 @@
 
 import pytest
 from _pytask.debugging import _pdbcls_callback
+from pytask import cli
 
 try:
     import pexpect
@@ -422,3 +423,25 @@ def test_pdb_used_outside_test(tmp_path):
     child.expect("Pdb")
     child.sendeof()
     _flush(child)
+
+
+@pytest.mark.end_to_end
+def test_printing_of_local_variables(tmp_path, runner):
+    source = """
+    def task_dummy():
+        a = 1
+        helper()
+
+    def helper():
+        b = 2
+        raise Exception
+    """
+    tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source))
+
+    result = runner.invoke(cli, [tmp_path.as_posix(), "--show-locals"])
+    assert result.exit_code == 1
+
+    captured = result.output
+    assert " locals " in captured
+    assert "a = 1" in captured
+    assert "b = 2" in captured