A new functional interface. (#411)

Author: tobiasraabe

.gitignore

@@ -20,3 +20,5 @@ build
 dist
 src/_pytask/_version.py
 *.pkl
+
+tests/test_jupyter/*.txt

.pre-commit-config.yaml

@@ -62,6 +62,7 @@ repos:
         additional_dependencies: [
             attrs>=21.3.0,
             click,
+            pluggy,
             types-setuptools
         ]
         pass_filenames: false
@@ -100,6 +101,10 @@ repos:
                 docs/source/tutorials/selecting_tasks.md|
                 docs/source/tutorials/set_up_a_project.md
             )$
+- repo: https://github.com/kynan/nbstripout
+  rev: 0.6.1
+  hooks:
+    - id: nbstripout
 -   repo: https://github.com/codespell-project/codespell
     rev: v2.2.5
     hooks:

docs/source/changes.md

@@ -28,9 +28,12 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
 - {pull}`408` removes `.value` from `Node` protocol.
 - {pull}`409` make `.from_annot` an optional feature of nodes.
 - {pull}`410` allows to pass functions to `PythonNode(hash=...)`.
+- {pull}`411` implements a new functional interface and adds experimental support for
+  defining and running tasks in REPLs or Jupyter notebooks.
 - {pull}`412` adds protocols for tasks.
 - {pull}`413` removes scripts to generate `.svg`s.
 - {pull}`414` allow more ruff rules.
+- {pull}`416` removes `.from_annot` again.
 
 ## 0.3.2 - 2023-06-07
 

docs/source/how_to_guides/invoking_pytask_extended.md

@@ -10,7 +10,7 @@ Invoke pytask programmatically with
 import pytask
 
 
-session = pytask.main({"paths": ...})
+session = pytask.build(paths=...)
 ```
 
 Pass command line arguments with their long name and hyphens replaced by underscores as

docs/source/reference_guides/api.md

@@ -325,7 +325,7 @@ outcome.
 
 ```{eval-rst}
 .. autofunction:: pytask.build_dag
-.. autofunction:: pytask.main
+.. autofunction:: pytask.build
 ```
 
 ## Reports

docs/source/tutorials/visualizing_the_dag.md

@@ -37,7 +37,7 @@ layouts, which are listed [here](https://graphviz.org/docs/layouts/).
 
 The programmatic and interactive interface allows customizing the figure.
 
-Similar to {func}`pytask.main`, there exists {func}`pytask.build_dag` which returns the
+Similar to {func}`pytask.build`, there exists {func}`pytask.build_dag` which returns the
 DAG as a {class}`networkx.DiGraph`.
 
 ```python

environment.yml

@@ -26,6 +26,7 @@ dependencies:
   - black
   - jupyterlab
   - matplotlib
+  - nbval
   - pre-commit
   - pygraphviz
   - pytest

src/_pytask/build.py

@@ -4,9 +4,13 @@
 import sys
 from pathlib import Path
 from typing import Any
+from typing import Callable
+from typing import Iterable
+from typing import Literal
 from typing import TYPE_CHECKING
 
 import click
+from _pytask.capture import CaptureMethod
 from _pytask.click import ColoredCommand
 from _pytask.config import hookimpl
 from _pytask.config_utils import _find_project_root_and_config
@@ -26,16 +30,50 @@
 
 
 if TYPE_CHECKING:
+    from _pytask.node_protocols import PTask
     from typing import NoReturn
 
 
 @hookimpl(tryfirst=True)
 def pytask_extend_command_line_interface(cli: click.Group) -> None:
     """Extend the command line interface."""
-    cli.add_command(build)
-
-
-def main(raw_config: dict[str, Any]) -> Session:  # noqa: C901, PLR0912, PLR0915
+    cli.add_command(build_command)
+
+
+def build(  # noqa: C901, PLR0912, PLR0913, PLR0915
+    *,
+    capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.NO,
+    check_casing_of_paths: bool = True,
+    config: Path | None = None,
+    database_url: str = "",
+    debug_pytask: bool = False,
+    disable_warnings: bool = False,
+    dry_run: bool = False,
+    editor_url_scheme: Literal["no_link", "file", "vscode", "pycharm"]  # noqa: PYI051
+    | str = "file",
+    expression: str = "",
+    force: bool = False,
+    ignore: Iterable[str] = (),
+    marker_expression: str = "",
+    max_failures: float = float("inf"),
+    n_entries_in_table: int = 15,
+    paths: str | Path | Iterable[str | Path] = (),
+    pdb: bool = False,
+    pdb_cls: str = "",
+    s: bool = False,
+    show_capture: bool = True,
+    show_errors_immediately: bool = False,
+    show_locals: bool = False,
+    show_traceback: bool = True,
+    sort_table: bool = True,
+    stop_after_first_failure: bool = False,
+    strict_markers: bool = False,
+    tasks: Callable[..., Any] | PTask | Iterable[Callable[..., Any] | PTask] = (),
+    task_files: str | Iterable[str] = "task_*.py",
+    trace: bool = False,
+    verbose: int = 1,
+    **kwargs: Any,
+) -> Session:
     """Run pytask.
 
     This is the main command to run pytask which usually receives kwargs from the
@@ -44,13 +82,73 @@ def main(raw_config: dict[str, Any]) -> Session:  # noqa: C901, PLR0912, PLR0915
 
     Parameters
     ----------
-    raw_config : dict[str, Any]
-        A dictionary with options passed to pytask. In general, this dictionary holds
-        the information passed via the command line interface.
+    capture
+        The capture method for stdout and stderr.
+    check_casing_of_paths
+        Whether errors should be raised when file names have different casings.
+    config
+        A path to the configuration file.
+    database_url
+        An URL to the database that tracks the status of tasks.
+    debug_pytask
+        Whether debug information should be shown.
+    disable_warnings
+        Whether warnings should be disabled and not displayed.
+    dry_run
+        Whether a dry-run should be performed that shows which tasks need to be rerun.
+    editor_url_scheme
+        An url scheme that allows to click on task names, node names and filenames and
+        jump right into you preferred edior to the right line.
+    expression
+        Same as ``-k`` on the command line. Select tasks via expressions on task ids.
+    force
+        Run tasks even though they would be skipped since nothing has changed.
+    ignore
+        A pattern to ignore files or directories. Refer to ``pathlib.Path.match``
+        for more info.
+    marker_expression
+        Same as ``-m`` on the command line. Select tasks via marker expressions.
+    max_failures
+        Stop after some failures.
+    n_entries_in_table
+        How many entries to display in the table during the execution. Tasks which are
+        running are always displayed.
+    paths
+        A path or collection of paths where pytask looks for the configuration and
+        tasks.
+    pdb
+        Start the interactive debugger on errors.
+    pdb_cls
+        Start a custom debugger on errors. For example:
+        ``--pdbcls=IPython.terminal.debugger:TerminalPdb``
+    s
+        Shortcut for ``pytask.build(capture"no")``.
+    show_capture
+        Choose which captured output should be shown for failed tasks.
+    show_errors_immediately
+        Show errors with tracebacks as soon as the task fails.
+    show_locals
+        Show local variables in tracebacks.
+    show_traceback
+        Choose whether tracebacks should be displayed or not.
+    sort_table
+        Sort the table of tasks at the end of the execution.
+    stop_after_first_failure
+        Stop after the first failure.
+    strict_markers
+        Raise errors for unknown markers.
+    tasks
+        A task or a collection of tasks that is passed to ``pytask.build(tasks=...)``.
+    task_files
+        A pattern to describe modules that contain tasks.
+    trace
+        Enter debugger in the beginning of each task.
+    verbose
+        Make pytask verbose (>= 0) or quiet (= 0).
 
     Returns
     -------
-    session : _pytask.session.Session
+    session : pytask.Session
         The session captures all the information of the current run.
 
     """
@@ -61,6 +159,39 @@ def main(raw_config: dict[str, Any]) -> Session:  # noqa: C901, PLR0912, PLR0915
         pm.register(cli)
         pm.hook.pytask_add_hooks(pm=pm)
 
+        raw_config = {
+            "capture": capture,
+            "check_casing_of_paths": check_casing_of_paths,
+            "config": config,
+            "database_url": database_url,
+            "debug_pytask": debug_pytask,
+            "disable_warnings": disable_warnings,
+            "dry_run": dry_run,
+            "editor_url_scheme": editor_url_scheme,
+            "expression": expression,
+            "force": force,
+            "ignore": ignore,
+            "marker_expression": marker_expression,
+            "max_failures": max_failures,
+            "n_entries_in_table": n_entries_in_table,
+            "paths": paths,
+            "pdb": pdb,
+            "pdb_cls": pdb_cls,
+            "s": s,
+            "show_capture": show_capture,
+            "show_errors_immediately": show_errors_immediately,
+            "show_locals": show_locals,
+            "show_traceback": show_traceback,
+            "sort_table": sort_table,
+            "stop_after_first_failure": stop_after_first_failure,
+            "strict_markers": strict_markers,
+            "tasks": tasks,
+            "task_files": task_files,
+            "trace": trace,
+            "verbose": verbose,
+            **kwargs,
+        }
+
         # If someone called the programmatic interface, we need to do some parsing.
         if "command" not in raw_config:
             raw_config["command"] = "build"
@@ -97,9 +228,9 @@ def main(raw_config: dict[str, Any]) -> Session:  # noqa: C901, PLR0912, PLR0915
 
                 raw_config = {**raw_config, **config_from_file}
 
-        config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config)
+        config_ = pm.hook.pytask_configure(pm=pm, raw_config=raw_config)
 
-        session = Session.from_config(config)
+        session = Session.from_config(config_)
 
     except (ConfigurationError, Exception):
         exc_info = sys.exc_info()
@@ -137,7 +268,7 @@ def main(raw_config: dict[str, Any]) -> Session:  # noqa: C901, PLR0912, PLR0915
     return session
 
 
-@click.command(cls=ColoredCommand)
+@click.command(cls=ColoredCommand, name="build")
 @click.option(
     "--debug-pytask",
     is_flag=True,
@@ -161,13 +292,13 @@ def main(raw_config: dict[str, Any]) -> Session:  # noqa: C901, PLR0912, PLR0915
     "--show-errors-immediately",
     is_flag=True,
     default=False,
-    help="Print errors with tracebacks as soon as the task fails.",
+    help="Show errors with tracebacks as soon as the task fails.",
 )
 @click.option(
     "--show-traceback/--show-no-traceback",
     type=bool,
     default=True,
-    help=("Choose whether tracebacks should be displayed or not."),
+    help="Choose whether tracebacks should be displayed or not.",
 )
 @click.option(
     "--dry-run", type=bool, is_flag=True, default=False, help="Perform a dry-run."
@@ -179,13 +310,13 @@ def main(raw_config: dict[str, Any]) -> Session:  # noqa: C901, PLR0912, PLR0915
     default=False,
     help="Execute a task even if it succeeded successfully before.",
 )
-def build(**raw_config: Any) -> NoReturn:
+def build_command(**raw_config: Any) -> NoReturn:
     """Collect tasks, execute them and report the results.
 
     The default command. pytask collects tasks from the given paths or the
     current working directory, executes them and reports the results.
 
     """
     raw_config["command"] = "build"
-    session = main(raw_config)
+    session = build(**raw_config)
     sys.exit(session.exit_code)

src/_pytask/capture.py

@@ -50,7 +50,7 @@
     from _pytask.node_protocols import PTask
 
 
-class _CaptureMethod(enum.Enum):
+class CaptureMethod(enum.Enum):
     FD = "fd"
     NO = "no"
     SYS = "sys"
@@ -63,8 +63,8 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None:
     additional_parameters = [
         click.Option(
             ["--capture"],
-            type=EnumChoice(_CaptureMethod),
-            default=_CaptureMethod.FD,
+            type=EnumChoice(CaptureMethod),
+            default=CaptureMethod.FD,
             help="Per task capturing method.",
         ),
         click.Option(
@@ -77,7 +77,7 @@ def pytask_extend_command_line_interface(cli: click.Group) -> None:
             ["--show-capture"],
             type=EnumChoice(ShowCapture),
             default=ShowCapture.ALL,
-            help=("Choose which captured output should be shown for failed tasks."),
+            help="Choose which captured output should be shown for failed tasks.",
         ),
     ]
     cli.commands["build"].params.extend(additional_parameters)
@@ -90,8 +90,11 @@ def pytask_parse_config(config: dict[str, Any]) -> None:
     Note that, ``-s`` is a shortcut for ``--capture=no``.
 
     """
+    if isinstance(config["capture"], str):
+        config["capture"] = CaptureMethod(config["capture"])
+
     if config["s"]:
-        config["capture"] = _CaptureMethod.NO
+        config["capture"] = CaptureMethod.NO
 
 
 @hookimpl
@@ -642,20 +645,20 @@ def readouterr(self) -> CaptureResult[AnyStr]:
         return CaptureResult(out, err)  # type: ignore
 
 
-def _get_multicapture(method: _CaptureMethod) -> MultiCapture[str]:
+def _get_multicapture(method: CaptureMethod) -> MultiCapture[str]:
     """Set up the MultiCapture class with the passed method.
 
     For each valid method, the function instantiates the :class:`MultiCapture` class
     with the specified buffers for ``stdin``, ``stdout``, and ``stderr``.
 
     """
-    if method == _CaptureMethod.FD:
+    if method == CaptureMethod.FD:
         return MultiCapture(in_=FDCapture(0), out=FDCapture(1), err=FDCapture(2))
-    if method == _CaptureMethod.SYS:
+    if method == CaptureMethod.SYS:
         return MultiCapture(in_=SysCapture(0), out=SysCapture(1), err=SysCapture(2))
-    if method == _CaptureMethod.NO:
+    if method == CaptureMethod.NO:
         return MultiCapture(in_=None, out=None, err=None)
-    if method == _CaptureMethod.TEE_SYS:
+    if method == CaptureMethod.TEE_SYS:
         return MultiCapture(
             in_=None, out=SysCapture(1, tee=True), err=SysCapture(2, tee=True)
         )
@@ -679,7 +682,7 @@ class CaptureManager:
 
     """
 
-    def __init__(self, method: _CaptureMethod) -> None:
+    def __init__(self, method: CaptureMethod) -> None:
         self._method = method
         self._capturing: MultiCapture[str] | None = None
 

src/_pytask/clean.py

@@ -101,7 +101,7 @@ def clean(**raw_config: Any) -> NoReturn:  # noqa: C901, PLR0912, PLR0915
     raw_config["command"] = "clean"
 
     try:
-        # Duplication of the same mechanism in :func:`pytask.main.main`.
+        # Duplication of the same mechanism in :func:`pytask.build`.
         pm = get_plugin_manager()
         from _pytask import cli