Implemented matching by expressions and marker expressions. (#13)

Author: tobiasraabe

docs/changes.rst

@@ -12,6 +12,7 @@ all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask>
 - :gh:`10` turns parametrization into a plugin.
 - :gh:`11` extends the documentation.
 - :gh:`12` replaces ``pytest.mark`` with ``pytask.mark``.
+- :gh:`13` implements selecting tasks via expressions or marker expressions.
 
 
 0.0.4 - 2020-07-22

docs/tutorials/how_to_select_tasks.rst

@@ -1,17 +1,81 @@
 How to select tasks
 ===================
 
-If you want to run only a subset of tasks, there exists currently one option.
+If you want to run only a subset of tasks, there exist multiple options.
 
 
-Selecting tasks via paths
--------------------------
+Paths
+-----
 
-You can run all tasks in one file by passing the path to the file to pytask. The same
-can be done for multiple paths
+You can run all tasks in one file or one directory by passing the corresponding path to
+pytask. The same can be done for multiple paths.
 
 .. code-block:: bash
 
-    $ pytask path/to/task_1.py
+    $ pytask src/task_1.py
 
-    $ pytask path/to/task_1.py path/to/task_2.py
+    $ pytask src
+
+    $ pytask src/task_1.py src/task_2.py
+
+
+Markers
+-------
+
+If you assign markers to task functions, you can use marker expressions to select tasks.
+For example, here is a task with the ``wip`` marker which indicates work-in-progress.
+
+.. code-block:: python
+
+    @pytask.mark.wip
+    def task_1():
+        pass
+
+To execute only tasks with the ``wip`` marker, use
+
+.. code-block:: bash
+
+    $ pytask -m wip
+
+You can pass more complex expressions to ``-m`` by using multiple markers and ``and``,
+``or``, ``not``, and brackets (``()``). The following pattern selects all tasks which
+belong to the data management, but not the ones which produce plots and plots produced
+for the analysis.
+
+.. code-block:: bash
+
+    $ pytask -m "(data_management and not plots) or (analysis and plots)"
+
+
+Expressions
+-----------
+
+Expressions are similar to markers and offer the same syntax but target the task ids.
+Assume you have the following tasks.
+
+.. code-block:: python
+
+    def task_1():
+        pass
+
+
+    def task_2():
+        pass
+
+
+    def task_12():
+        pass
+
+Then,
+
+.. code-block:: bash
+
+    $ pytask -k 1
+
+will execute the first and third task and
+
+.. code-block:: bash
+
+    $ pytask -k "1 and not 2"
+
+executes only the first task.

src/pytask/__init__.py

@@ -1,8 +1,5 @@
-import pluggy
-from pytask.mark.structures import MARK_GEN as mark  # noqa: F401, N811
+from pytask.config import hookimpl
+from pytask.mark_ import MARK_GEN as mark  # noqa: N811
 
-hookimpl = pluggy.HookimplMarker("pytask")
-
-
-__all__ = ["hookimpl", "mark"]
+__all__ = ["hookimpl", "main", "mark"]
 __version__ = "0.0.4"

src/pytask/cli.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 
 import click
-import pytask.mark.cli
+from pytask.config import hookimpl
 from pytask.pluginmanager import get_plugin_manager
 
 
@@ -11,42 +11,54 @@
 
 def add_parameters(func):
     """Add parameters from plugins to the commandline interface."""
-    pm = get_plugin_manager()
-    pm.register(sys.modules[__name__])
-    pm.hook.pytask_add_hooks(pm=pm)
+    pm = _prepare_plugin_manager()
     pm.hook.pytask_add_parameters_to_cli(command=func)
-
     # Hack to pass the plugin manager via a hidden option to the ``config_from_cli``.
     func.params.append(click.Option(["--pm"], default=pm, hidden=True))
 
     return func
 
 
-@pytask.hookimpl
-def pytask_add_hooks(pm):
-    """Add some hooks and plugins.
+def _prepare_plugin_manager():
+    pm = get_plugin_manager()
+    pm.register(sys.modules[__name__])
+    pm.hook.pytask_add_hooks(pm=pm)
+    return pm
 
-    This hook implementation registers only plugins which extend the command line
-    interface or patch the main entry-point :func:`pytask.hookspecs.pytask_main`.
 
-    """
+@hookimpl
+def pytask_add_hooks(pm):
+    from pytask import collect
+    from pytask import config
     from pytask import database
     from pytask import debugging
+    from pytask import execute
+    from pytask import logging
     from pytask import main
-    from pytask.mark import cli as mark_cli
+    from pytask import parametrize
+    from pytask import resolve_dependencies
+    from pytask import skipping
+    from pytask import mark_
 
+    pm.register(collect)
+    pm.register(config)
     pm.register(database)
     pm.register(debugging)
+    pm.register(execute)
+    pm.register(logging)
     pm.register(main)
-    pm.register(mark_cli)
+    pm.register(parametrize)
+    pm.register(resolve_dependencies)
+    pm.register(skipping)
+    pm.register(mark_)
 
 
 def _to_path(ctx, param, value):  # noqa: U100
     """Callback for :class:`click.Argument` or :class:`click.Option`."""
     return [Path(i).resolve() for i in value]
 
 
-@pytask.hookimpl
+@hookimpl
 def pytask_add_parameters_to_cli(command):
     additional_parameters = [
         click.Argument(
@@ -72,6 +84,12 @@ def pytask_add_parameters_to_cli(command):
 @click.version_option()
 def pytask(**config_from_cli):
     """Command-line interface for pytask."""
-    pm = config_from_cli["pm"]
-    session = pm.hook.pytask_main(config_from_cli=config_from_cli)
+    session = main(config_from_cli)
     sys.exit(session.exit_code)
+
+
+def main(config_from_cli):
+    pm = config_from_cli.get("pm", _prepare_plugin_manager())
+    session = pm.hook.pytask_main(config_from_cli=config_from_cli)
+
+    return session

src/pytask/collect.py

@@ -11,9 +11,10 @@
 import pytask
 from pytask.exceptions import CollectionError
 from pytask.exceptions import TaskDuplicatedError
-from pytask.mark.structures import has_marker
+from pytask.mark_ import has_marker
 from pytask.nodes import FilePathNode
 from pytask.nodes import PythonFunctionTask
+from pytask.report import CollectionReport
 from pytask.report import CollectionReportFile
 from pytask.report import CollectionReportTask
 
@@ -22,8 +23,16 @@
 def pytask_collect(session):
     reports = _collect_from_paths(session)
     tasks = _extract_tasks_from_reports(reports)
-    session.hook.pytask_collect_modify_tasks(tasks=tasks, config=session.config)
-    session.hook.pytask_collect_log(reports=reports, tasks=tasks, config=session.config)
+
+    try:
+        session.hook.pytask_collect_modify_tasks(session=session, tasks=tasks)
+    except Exception:
+        report = CollectionReport(
+            " Modification of collected tasks failed ", sys.exc_info()
+        )
+        reports.append(report)
+
+    session.hook.pytask_collect_log(session=session, reports=reports, tasks=tasks)
 
     session.collection_reports = reports
     session.tasks = tasks
@@ -211,10 +220,13 @@ def _extract_tasks_from_reports(reports):
 
 
 @pytask.hookimpl
-def pytask_collect_log(reports, tasks, config):
-    tm_width = config["terminal_width"]
+def pytask_collect_log(session, reports, tasks):
+    tm_width = session.config["terminal_width"]
 
-    click.echo(f"Collected {len(tasks)} task(s).")
+    message = f"Collected {len(tasks)} task(s)."
+    if session.deselected:
+        message += f" Deselected {len(session.deselected)} task(s)."
+    click.echo(message)
 
     failed_reports = [i for i in reports if not i.successful]
     if failed_reports:

src/pytask/config.py

@@ -7,11 +7,14 @@
 from pathlib import Path
 
 import click
-import pytask
-from pytask.mark.structures import MARK_GEN
+import pluggy
 from pytask.shared import get_first_not_none_value
 from pytask.shared import to_list
 
+
+hookimpl = pluggy.HookimplMarker("pytask")
+
+
 IGNORED_FILES_AND_FOLDERS = [
     "*/.git/*",
     "*/__pycache__/*",
@@ -20,7 +23,7 @@
 ]
 
 
-@pytask.hookimpl
+@hookimpl
 def pytask_configure(pm, config_from_cli):
     config = {"pm": pm, "terminal_width": _get_terminal_width()}
 
@@ -38,20 +41,18 @@ def pytask_configure(pm, config_from_cli):
         "produces": "Attach a product/products to a task.",
     }
 
-    config["pm"].hook.pytask_parse_config(
+    pm.hook.pytask_parse_config(
         config=config,
         config_from_cli=config_from_cli,
         config_from_file=config_from_file,
     )
 
-    config["pm"].hook.pytask_post_parse(config=config)
-
-    MARK_GEN.config = config
+    pm.hook.pytask_post_parse(config=config)
 
     return config
 
 
-@pytask.hookimpl
+@hookimpl
 def pytask_parse_config(config, config_from_cli, config_from_file):
     config["ignore"] = (
         get_first_not_none_value(

src/pytask/execute.py

@@ -12,7 +12,7 @@
 from pytask.database import create_or_update_state
 from pytask.exceptions import ExecutionError
 from pytask.exceptions import NodeNotFoundError
-from pytask.mark.structures import Mark
+from pytask.mark_ import Mark
 from pytask.nodes import FilePathNode
 from pytask.report import ExecutionReport
 from pytask.report import format_execute_footer

src/pytask/hookspecs.py

@@ -104,7 +104,7 @@ def pytask_ignore_collect(path, config):
 
 
 @hookspec
-def pytask_collect_modify_tasks(tasks, config):
+def pytask_collect_modify_tasks(session, tasks):
     """Modify tasks after they have been collected.
 
     This hook can be used to deselect tasks when they match a certain keyword or mark.
@@ -161,7 +161,7 @@ def pytask_collect_node(path, node):
 
 
 @hookspec(firstresult=True)
-def pytask_collect_log(reports, tasks, config):
+def pytask_collect_log(session, reports, tasks):
     """Log errors occurring during the collection.
 
     This hook reports errors during the collection.