Release 0.0.4 and refactor parametrization and add hooks. (#9)

Author: tobiasraabe

.github/workflows/continuous-integration-workflow.yml

@@ -61,7 +61,7 @@ jobs:
         run: bash <(curl -s https://codecov.io/bash) -F end_to_end -c
 
       - name: Validate codecov.yml
-        if: runner.os == 'Linux' && matrix.python-version == '3.7'
+        if: runner.os == 'Linux' && matrix.python-version == '3.8'
         shell: bash -l {0}
         run: cat codecov.yml | curl --data-binary @- https://codecov.io/validate
 

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
         exclude: (debugging\.py|main\.py)
     -   id: end-of-file-fixer
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.6.2
+    rev: v2.7.1
     hooks:
     -   id: pyupgrade
         args: [--py36-plus]

docs/changes.rst

@@ -5,6 +5,13 @@ This is a record of all past pytask releases and what went into them in reverse
 chronological order. Releases follow `semantic versioning <https://semver.org/>`_ and
 all releases are available on `Anaconda.org <https://anaconda.org/pytask/pytask>`_.
 
+0.0.4 - 2020-07-22
+------------------
+
+- :gh:`9` adds hook specifications to the parametrization of tasks which allows
+  ``pytask-latex`` and ``pytask-r`` to pass different command line arguments to a
+  parametrized task and its script. Also, it prepares the release of 0.0.4.
+
 
 0.0.3 - 2020-07-19
 ------------------

docs/conf.py

@@ -19,7 +19,7 @@
 author = "Tobias Raabe"
 
 # The full version, including alpha/beta/rc tags
-release = "0.0.3"
+release = "0.0.4"
 
 
 # -- General configuration -------------------------------------------------------------

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.3
+current_version = 0.0.4
 parse = (?P<major>\d+)\.(?P<minor>\d+)(\.(?P<patch>\d+))(\-?((dev)?(?P<dev>\d+))?)
 serialize = 
 	{major}.{minor}.{patch}dev{dev}

setup.py

@@ -14,7 +14,7 @@
 
 setup(
     name="pytask",
-    version="0.0.3",
+    version="0.0.4",
     description=DESCRIPTION,
     long_description=DESCRIPTION + "\n\n" + README,
     long_description_content_type="text/x-rst",

src/pytask/__init__.py

@@ -1,7 +1,7 @@
 import pluggy
 from pytask.mark import MARK_GEN as mark  # noqa: F401, N811
 
-__version__ = "0.0.3"
+__version__ = "0.0.4"
 
 
 hookimpl = pluggy.HookimplMarker("pytask")

src/pytask/collect.py

@@ -2,7 +2,6 @@
 import glob
 import importlib
 import inspect
-import itertools
 import pprint
 import sys
 import traceback
@@ -94,7 +93,6 @@ def pytask_collect_file(session, path, reports):
                 names_and_objects = session.hook.pytask_generate_tasks(
                     session=session, name=name, obj=obj
                 )
-                names_and_objects = itertools.chain.from_iterable(names_and_objects)
             else:
                 names_and_objects = [(name, obj)]
 

src/pytask/hookspecs.py

@@ -73,11 +73,23 @@ def pytask_collect_file(session, path, reports):
     """Collect tasks from files."""
 
 
-@hookspec
+@hookspec(firstresult=True)
 def pytask_generate_tasks(session, name, obj):
     """Generate multiple tasks from name and object with parametrization."""
 
 
+@hookspec(firstresult=True)
+def generate_product_of_names_and_functions(
+    session, name, obj, base_arg_names, arg_names, arg_values
+):
+    """Generate names and functions from Cartesian product."""
+
+
+@hookspec
+def pytask_generate_tasks_add_marker(obj, kwargs):
+    """Add some keyword arguments as markers to task."""
+
+
 @hookspec(firstresult=True)
 def pytask_collect_task_protocol(session, reports, path, name, obj):
     """Start protocol to collect tasks."""

src/pytask/parametrize.py

@@ -1,50 +1,42 @@
 import copy
 import functools
-import inspect
 import itertools
 import types
 from collections.abc import Iterable
 
 import pytask
-from pytask.mark import Mark
 
 
-def parametrize(argnames, argvalues):
+def parametrize(arg_names, arg_values):
     """Parametrize task function.
 
     Parameters
     ----------
-    argnames : str, tuple of str, list of str
+    arg_names : str, tuple of str, list of str
         The names of the arguments.
-    argvalues : list, list of iterables
-        The values which correspond to names in ``argnames``.
+    arg_values : iterable
+        The values which correspond to names in ``arg_names``.
 
     This functions is more a helper function to parse the arguments of the decorator and
     to document the marker than a real function.
 
     """
-    return argnames, argvalues
+    return arg_names, arg_values
 
 
 @pytask.hookimpl
-def pytask_generate_tasks(name, obj):
+def pytask_generate_tasks(session, name, obj):
     if callable(obj):
         obj, markers = _remove_parametrize_markers_from_func(obj)
         base_arg_names, arg_names, arg_values = _parse_parametrize_markers(markers)
 
-        diff_arg_names = (
-            set(itertools.chain.from_iterable(base_arg_names))
-            - set(inspect.getfullargspec(obj).args)
-            - {"depends_on", "produces"}
-        )
-        if diff_arg_names:
-            raise ValueError(
-                f"Parametrized function '{name}' does not have the following "
-                f"parametrized arguments: {diff_arg_names}."
-            )
-
-        names_and_functions = _generate_product_of_names_and_functions(
-            name, obj, base_arg_names, arg_names, arg_values
+        names_and_functions = session.hook.generate_product_of_names_and_functions(
+            session=session,
+            name=name,
+            obj=obj,
+            base_arg_names=base_arg_names,
+            arg_names=arg_names,
+            arg_values=arg_values,
         )
 
         return names_and_functions
@@ -58,40 +50,60 @@ def _remove_parametrize_markers_from_func(obj):
     return obj, parametrize
 
 
-def _parse_parametrize_markers(markers):
-    base_arg_names = []
-    processed_arg_names = []
-    processed_arg_values = []
+def _parse_parametrize_marker(marker):
+    """Parse parametrize marker.
+
+    Parameters
+    ----------
+    marker : pytask.mark.Mark
+        A parametrize mark.
+
+    Returns
+    -------
+    base_arg_names : tuple of str
+        Contains the names of the arguments.
+    processed_arg_names : list of tuple of str
+        Each tuple in the list represents the processed names of the arguments suffixed
+        with a number indicating the iteration.
+    processed_arg_values : list of tuple of obj
+        Each tuple in the list represents the values of the arguments for each
+        iteration.
+
+    """
+    arg_names, arg_values = parametrize(*marker.args, **marker.kwargs)
 
-    for marker in markers:
-        arg_names, arg_values = parametrize(*marker.args, **marker.kwargs)
+    parsed_arg_names = _parse_arg_names(arg_names)
+    parsed_arg_values = _parse_arg_values(arg_values)
 
-        parsed_arg_names = _parse_arg_names(arg_names)
-        parsed_arg_values = _parse_arg_values(arg_values)
+    n_runs = len(parsed_arg_values)
 
-        n_runs = len(parsed_arg_values)
+    expanded_arg_names = _expand_arg_names(parsed_arg_names, n_runs)
 
-        expanded_arg_names = _expand_arg_names(parsed_arg_names, n_runs)
+    return parsed_arg_names, expanded_arg_names, parsed_arg_values
 
-        base_arg_names.append(parsed_arg_names)
-        processed_arg_names.append(expanded_arg_names)
-        processed_arg_values.append(parsed_arg_values)
+
+def _parse_parametrize_markers(markers):
+    """Parse parametrize markers."""
+    parsed_markers = [_parse_parametrize_marker(marker) for marker in markers]
+    base_arg_names = [i[0] for i in parsed_markers]
+    processed_arg_names = [i[1] for i in parsed_markers]
+    processed_arg_values = [i[2] for i in parsed_markers]
 
     return base_arg_names, processed_arg_names, processed_arg_values
 
 
-def _parse_arg_names(argnames):
-    """Parse argnames argument of parametrize decorator.
+def _parse_arg_names(arg_names):
+    """Parse arg_names argument of parametrize decorator.
 
     Parameters
     ----------
-    argnames : str, tuple of str, list or str
+    arg_names : str, tuple of str, list or str
         The names of the arguments which are parametrized.
 
     Returns
     -------
     out : str, tuples of str
-        The parse argnames.
+        The parse arg_names.
 
     Example
     -------
@@ -101,10 +113,10 @@ def _parse_arg_names(argnames):
     ('i', 'j')
 
     """
-    if isinstance(argnames, str):
-        out = tuple(i.strip() for i in argnames.split(","))
-    elif isinstance(argnames, (tuple, list)):
-        out = tuple(argnames)
+    if isinstance(arg_names, str):
+        out = tuple(i.strip() for i in arg_names.split(","))
+    elif isinstance(arg_names, (tuple, list)):
+        out = tuple(arg_names)
 
     return out
 
@@ -126,12 +138,12 @@ def _parse_arg_values(arg_values):
     ]
 
 
-def _expand_arg_names(argnames, n_runs):
+def _expand_arg_names(arg_names, n_runs):
     """Expands the names of the arguments for each run.
 
     Parameters
     ----------
-    argnames : str, list of str
+    arg_names : str, list of str
         The names of the arguments of the parametrized function.
     n_runs : int
         How many argument values are passed to the function.
@@ -145,57 +157,73 @@ def _expand_arg_names(argnames, n_runs):
     [('i0', 'j0'), ('i1', 'j1')]
 
     """
-    return [tuple(name + str(i) for name in argnames) for i in range(n_runs)]
+    return [tuple(name + str(i) for name in arg_names) for i in range(n_runs)]
 
 
-def _generate_product_of_names_and_functions(
-    name, obj, base_arg_names, arg_names, arg_values
+@pytask.hookimpl
+def generate_product_of_names_and_functions(
+    session, name, obj, base_arg_names, arg_names, arg_values
 ):
-    names_and_functions = []
-    product_arg_names = list(itertools.product(*arg_names))
-    product_arg_values = list(itertools.product(*arg_values))
-
-    for names, values in zip(product_arg_names, product_arg_values):
-        kwargs = dict(
-            zip(
-                itertools.chain.from_iterable(base_arg_names),
-                itertools.chain.from_iterable(values),
-            )
-        )
+    """Generate product of names and functions.
 
-        # Convert parametrized dependencies and products to decorator.
-        func = _copy_func(obj)
-        func.pytestmark = copy.deepcopy(obj.pytestmark)
+    This function takes all ``@pytask.mark.parametrize`` decorators applied to a
+    function and generates all combinations of parametrized arguments.
 
-        for marker_name in ["depends_on", "produces"]:
-            if marker_name in kwargs:
-                func.pytestmark.append(
-                    Mark(marker_name, _to_tuple(kwargs.pop(marker_name)), {})
+    Note that, while a single :func:`parametrize` is handled like a loop or a
+    :func:`zip`, two :func:`parametrize` decorators form a Cartesian product.
+
+    """
+    if callable(obj):
+        names_and_functions = []
+        product_arg_names = list(itertools.product(*arg_names))
+        product_arg_values = list(itertools.product(*arg_values))
+
+        for names, values in zip(product_arg_names, product_arg_values):
+            kwargs = dict(
+                zip(
+                    itertools.chain.from_iterable(base_arg_names),
+                    itertools.chain.from_iterable(values),
                 )
+            )
+
+            # Copy function and attributes to allow in-place changes.
+            func = _copy_func(obj)
+            func.pytestmark = copy.deepcopy(obj.pytestmark)
 
-        # Attach remaining parametrized arguments to the function.
-        partialed_func = functools.partial(func, **kwargs)
-        wrapped_func = functools.update_wrapper(partialed_func, func)
+            # Convert parametrized dependencies and products to decorator.
+            session.hook.pytask_generate_tasks_add_marker(obj=func, kwargs=kwargs)
+            # Attach remaining parametrized arguments to the function.
+            partialed_func = functools.partial(func, **kwargs)
+            wrapped_func = functools.update_wrapper(partialed_func, func)
 
-        name_ = f"{name}[{'-'.join(itertools.chain.from_iterable(names))}]"
-        names_and_functions.append((name_, wrapped_func))
+            name_ = f"{name}[{'-'.join(itertools.chain.from_iterable(names))}]"
+            names_and_functions.append((name_, wrapped_func))
 
-    return names_and_functions
+        return names_and_functions
+
+
+@pytask.hookimpl
+def pytask_generate_tasks_add_marker(obj, kwargs):
+    """Add some parametrized keyword arguments as decorator."""
+    if callable(obj):
+        for marker_name in ["depends_on", "produces"]:
+            if marker_name in kwargs:
+                pytask.mark.__getattr__(marker_name)(kwargs.pop(marker_name))(obj)
 
 
 def _to_tuple(x):
     return (x,) if not isinstance(x, Iterable) or isinstance(x, str) else tuple(x)
 
 
-def _copy_func(f):
+def _copy_func(func):
     """Based on https://stackoverflow.com/a/13503277/7523785."""
-    g = types.FunctionType(
-        f.__code__,
-        f.__globals__,
-        name=f.__name__,
-        argdefs=f.__defaults__,
-        closure=f.__closure__,
+    new_func = types.FunctionType(
+        func.__code__,
+        func.__globals__,
+        name=func.__name__,
+        argdefs=func.__defaults__,
+        closure=func.__closure__,
     )
-    g = functools.update_wrapper(g, f)
-    g.__kwdefaults__ = f.__kwdefaults__
-    return g
+    new_func = functools.update_wrapper(new_func, func)
+    new_func.__kwdefaults__ = func.__kwdefaults__
+    return new_func