Deprecate decorators for depends_on and produces. (#398)

Author: tobiasraabe

MANIFEST.in

@@ -1,6 +1,7 @@
 include CITATION
 include LICENSE
 
+recursive-include src *.pyi
 recursive-include src py.typed
 
 exclude .coveragerc

pyproject.toml

@@ -92,3 +92,4 @@ markers = [
     "end_to_end: Flag for tests that cover the whole program.",
 ]
 norecursedirs = [".idea", ".tox"]
+filterwarnings = ["ignore:'@pytask.mark.*. is deprecated:DeprecationWarning"]

src/_pytask/collect_utils.py

@@ -3,6 +3,7 @@
 
 import itertools
 import uuid
+import warnings
 from pathlib import Path
 from typing import Any
 from typing import Callable
@@ -21,6 +22,7 @@
 from _pytask.nodes import PythonNode
 from _pytask.shared import find_duplicates
 from _pytask.task_utils import parse_keyword_arguments_from_signature_defaults
+from _pytask.tree_util import PyTree
 from _pytask.tree_util import tree_leaves
 from _pytask.tree_util import tree_map
 from _pytask.tree_util import tree_map_with_path
@@ -42,9 +44,7 @@
 ]
 
 
-def depends_on(
-    objects: Any | Iterable[Any] | dict[Any, Any]
-) -> Any | Iterable[Any] | dict[Any, Any]:
+def depends_on(objects: PyTree[Any]) -> PyTree[Any]:
     """Specify dependencies for a task.
 
     Parameters
@@ -58,9 +58,7 @@ def depends_on(
     return objects
 
 
-def produces(
-    objects: Any | Iterable[Any] | dict[Any, Any]
-) -> Any | Iterable[Any] | dict[Any, Any]:
+def produces(objects: PyTree[Any]) -> PyTree[Any]:
     """Specify products of a task.
 
     Parameters
@@ -342,6 +340,18 @@ def _find_args_with_node_annotation(func: Callable[..., Any]) -> dict[str, MetaN
 Read more about products in the documentation: https://tinyurl.com/yrezszr4.
 """
 
+_WARNING_PRODUCES_AS_KWARG = """Using 'produces' as an argument name to specify \
+products is deprecated and won't be available in pytask v0.5. Instead, use the product \
+annotation, described in this tutorial: https://tinyurl.com/yrezszr4.
+
+    from typing_extensions import Annotated
+    from pytask import Product
+
+    def task_example(produces: Annotated[..., Product]):
+        ...
+
+"""
+
 
 def parse_products_from_task_function(
     session: Session, path: Path, name: str, obj: Any
@@ -369,8 +379,14 @@ def parse_products_from_task_function(
     signature_defaults = parse_keyword_arguments_from_signature_defaults(obj)
     kwargs = {**signature_defaults, **task_kwargs}
 
+    parameters_with_product_annot = _find_args_with_product_annotation(obj)
+
     # Parse products from task decorated with @task and that uses produces.
     if "produces" in kwargs:
+        if "produces" not in parameters_with_product_annot:
+            warnings.warn(
+                _WARNING_PRODUCES_AS_KWARG, category=FutureWarning, stacklevel=1
+            )
         has_produces_argument = True
         collected_products = tree_map_with_path(
             lambda p, x: _collect_product(
@@ -384,7 +400,6 @@ def parse_products_from_task_function(
         )
         out = {"produces": collected_products}
 
-    parameters_with_product_annot = _find_args_with_product_annotation(obj)
     if parameters_with_product_annot:
         has_annotation = True
         for parameter_name in parameters_with_product_annot:
@@ -436,6 +451,11 @@ def _find_args_with_product_annotation(func: Callable[..., Any]) -> list[str]:
 """
 
 
+_WARNING_STRING_DEPRECATED = """Using strings to specify a {kind} is deprecated. Pass \
+a 'pathlib.Path' instead with 'Path("{node}")'.
+"""
+
+
 def _collect_decorator_nodes(
     session: Session, path: Path, name: str, node_info: NodeInfo
 ) -> dict[str, MetaNode]:
@@ -448,23 +468,26 @@ def _collect_decorator_nodes(
 
     """
     node = node_info.value
+    kind = {"depends_on": "dependency", "produces": "product"}.get(node_info.arg_name)
 
     if not isinstance(node, (str, Path)):
         raise NodeNotCollectedError(
             _ERROR_WRONG_TYPE_DECORATOR.format(node=node, node_type=type(node))
         )
 
     if isinstance(node, str):
+        warnings.warn(
+            _WARNING_STRING_DEPRECATED.format(kind=kind, node=node),
+            category=FutureWarning,
+            stacklevel=1,
+        )
         node = Path(node)
         node_info = node_info._replace(value=node)
 
     collected_node = session.hook.pytask_collect_node(
         session=session, path=path, node_info=node_info
     )
     if collected_node is None:
-        kind = {"depends_on": "dependency", "produces": "product"}.get(
-            node_info.arg_name
-        )
         raise NodeNotCollectedError(
             f"{node!r} cannot be parsed as a {kind} for task {name!r} in {path!r}."
         )
@@ -525,10 +548,9 @@ def _collect_product(
     # The parameter defaults only support Path objects.
     if not isinstance(node, Path) and not is_string_allowed:
         raise ValueError(
-            "If you use 'produces' as a function argument of a task and pass values as "
-            "function defaults, it can only accept values of type 'pathlib.Path' or "
-            "the same value nested in tuples, lists, and dictionaries. Here, "
-            f"{node!r} has type {type(node)}."
+            "If you declare products with 'Annotated[..., Product]', only values of "
+            "type 'pathlib.Path' optionally nested in tuples, lists, and "
+            f"dictionaries are allowed. Here, {node!r} has type {type(node)}."
         )
 
     if isinstance(node, str):

src/_pytask/mark/__init__.py

@@ -39,6 +39,8 @@
     "MarkDecorator",
     "MarkGenerator",
     "ParseError",
+    "select_by_keyword",
+    "select_by_mark",
 ]
 
 

src/_pytask/mark/__init__.pyi

@@ -0,0 +1,41 @@
+from pathlib import Path
+from typing_extensions import deprecated
+from _pytask.mark.expression import Expression
+from _pytask.mark.expression import ParseError
+from _pytask.mark.structures import Mark
+from _pytask.mark.structures import MarkDecorator
+from _pytask.mark.structures import MarkGenerator
+from _pytask.tree_util import PyTree
+
+from _pytask.session import Session
+import networkx as nx
+
+def select_by_keyword(session: Session, dag: nx.DiGraph) -> set[str]: ...
+def select_by_mark(session: Session, dag: nx.DiGraph) -> set[str]: ...
+
+class MARK_GEN:  # noqa: N801
+    @deprecated(
+        "'@pytask.mark.produces' is deprecated starting pytask v0.4.0 and will be removed in v0.5.0. To upgrade your project to the new syntax, read the tutorial on product and dependencies: https://tinyurl.com/yrezszr4.",  # noqa: E501
+        category=DeprecationWarning,
+        stacklevel=1,
+    )
+    @staticmethod
+    def produces(objects: PyTree[str | Path]) -> None: ...
+    @deprecated(
+        "'@pytask.mark.depends_on' is deprecated starting pytask v0.4.0 and will be removed in v0.5.0. To upgrade your project to the new syntax, read the tutorial on product and dependencies: https://tinyurl.com/yrezszr4.",  # noqa: E501
+        category=DeprecationWarning,
+        stacklevel=1,
+    )
+    @staticmethod
+    def depends_on(objects: PyTree[str | Path]) -> None: ...
+
+__all__ = [
+    "Expression",
+    "MARK_GEN",
+    "Mark",
+    "MarkDecorator",
+    "MarkGenerator",
+    "ParseError",
+    "select_by_keyword",
+    "select_by_mark",
+]

src/_pytask/mark/structures.py

@@ -167,6 +167,12 @@ def store_mark(obj: Callable[..., Any], mark: Mark) -> None:
         )
 
 
+_DEPRECATION_DECORATOR = """'@pytask.mark.{}' is deprecated starting pytask \
+v0.4.0 and will be removed in v0.5.0. To upgrade your project to the new syntax, read \
+the tutorial on product and dependencies: https://tinyurl.com/yrezszr4.
+"""
+
+
 class MarkGenerator:
     """Factory for :class:`MarkDecorator` objects.
 
@@ -191,6 +197,13 @@ def __getattr__(self, name: str) -> MarkDecorator | Any:
         if name[0] == "_":
             raise AttributeError("Marker name must NOT start with underscore")
 
+        if name in ("depends_on", "produces"):
+            warnings.warn(
+                _DEPRECATION_DECORATOR.format(name),
+                category=DeprecationWarning,
+                stacklevel=1,
+            )
+
         # If the name is not in the set of known marks after updating,
         # then it really is time to issue a warning or an error.
         if self.config is not None and name not in self.config["markers"]:

src/_pytask/profile.py

@@ -47,7 +47,7 @@ class _ExportFormats(enum.Enum):
     CSV = "csv"
 
 
-class Runtime(BaseTable):  # type: ignore[valid-type, misc]
+class Runtime(BaseTable):
     """Record of runtimes of tasks."""
 
     __tablename__ = "runtime"

tests/test_collect.py

@@ -312,7 +312,7 @@ def task_my_task():
 
 
 @pytest.mark.end_to_end()
-def test_collect_string_product_with_task_decorator(tmp_path):
+def test_collect_string_product_with_task_decorator(runner, tmp_path):
     source = """
     import pytask
 
@@ -321,25 +321,38 @@ def task_write_text(produces="out.txt"):
         produces.touch()
     """
     tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
-    session = main({"paths": tmp_path})
-    assert session.exit_code == ExitCode.OK
+    result = runner.invoke(cli, [tmp_path.as_posix()])
+    assert result.exit_code == ExitCode.OK
     assert tmp_path.joinpath("out.txt").exists()
 
 
 @pytest.mark.end_to_end()
-def test_collect_string_product_as_function_default(tmp_path):
+def test_collect_string_product_as_function_default(runner, tmp_path):
     source = """
-    import pytask
-
     def task_write_text(produces="out.txt"):
         produces.touch()
     """
     tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
-    session = main({"paths": tmp_path})
-    assert session.exit_code == ExitCode.OK
+    result = runner.invoke(cli, [tmp_path.as_posix()])
+    assert result.exit_code == ExitCode.OK
     assert tmp_path.joinpath("out.txt").exists()
 
 
+@pytest.mark.end_to_end()
+def test_collect_string_product_raises_error_with_annotation(runner, tmp_path):
+    source = """
+    from pytask import Product
+    from typing_extensions import Annotated
+
+    def task_write_text(out: Annotated[str, Product] = "out.txt") -> None:
+        out.touch()
+    """
+    tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+    result = runner.invoke(cli, [tmp_path.as_posix()])
+    assert result.exit_code == ExitCode.COLLECTION_FAILED
+    assert "If you declare products with 'Annotated[..., Product]'" in result.output
+
+
 @pytest.mark.end_to_end()
 def test_product_cannot_mix_different_product_types(tmp_path):
     source = """
@@ -389,3 +402,23 @@ def task_example(
     report = session.collection_reports[0]
     assert report.outcome == CollectionOutcome.FAIL
     assert "The task uses multiple" in str(report.exc_info[1])
+
+
+@pytest.mark.end_to_end()
+def test_deprecation_warning_for_strings_in_depends_on(runner, tmp_path):
+    source = """
+    import pytask
+    from pathlib import Path
+
+    @pytask.mark.depends_on("in.txt")
+    @pytask.mark.produces("out.txt")
+    def task_write_text(depends_on, produces):
+        produces.touch()
+    """
+    tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+    tmp_path.joinpath("in.txt").touch()
+
+    result = runner.invoke(cli, [tmp_path.as_posix()])
+    assert "FutureWarning" in result.output
+    assert "Using strings to specify a dependency" in result.output
+    assert "Using strings to specify a product" in result.output

tests/test_mark.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import subprocess
 import sys
 import textwrap
 
@@ -354,3 +355,23 @@ def task_example():
     assert result.exit_code == ExitCode.OK
     assert "1  Succeeded" in result.output
     assert "Warnings" in result.output
+
+
+@pytest.mark.end_to_end()
+def test_deprecation_warnings_for_decorators(tmp_path):
+    source = """
+    import pytask
+
+    @pytask.mark.depends_on("in.txt")
+    @pytask.mark.produces("out.txt")
+    def task_write_text(depends_on, produces):
+        ...
+    """
+    tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source))
+    tmp_path.joinpath("in.txt").touch()
+
+    result = subprocess.run(
+        ("pytest", tmp_path.joinpath("task_module.py").as_posix()), capture_output=True
+    )
+    assert b"DeprecationWarning: '@pytask.mark.depends_on'" in result.stdout
+    assert b"DeprecationWarning: '@pytask.mark.produces'" in result.stdout