Merge branch '3.x' into feat/fact-preconditions-and-requires-command

Author: DonDebonair

.github/workflows/test.yml

@@ -19,21 +19,23 @@ jobs:
       - uses: actions/checkout@v5
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
+          python-version: "3.14"
       - name: Install uv
         uses: astral-sh/setup-uv@v6
       - uses: astral-sh/ruff-action@v3
+        with:
+          args: "check --diff"
       - uses: astral-sh/ruff-action@v3
         with:
-          args: "format --check"
+          args: "format --diff"
 
   type-check:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v5
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
+          python-version: "3.14"
       - name: Install uv
         uses: astral-sh/setup-uv@v6
       - run: uv sync --group test --no-default-groups
@@ -55,9 +57,11 @@ jobs:
     strategy:
       matrix:
         os: [macos-26, macos-15, macos-14, windows-2025, windows-2022, ubuntu-24.04, ubuntu-22.04]
-        # Test every OS vs. Python 3.13, and only one for 3.1[012]
-        python-version: ["3.13"]
+        # Test every OS vs. Python 3.14, and only one for 3.1[012]
+        python-version: ["3.14"]
         include:
+          - os: ubuntu-24.04
+            python-version: "3.13"
           - os: ubuntu-24.04
             python-version: "3.12"
           - os: ubuntu-24.04
@@ -75,7 +79,7 @@ jobs:
       - run: uv sync --group test --no-default-groups --python=${{ matrix.python-version }}
       - run: uv run pytest --cov --disable-warnings
       - name: Upload coverage report to codecov
-        if: ${{ matrix.os == 'ubuntu-24.04' && matrix.python-version == '3.13' }}
+        if: ${{ matrix.os == 'ubuntu-24.04' && matrix.python-version == '3.14' }}
         uses: codecov/codecov-action@v5
 
   # End-to-end tests

docs/utils.py

@@ -1,4 +1,8 @@
 import re
+from inspect import getmembers, ismodule
+from pathlib import Path
+from types import ModuleType
+from typing import Generator, Any
 
 
 def title_line(char, string):
@@ -15,3 +19,47 @@ def _bold_arg(m):
 
     # Python's __doc__ attribute already dedents docstrings, so we don't need to strip anything
     return line
+
+
+def including_sub_modules(module: ModuleType) -> Generator[ModuleType]:
+    """Yield all modules to be examined, including the base modules."""
+    yield module
+    module_name = module.__name__
+    for key, value in getmembers(module):
+        if (
+            ismodule(value)
+            and value.__name__.startswith(module_name)
+            and (not key.startswith("__"))
+        ):
+            yield from including_sub_modules(value)
+
+
+def get_module_names(
+    src_dir: Path,
+    *,
+    exclude_dir: str | list[str] | None = None,
+    exclude_file: str | list[str] | None = None,
+) -> list[str]:
+    """Return file names of all modules found in src_dir."""
+    exclude_path = set(exclude_dir or ["util", "__pycache__"])
+    exclude_name = set(exclude_file or ["__init__.py"])
+
+    module_names = [
+        path.stem
+        for path in (src_dir.iterdir())
+        if (
+            (path.is_dir() and (path.name not in exclude_path))
+            or ((path.suffix == ".py") and (path.name not in exclude_name))
+        )
+    ]
+    return module_names
+
+
+def remove_dups(all: list[tuple[str, Any]]) -> list[tuple[str, Any]]:
+    """Remove items with duplicate values, i.e. the same function or module found again."""
+    unique, seen = [], set()
+    for key, value in all:
+        if value not in seen:
+            seen.add(value)
+            unique.append((key, value))
+    return unique

pyproject.toml

@@ -36,6 +36,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: System :: Systems Administration",
     "Topic :: System :: Installation/Setup",
     "Topic :: Utilities",
@@ -134,7 +135,6 @@ markers = [
 ]
 
 [tool.coverage.run]
-concurrency = ["gevent"]
 
 [tool.coverage.report]
 show_missing = true

scripts/dev-lint.sh

@@ -3,7 +3,8 @@
 set -euo pipefail
 
 echo "Execute ruff check..."
-uv run ruff check
+uv run ruff check --diff
+uv run ruff format --diff
 
 echo "Execute mypy..."
 uv run mypy

scripts/generate_facts_docs.py

@@ -1,30 +1,32 @@
 #!/usr/bin/env python
 
 import sys
-from glob import glob
 from importlib import import_module
 from inspect import getfullargspec, getmembers, isclass
 from os import makedirs, path
+from pathlib import Path
 from types import FunctionType, MethodType
 
 from pyinfra.api.facts import FactBase, ShortFactBase
 
 sys.path.append(".")
-from docs.utils import format_doc_line, title_line  # noqa: E402
+from docs.utils import (
+    format_doc_line,
+    title_line,
+    including_sub_modules,
+    get_module_names,
+    remove_dups,
+)  # noqa: E402
 
 
 def build_facts_docs():
     this_dir = path.dirname(path.realpath(__file__))
     docs_dir = path.abspath(path.join(this_dir, "..", "docs"))
-    facts_dir = path.join(this_dir, "..", "src", "pyinfra", "facts", "*.py")
+    pyinfra_dir = Path(docs_dir).parent / "src" / "pyinfra"
 
     makedirs(path.join(docs_dir, "facts"), exist_ok=True)
 
-    fact_module_names = [
-        path.basename(name)[:-3] for name in glob(facts_dir) if not name.endswith("__init__.py")
-    ]
-
-    for module_name in sorted(fact_module_names):
+    for module_name in sorted(get_module_names(pyinfra_dir / "facts")):
         lines = []
         print("--> Doing fact module: {0}".format(module_name))
         module = import_module("pyinfra.facts.{0}".format(module_name))
@@ -37,22 +39,27 @@ def build_facts_docs():
         if module.__doc__:
             lines.append(module.__doc__)
 
-        if path.exists(path.join(this_dir, "..", "pyinfra", "operations", f"{module_name}.py")):
+        ops_paths = {
+            pyinfra_dir / "operations" / name for name in [module_name, f"{module_name}.py"]
+        }
+        if any(p.exists() for p in ops_paths):
             lines.append(f"See also: :doc:`../operations/{module_name}`.")
             lines.append("")
 
-        fact_classes = [
+        all_fact_classes = [
             (key, value)
-            for key, value in getmembers(module)
+            for m in including_sub_modules(module)
+            for key, value in getmembers(m)
             if (
                 isclass(value)
                 and (issubclass(value, FactBase) or issubclass(value, ShortFactBase))
-                and value.__module__ == module.__name__
+                and value.__module__.startswith(m.__name__)
                 and value is not FactBase
                 and not value.__name__.endswith("Base")  # hacky!
             )
         ]
 
+        fact_classes = remove_dups(all_fact_classes)
         for fact, cls in fact_classes:
             # FactClass -> fact_accessor on host object
             name = fact

scripts/generate_operations_docs.py

@@ -1,34 +1,34 @@
 #!/usr/bin/env python
 
 import sys
-from glob import glob
 from importlib import import_module
 from inspect import getmembers, isclass, signature
 from os import makedirs, path
+from pathlib import Path
 from types import FunctionType
 
 from pyinfra.api.facts import FactBase
 
 sys.path.append(".")
-from docs.utils import format_doc_line, title_line  # noqa: E402
+from docs.utils import (
+    format_doc_line,
+    get_module_names,
+    including_sub_modules,
+    remove_dups,
+    title_line,
+)  # noqa: E402
 
 MODULE_DEF_LINE_MAX = 90
 
 
 def build_operations_docs():
     this_dir = path.dirname(path.realpath(__file__))
     docs_dir = path.abspath(path.join(this_dir, "..", "docs"))
-    operations_dir = path.join(this_dir, "..", "src", "pyinfra", "operations", "*.py")
+    pyinfra_dir = Path(docs_dir).parent / "src" / "pyinfra"
 
     makedirs(path.join(docs_dir, "operations"), exist_ok=True)
 
-    op_module_names = [
-        path.basename(name)[:-3]
-        for name in glob(operations_dir)
-        if not name.endswith("__init__.py")
-    ]
-
-    for module_name in op_module_names:
+    for module_name in get_module_names(pyinfra_dir / "operations"):
         lines = []
 
         print("--> Doing module: {0}".format(module_name))
@@ -44,7 +44,8 @@ def build_operations_docs():
 
         operation_facts = [
             (key, value)
-            for key, value in getmembers(module)
+            for m in including_sub_modules(module)
+            for key, value in getmembers(m)
             if (isclass(value) and issubclass(value, FactBase))
         ]
 
@@ -59,17 +60,20 @@ def build_operations_docs():
             lines.append("Facts used in these operations: {0}.".format(", ".join(items)))
             lines.append("")
 
-        operation_functions = [
-            (key, value._inner)
-            for key, value in getmembers(module)
+        all_operation_functions = [
+            (f"{m.__name__.split('.')[-1]}.{key}" if m != module else key, value._inner)
+            for m in including_sub_modules(module)
+            for key, value in getmembers(m)
             if (
                 isinstance(value, FunctionType)
-                and value.__module__ == module.__name__
+                and value.__module__.startswith(m.__name__)
                 and getattr(value, "_inner", False)
                 and not value.__name__.startswith("_")
                 and not key.startswith("_")
             )
         ]
+        # remove duplicates in case functions imported by __init__.py and then also seen where defined
+        operation_functions = remove_dups(all_operation_functions)
 
         for name, func in operation_functions:
             decorated_func = getattr(func, "_inner", None)

src/pyinfra/api/arguments.py

@@ -61,6 +61,8 @@ class ConnectorArguments(TypedDict, total=False):
     _su_password: str
     _doas: bool
     _doas_user: str
+    _dzdo: bool
+    _dzdo_user: str
 
     # Shell arguments
     _shell_executable: str
@@ -139,6 +141,14 @@ def generate_env(config: "Config", value: dict) -> dict:
         "Execute/apply any changes with doas as a non-root user.",
         default=lambda config: config.DOAS_USER,
     ),
+    "_dzdo": ArgumentMeta(
+        "Execute/apply any changes with dzdo.",
+        default=lambda config: config.DZDO,
+    ),
+    "_dzdo_user": ArgumentMeta(
+        "Execute/apply any changes with dzdo as a non-root user.",
+        default=lambda config: config.DZDO_USER,
+    ),
 }
 
 shell_argument_meta: dict[str, ArgumentMeta] = {
@@ -282,7 +292,7 @@ def all_global_arguments() -> List[tuple[str, Type]]:
         """
         .. caution::
             When combining privilege escalation arguments it is important to know the order they
-            are applied: ``doas`` -> ``sudo`` -> ``su``. For example
+            are applied: ``doas`` -> ``dzdo`` -> ``sudo`` -> ``su``. For example
             ``_sudo=True,_su_user="pyinfra"`` yields a command like ``sudo su pyinfra..``.
         """,
         """

src/pyinfra/api/arguments_typed.py

@@ -37,6 +37,8 @@ def __call__(
         _su_password: None | str = None,
         _doas: bool = False,
         _doas_user: None | str = None,
+        _dzdo: bool = False,
+        _dzdo_user: None | str = None,
         # Shell arguments
         _shell_executable: None | str = None,
         _chdir: None | str = None,

src/pyinfra/api/config.py

@@ -50,6 +50,9 @@ class ConfigDefaults:
     # Use doas and optional user
     DOAS: bool = False
     DOAS_USER: Optional[str] = None
+    # Use dzdo and optional user
+    DZDO: bool = False
+    DZDO_USER: Optional[str] = None
     # Only show errors but don't count as failure
     IGNORE_ERRORS: bool = False
     # Shell to use to execute commands

src/pyinfra/connectors/chroot.py

@@ -1,4 +1,5 @@
 import os
+import shlex
 from tempfile import mkstemp
 from typing import TYPE_CHECKING, Optional
 
@@ -64,7 +65,7 @@ def connect(self) -> None:
         try:
             with progress_spinner({"chroot run"}):
                 local.shell(
-                    "chroot {0} ls".format(chroot_directory),
+                    "chroot {0} ls".format(shlex.quote(chroot_directory)),
                     splitlines=True,
                 )
         except PyinfraError as e:
@@ -91,7 +92,7 @@ def run_shell_command(
 
         chroot_command = StringCommand(
             "chroot",
-            chroot_directory,
+            QuoteString(chroot_directory),
             "sh",
             "-c",
             command,
@@ -130,8 +131,8 @@ def put_file(
             chroot_directory = self.host.connector_data["chroot_directory"]
             chroot_command = StringCommand(
                 "cp",
-                temp_filename,
-                f"{chroot_directory}/{remote_filename}",
+                QuoteString(temp_filename),
+                QuoteString(f"{chroot_directory}/{remote_filename}"),
             )
 
             status, output = self.local.run_shell_command(
@@ -172,8 +173,8 @@ def get_file(
             chroot_directory = self.host.connector_data["chroot_directory"]
             chroot_command = StringCommand(
                 "cp",
-                f"{chroot_directory}/{remote_filename}",
-                temp_filename,
+                QuoteString(f"{chroot_directory}/{remote_filename}"),
+                QuoteString(temp_filename),
             )
 
             status, output = self.local.run_shell_command(