diff --git a/src/pyinfra/facts/pkg.py b/src/pyinfra/facts/pkg.py index 04aa21551..d58b7a0b1 100644 --- a/src/pyinfra/facts/pkg.py +++ b/src/pyinfra/facts/pkg.py @@ -1,30 +1,86 @@ from __future__ import annotations +import re + from typing_extensions import override from pyinfra.api import FactBase -from .util.packaging import parse_packages +from .util.packages import PackageInfo, build_package_map + +# Separates installed-package lines from the `pkg version` remote comparison +# lines in the compound command output below. +_REMOTE_MARKER = "__pyinfra_pkg_remote__" +_REMOTE_HAS_RE = re.compile(r"\(remote has ([^)]+)\)") -class PkgPackages(FactBase): + +class PkgPackages(FactBase[list[PackageInfo]]): """ - Returns a dict of installed pkg packages: + Returns a list of installed pkg packages as ``PackageInfo``, sorted by + name. On FreeBSD (``pkg``) entries carry the locked status (``HELD``, via + ``pkg lock``) and available upgrades (``UPGRADEABLE``, compared against + the remote catalogues). With the ``pkg_info`` fallback used on other BSDs + every entry is ``INSTALLED``. .. code:: python - { - "package_name": ["version"], - } + [ + PackageInfo( + name="zsh", + installed_versions=("5.9_5",), + available_version="5.9_6", + status=PackageStatus.UPGRADEABLE, + ), + ] """ regex = r"^([a-zA-Z0-9_\-\+]+)\-([0-9a-z\.]+)" - default = dict + default = list @override def command(self) -> str: - return "pkg info || pkg_info || true" + return ( + f"(pkg query '%n %v %k' && echo {_REMOTE_MARKER} && (pkg version -vRL= || true))" + " || pkg_info || true" + ) @override - def process(self, output): - return parse_packages(self.regex, output) + def process(self, output: list[str]) -> list[PackageInfo]: + installed: dict[str, set[str]] = {} + upgradeable: dict[str, str] = {} + held: set[str] = set() + + # The marker only appears when `pkg query` ran, so its presence tells + # us whether to parse `name version locked` fields or pkg_info lines. + is_pkg_query = any(line.strip() == _REMOTE_MARKER for line in output) + + in_remote = False + for line in output: + if line.strip() == _REMOTE_MARKER: + in_remote = True + continue + + if in_remote: + # `pkg version -vRL=` line: `name-version < needs updating (remote has X)` + parts = line.split() + if len(parts) >= 2 and parts[1] == "<": + match = _REMOTE_HAS_RE.search(line) + if match: + upgradeable[parts[0].rsplit("-", 1)[0]] = match.group(1) + continue + + if is_pkg_query: + # `pkg query '%n %v %k'` line: `name version locked(0|1)` + parts = line.split() + if len(parts) == 3 and parts[2] in ("0", "1"): + name, version, locked = parts + installed.setdefault(name, set()).add(version) + if locked == "1": + held.add(name) + else: + match = re.match(self.regex, line) + if match: + installed.setdefault(match.group(1), set()).add(match.group(2)) + + return build_package_map(installed, upgradeable, held) diff --git a/src/pyinfra/facts/util/packages.py b/src/pyinfra/facts/util/packages.py index 6113ab220..c15ede299 100644 --- a/src/pyinfra/facts/util/packages.py +++ b/src/pyinfra/facts/util/packages.py @@ -6,6 +6,8 @@ ``dict[str, set[str]]``. """ +from __future__ import annotations + import re from dataclasses import dataclass from enum import Enum @@ -57,29 +59,46 @@ class PackageInfo: def installed_version(self) -> str | None: return self.installed_versions[-1] if self.installed_versions else None + @classmethod + def from_dict(cls, data: dict) -> PackageInfo: + """Build a :class:`PackageInfo` from a dict-shaped description. + + This is useful for consumers that receive plain JSON-like package + data (for example test fixtures or inventory variables) and need a + real :class:`PackageInfo` instance. + """ + return cls( + name=data["name"], + installed_versions=tuple(data.get("installed_versions", ())), + available_version=data.get("available_version"), + status=PackageStatus(data.get("status", "installed")), + ) + def build_package_map( installed: dict[str, set[str]], upgradeable: dict[str, str] | None = None, held: set[str] | None = None, -) -> dict[str, PackageInfo]: - """Build a :class:`PackageInfo` map by combining sub-fact data. +) -> list[PackageInfo]: + """Build a list of :class:`PackageInfo` by combining package data. - + installed: installed packages from a fact (name to set of versions). + + installed: installed packages (name to set of versions). + upgradeable: packages with available upgrades (name to available version). + held: names of held/locked/pinned packages. - Versions are sorted with a natural-order key (digit runs as integers) so - multi-version output is deterministic across runs and the highest version - is last. + The result is a flat ``list[PackageInfo]`` sorted by name (each entry + carries its own name, so a name-keyed dict would only duplicate it). + Versions within an entry are sorted with a natural-order key (digit runs + as integers) so multi-version output is deterministic across runs and the + highest version is last. """ - result: dict[str, PackageInfo] = {} + result: list[PackageInfo] = [] _upgradeable = upgradeable or {} _held = held or set() - for name, versions in installed.items(): - sorted_versions = tuple(sorted(versions, key=_version_sort_key)) + for name in sorted(installed): + sorted_versions = tuple(sorted(installed[name], key=_version_sort_key)) if name in _held: status = PackageStatus.HELD @@ -88,11 +107,13 @@ def build_package_map( else: status = PackageStatus.INSTALLED - result[name] = PackageInfo( - name=name, - installed_versions=sorted_versions, - available_version=_upgradeable.get(name), - status=status, + result.append( + PackageInfo( + name=name, + installed_versions=sorted_versions, + available_version=_upgradeable.get(name), + status=status, + ) ) return result diff --git a/src/pyinfra/operations/pkg.py b/src/pyinfra/operations/pkg.py index 3da9bf21c..df35ec614 100644 --- a/src/pyinfra/operations/pkg.py +++ b/src/pyinfra/operations/pkg.py @@ -9,20 +9,70 @@ from pyinfra.facts.files import File from pyinfra.facts.pkg import PkgPackages from pyinfra.facts.server import Arch, Os, OsVersion, Which +from pyinfra.facts.util.packages import PackageInfo, PackageStatus from .util.packaging import ensure_packages +@operation(is_idempotent=False) +def update(force: bool = False): + """ + Update the local pkg repository catalogues (FreeBSD ``pkg`` only). + + + force: force a full download of the repository catalogues + """ + + yield "pkg update -f" if force else "pkg update" + + @operation() -def packages(packages: str | list[str] | None = None, present=True, pkg_path: str | None = None): +def upgrade(): + """ + Upgrade all pkg packages with an available upgrade (FreeBSD ``pkg`` only). + + Noops when no installed package has an upgrade available. + + **Example:** + + .. code:: python + + from pyinfra.operations import pkg + pkg.upgrade( + name="Upgrade all pkg packages", + ) + + """ + + current_packages = [ + PackageInfo.from_dict(package) if isinstance(package, dict) else package + for package in host.get_fact(PkgPackages) + ] + + if not any(package.status is PackageStatus.UPGRADEABLE for package in current_packages): + host.noop("all packages are up to date") + return + + yield "pkg upgrade -y" + + +@operation() +def packages( + packages: str | list[str] | None = None, + present: bool = True, + latest: bool = False, + pkg_path: str | None = None, +): """ Install/remove/update pkg packages. This will use ``pkg ...`` where available (FreeBSD) and the ``pkg_*`` variants elsewhere. + packages: list of packages to ensure + present: whether the packages should be installed + + latest: upgrade installed packages with an available upgrade (FreeBSD ``pkg`` only) + pkg_path: the PKG_PATH environment variable to set + Packages locked with ``pkg lock`` always noop, even with ``latest=True``. + pkg_path: By default this is autogenerated as follows (tested/working for OpenBSD): ``http://ftp..org/pub///packages//``. Note that OpenBSD's @@ -56,11 +106,18 @@ def packages(packages: str | list[str] | None = None, present=True, pkg_path: st if pkg_path: install_command = f"PKG_PATH={pkg_path} {install_command}" + current_packages = [ + PackageInfo.from_dict(package) if isinstance(package, dict) else package + for package in host.get_fact(PkgPackages) + ] + yield from ensure_packages( host, packages, - host.get_fact(PkgPackages), + current_packages, present, install_command=install_command, uninstall_command=uninstall_command, + latest=latest, + upgrade_command="pkg upgrade -y" if is_pkg else None, ) diff --git a/src/pyinfra/operations/util/packaging.py b/src/pyinfra/operations/util/packaging.py index 8b3c61c81..20613bc2c 100644 --- a/src/pyinfra/operations/util/packaging.py +++ b/src/pyinfra/operations/util/packaging.py @@ -176,7 +176,7 @@ def _format_version( def ensure_packages( host: Host, packages_to_ensure: str | list[str] | list[PkgInfo] | None, - current_packages: dict[str, set[str]] | dict[str, PackageInfo], + current_packages: dict[str, set[str]] | dict[str, PackageInfo] | list[PackageInfo], present: bool, install_command: str | StringCommand, uninstall_command: str | StringCommand, @@ -190,12 +190,13 @@ def ensure_packages( Handles this common scenario: + We have a list of packages(/versions/urls) to ensure - + We have a map of existing package -> versions (old) or PackageInfo (new) + + We have the existing packages: ``dict[str, set[str]]`` (old), + ``dict[str, PackageInfo]`` or ``list[PackageInfo]`` (new) + We have the common command bits (install, uninstall, version "joiner") + Outputs commands to ensure our desired packages/versions + Optionally upgrades packages w/o specified version when present - When ``current_packages`` values are :class:`PackageInfo` objects, the richer + When ``current_packages`` carries :class:`PackageInfo` objects, the richer status information is used: * **HELD** packages always produce a noop, even when ``latest=True``. @@ -207,7 +208,8 @@ def ensure_packages( Args: packages_to_ensure (list): list of packages or package/versions or PkgInfo's - current_packages (dict): dict of package names -> version, or name -> PackageInfo + current_packages: dict of package names -> versions, dict of name -> \ + PackageInfo, or list of PackageInfo present (bool): whether packages should exist or not install_command (str): command to prefix to list of packages to install uninstall_command (str): as above for uninstalling packages @@ -226,6 +228,9 @@ def ensure_packages( if len(packages_to_ensure) == 0: return + if isinstance(current_packages, list): + current_packages = {package.name: package for package in current_packages} + packages: list[PkgInfo] = [] if isinstance(packages_to_ensure[0], PkgInfo): packages = cast("list[PkgInfo]", packages_to_ensure) diff --git a/src/pyinfra_cli/util.py b/src/pyinfra_cli/util.py index d717382d3..9501885ab 100644 --- a/src/pyinfra_cli/util.py +++ b/src/pyinfra_cli/util.py @@ -1,5 +1,7 @@ from __future__ import annotations +import dataclasses +import enum import json import os from datetime import datetime @@ -127,6 +129,12 @@ def json_encode(obj): if hasattr(obj, "to_json"): return obj.to_json() + if isinstance(obj, enum.Enum): + return obj.value + + if dataclasses.is_dataclass(obj) and not isinstance(obj, type): + return dataclasses.asdict(obj) + raise TypeError(f"Cannot serialize: {type(obj)} ({obj})") diff --git a/tests/facts/pkg.PkgPackages/empty.yaml b/tests/facts/pkg.PkgPackages/empty.yaml new file mode 100644 index 000000000..7545ce071 --- /dev/null +++ b/tests/facts/pkg.PkgPackages/empty.yaml @@ -0,0 +1,3 @@ +command: (pkg query '%n %v %k' && echo __pyinfra_pkg_remote__ && (pkg version -vRL= || true)) || pkg_info || true +output: [] +fact: [] diff --git a/tests/facts/pkg.PkgPackages/fallback_pkg_info.yaml b/tests/facts/pkg.PkgPackages/fallback_pkg_info.yaml new file mode 100644 index 000000000..282185623 --- /dev/null +++ b/tests/facts/pkg.PkgPackages/fallback_pkg_info.yaml @@ -0,0 +1,13 @@ +command: (pkg query '%n %v %k' && echo __pyinfra_pkg_remote__ && (pkg version -vRL= || true)) || pkg_info || true +output: | + nano-3.48 Nano editor + py-pip-20.1.1 Python pip +fact: + - name: nano + installed_versions: ["3.48"] + available_version: null + status: installed + - name: py-pip + installed_versions: ["20.1.1"] + available_version: null + status: installed diff --git a/tests/facts/pkg.PkgPackages/freebsd.yaml b/tests/facts/pkg.PkgPackages/freebsd.yaml new file mode 100644 index 000000000..2a4eb12d6 --- /dev/null +++ b/tests/facts/pkg.PkgPackages/freebsd.yaml @@ -0,0 +1,29 @@ +command: (pkg query '%n %v %k' && echo __pyinfra_pkg_remote__ && (pkg version -vRL= || true)) || pkg_info || true +output: | + zsh 5.9_5 0 + curl 8.7.1 0 + vim 9.1.0 1 + compat-lib 1.0 0 + compat-lib 1.0_1 0 + __pyinfra_pkg_remote__ + zsh-5.9_5 < needs updating (remote has 5.9_6) + vim-9.1.0 < needs updating (remote has 9.1.1) + weird-1.0 > succeeds remote (remote has 0.9) + orphan-2.0 ? orphaned: misc/orphan +fact: + - name: compat-lib + installed_versions: ["1.0", "1.0_1"] + available_version: null + status: installed + - name: curl + installed_versions: ["8.7.1"] + available_version: null + status: installed + - name: vim + installed_versions: ["9.1.0"] + available_version: "9.1.1" + status: held + - name: zsh + installed_versions: ["5.9_5"] + available_version: "5.9_6" + status: upgradeable diff --git a/tests/facts/pkg.PkgPackages/packages.json b/tests/facts/pkg.PkgPackages/packages.json deleted file mode 100644 index 0712ef2d2..000000000 --- a/tests/facts/pkg.PkgPackages/packages.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "command": "pkg info || pkg_info || true", - "output": [ - "nano-3.48" - ], - "fact": { - "nano": ["3.48"] - } -} diff --git a/tests/operations/pkg.packages/latest_up_to_date_noop.yaml b/tests/operations/pkg.packages/latest_up_to_date_noop.yaml new file mode 100644 index 000000000..da875dc40 --- /dev/null +++ b/tests/operations/pkg.packages/latest_up_to_date_noop.yaml @@ -0,0 +1,15 @@ +args: + - vim +kwargs: + latest: true +facts: + pkg.PkgPackages: + - name: vim + installed_versions: ["9.0"] + server.Which: + command=pkg: /usr/sbin/pkg + files.File: + path=/etc/installurl: + size: 0 +commands: [] +noop_description: package vim is up to date (9.0) diff --git a/tests/operations/pkg.packages/latest_upgrades.yaml b/tests/operations/pkg.packages/latest_upgrades.yaml new file mode 100644 index 000000000..bbf7bc8e1 --- /dev/null +++ b/tests/operations/pkg.packages/latest_upgrades.yaml @@ -0,0 +1,17 @@ +args: + - vim +kwargs: + latest: true +facts: + pkg.PkgPackages: + - name: vim + installed_versions: ["9.0"] + available_version: "9.1" + status: upgradeable + server.Which: + command=pkg: /usr/sbin/pkg + files.File: + path=/etc/installurl: + size: 0 +commands: + - pkg upgrade -y vim diff --git a/tests/operations/pkg.packages/locked_noop.yaml b/tests/operations/pkg.packages/locked_noop.yaml new file mode 100644 index 000000000..add95d7a2 --- /dev/null +++ b/tests/operations/pkg.packages/locked_noop.yaml @@ -0,0 +1,17 @@ +args: + - vim +kwargs: + latest: true +facts: + pkg.PkgPackages: + - name: vim + installed_versions: ["9.0"] + available_version: "9.1" + status: held + server.Which: + command=pkg: /usr/sbin/pkg + files.File: + path=/etc/installurl: + size: 0 +commands: [] +noop_description: package vim is held diff --git a/tests/operations/pkg.packages/pkg_add_packages.json b/tests/operations/pkg.packages/pkg_add_packages.json index 35a55a503..09539011c 100644 --- a/tests/operations/pkg.packages/pkg_add_packages.json +++ b/tests/operations/pkg.packages/pkg_add_packages.json @@ -4,7 +4,7 @@ "server.Os": "os", "server.OsVersion": "os_version", "server.Arch": "arch", - "pkg.PkgPackages": {}, + "pkg.PkgPackages": [], "server.Which": { "pkg": null }, diff --git a/tests/operations/pkg.packages/pkg_add_packages_with_installurl.json b/tests/operations/pkg.packages/pkg_add_packages_with_installurl.json index 98a2992c3..36742cfa2 100644 --- a/tests/operations/pkg.packages/pkg_add_packages_with_installurl.json +++ b/tests/operations/pkg.packages/pkg_add_packages_with_installurl.json @@ -4,7 +4,7 @@ "server.Os": "os", "server.OsVersion": "os_version", "server.Arch": "arch", - "pkg.PkgPackages": {}, + "pkg.PkgPackages": [], "server.Which": { "pkg": null }, diff --git a/tests/operations/pkg.packages/pkg_delete_packages.json b/tests/operations/pkg.packages/pkg_delete_packages.json index 7a3a0015d..5b4a2bd22 100644 --- a/tests/operations/pkg.packages/pkg_delete_packages.json +++ b/tests/operations/pkg.packages/pkg_delete_packages.json @@ -7,9 +7,9 @@ "server.Os": "os", "server.OsVersion": "os_version", "server.Arch": "arch", - "pkg.PkgPackages": { - "py-pip": [] - }, + "pkg.PkgPackages": [ + {"name": "py-pip", "installed_versions": ["20.1.1"]} + ], "server.Which": { "command=pkg": null } diff --git a/tests/operations/pkg.packages/pkg_install_packages.json b/tests/operations/pkg.packages/pkg_install_packages.json index 18964ff71..425a01c85 100644 --- a/tests/operations/pkg.packages/pkg_install_packages.json +++ b/tests/operations/pkg.packages/pkg_install_packages.json @@ -4,7 +4,7 @@ "server.Os": "os", "server.OsVersion": "os_version", "server.Arch": "arch", - "pkg.PkgPackages": {}, + "pkg.PkgPackages": [], "server.Which": { "command=pkg": "/bin/pkg" }, diff --git a/tests/operations/pkg.packages/pkg_remove_packages.json b/tests/operations/pkg.packages/pkg_remove_packages.json index 7c99e6dd8..615db56f9 100644 --- a/tests/operations/pkg.packages/pkg_remove_packages.json +++ b/tests/operations/pkg.packages/pkg_remove_packages.json @@ -7,9 +7,9 @@ "server.Os": "os", "server.OsVersion": "os_version", "server.Arch": "arch", - "pkg.PkgPackages": { - "py-pip": [] - }, + "pkg.PkgPackages": [ + {"name": "py-pip", "installed_versions": ["20.1.1"]} + ], "server.Which": { "command=pkg": "/bin/pkg" } diff --git a/tests/operations/pkg.update/force.yaml b/tests/operations/pkg.update/force.yaml new file mode 100644 index 000000000..81b5189a7 --- /dev/null +++ b/tests/operations/pkg.update/force.yaml @@ -0,0 +1,4 @@ +kwargs: + force: true +commands: + - pkg update -f diff --git a/tests/operations/pkg.update/update.yaml b/tests/operations/pkg.update/update.yaml new file mode 100644 index 000000000..661425d7f --- /dev/null +++ b/tests/operations/pkg.update/update.yaml @@ -0,0 +1,2 @@ +commands: + - pkg update diff --git a/tests/operations/pkg.upgrade/noop_up_to_date.yaml b/tests/operations/pkg.upgrade/noop_up_to_date.yaml new file mode 100644 index 000000000..668754806 --- /dev/null +++ b/tests/operations/pkg.upgrade/noop_up_to_date.yaml @@ -0,0 +1,6 @@ +facts: + pkg.PkgPackages: + - name: vim + installed_versions: ["9.0"] +commands: [] +noop_description: all packages are up to date diff --git a/tests/operations/pkg.upgrade/upgrades_available.yaml b/tests/operations/pkg.upgrade/upgrades_available.yaml new file mode 100644 index 000000000..b9a411424 --- /dev/null +++ b/tests/operations/pkg.upgrade/upgrades_available.yaml @@ -0,0 +1,10 @@ +facts: + pkg.PkgPackages: + - name: vim + installed_versions: ["9.0"] + available_version: "9.1" + status: upgradeable + - name: git + installed_versions: ["2.40"] +commands: + - pkg upgrade -y diff --git a/tests/test_cli/test_cli_util.py b/tests/test_cli/test_cli_util.py index d380bf492..fd7a8a719 100644 --- a/tests/test_cli/test_cli_util.py +++ b/tests/test_cli/test_cli_util.py @@ -1,6 +1,8 @@ import os import sys +from dataclasses import dataclass from datetime import datetime, timezone +from enum import Enum from io import StringIO from unittest import TestCase @@ -27,6 +29,30 @@ def test_json_encode_file(self): def test_json_encode_set(self): assert json_encode({1, 2, 3}) == [1, 2, 3] + def test_json_encode_enum(self): + class Color(Enum): + RED = "red" + + assert json_encode(Color.RED) == "red" + + def test_json_encode_dataclass(self): + @dataclass + class Point: + x: int + y: int + + assert json_encode(Point(1, 2)) == {"x": 1, "y": 2} + + def test_json_encode_dataclass_prefers_to_json(self): + @dataclass + class WithCustomJson: + value: int + + def to_json(self): + return {"custom": self.value} + + assert json_encode(WithCustomJson(1)) == {"custom": 1} + def test_setup_no_module(self): with self.assertRaises(CliError) as context: get_func_and_args(("no.op",)) diff --git a/tests/test_facts_packages.py b/tests/test_facts_packages.py index dbbbab4a3..e345e1912 100644 --- a/tests/test_facts_packages.py +++ b/tests/test_facts_packages.py @@ -1,3 +1,5 @@ +import json +from dataclasses import asdict from unittest import TestCase from pyinfra.facts.util.packages import ( @@ -6,6 +8,7 @@ _version_sort_key, build_package_map, ) +from pyinfra_cli.util import json_encode class TestVersionSortKey(TestCase): @@ -50,21 +53,25 @@ def test_installed_version_none_when_no_versions(self): assert info.installed_version is None +def _by_name(packages: list[PackageInfo]) -> dict[str, PackageInfo]: + return {package.name: package for package in packages} + + class TestBuildPackageMap(TestCase): def test_only_installed(self): result = build_package_map({"vim": {"9.0"}, "git": {"2.40"}}) - assert set(result.keys()) == {"vim", "git"} - assert result["vim"] == PackageInfo( - name="vim", installed_versions=("9.0",), status=PackageStatus.INSTALLED - ) - assert result["git"] == PackageInfo( - name="git", installed_versions=("2.40",), status=PackageStatus.INSTALLED - ) + # Output is a list sorted by name + assert result == [ + PackageInfo(name="git", installed_versions=("2.40",), status=PackageStatus.INSTALLED), + PackageInfo(name="vim", installed_versions=("9.0",), status=PackageStatus.INSTALLED), + ] def test_marks_upgradeable(self): - result = build_package_map( - installed={"vim": {"9.0"}, "git": {"2.40"}}, - upgradeable={"vim": "9.1"}, + result = _by_name( + build_package_map( + installed={"vim": {"9.0"}, "git": {"2.40"}}, + upgradeable={"vim": "9.1"}, + ) ) assert result["vim"].status == PackageStatus.UPGRADEABLE assert result["vim"].available_version == "9.1" @@ -72,25 +79,29 @@ def test_marks_upgradeable(self): assert result["git"].available_version is None def test_marks_held(self): - result = build_package_map( - installed={"vim": {"9.0"}, "git": {"2.40"}}, - held={"vim"}, + result = _by_name( + build_package_map( + installed={"vim": {"9.0"}, "git": {"2.40"}}, + held={"vim"}, + ) ) assert result["vim"].status == PackageStatus.HELD def test_held_takes_precedence_over_upgradeable(self): - result = build_package_map( - installed={"vim": {"9.0"}}, - upgradeable={"vim": "9.1"}, - held={"vim"}, + result = _by_name( + build_package_map( + installed={"vim": {"9.0"}}, + upgradeable={"vim": "9.1"}, + held={"vim"}, + ) ) assert result["vim"].status == PackageStatus.HELD assert result["vim"].available_version == "9.1" def test_handles_empty_versions(self): result = build_package_map({"foo": set()}) - assert result["foo"].installed_versions == () - assert result["foo"].installed_version is None + assert result[0].installed_versions == () + assert result[0].installed_version is None def test_multiple_versions_sorted_natural_order(self): # Sets are hash-randomized; build_package_map sorts versions ascending @@ -100,7 +111,7 @@ def test_multiple_versions_sorted_natural_order(self): result = build_package_map( {"linux-image": {"6.1.0-13", "6.1.0-12", "5.10.0-26", "5.10.0-9"}} ) - info = result["linux-image"] + info = result[0] assert info.installed_versions == ("5.10.0-9", "5.10.0-26", "6.1.0-12", "6.1.0-13") assert info.installed_version == "6.1.0-13" @@ -108,6 +119,21 @@ def test_multiple_versions_natural_order_beats_lexicographic(self): # Lexicographic sort of this set is ["1.10", "1.2", "1.9"]; the # natural-order key must instead yield 1.2 < 1.9 < 1.10. result = build_package_map({"libfoo": {"1.10", "1.2", "1.9"}}) - info = result["libfoo"] + info = result[0] assert info.installed_versions == ("1.2", "1.9", "1.10") assert info.installed_version == "1.10" + + def test_json_encode_round_trip(self): + (info,) = build_package_map({"vim": {"9.0"}}, upgradeable={"vim": "9.1"}) + assert json.loads(json.dumps(info, default=json_encode)) == { + "name": "vim", + "installed_versions": ["9.0"], + "available_version": "9.1", + "status": "upgradeable", + } + + def test_asdict_uses_enum(self): + (info,) = build_package_map({"vim": {"9.0"}}, upgradeable={"vim": "9.1"}) + data = asdict(info) + assert data["name"] == "vim" + assert data["status"] == PackageStatus.UPGRADEABLE diff --git a/tests/test_operations_utils.py b/tests/test_operations_utils.py index 8523c2537..e9dbf4aeb 100644 --- a/tests/test_operations_utils.py +++ b/tests/test_operations_utils.py @@ -414,6 +414,20 @@ def test_new_format_missing_installs(self): commands, _ = self._run({}, latest=False) assert commands == ["install vim"] + def test_list_format_is_keyed_by_name(self): + current = [ + PackageInfo( + name="vim", + installed_versions=("9.0",), + available_version="9.1", + status=PackageStatus.UPGRADEABLE, + ), + PackageInfo(name="git", installed_versions=("2.40",), status=PackageStatus.INSTALLED), + ] + commands, host = self._run(current, packages=("vim", "git", "curl"), latest=True) + assert commands == ["install curl", "upgrade vim"] + host.noop.assert_any_call("package git is up to date (2.40)") + def test_new_format_versioned_match_is_noop(self): current = { "vim": PackageInfo( diff --git a/tests/util.py b/tests/util.py index f48ab08d1..3da3fafe8 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,4 +1,24 @@ -from pyinfra.api import Inventory +import copy +import json +import os +import re +from datetime import datetime, timezone +from inspect import getcallargs, getfullargspec +from os import path +from pathlib import Path +from unittest.mock import patch + +from pyinfra.api import Config, Inventory +from pyinfra.api.util import get_kwargs_str +from pyinfra.facts.util.packages import PackageInfo, PackageStatus + + +def get_command_string(command): + value = command.get_raw_value() + masked_value = command.get_masked_value() + if value == masked_value: + return value + return {"raw": value, "masked": masked_value} def make_inventory(hosts=("somehost", "anotherhost"), **kwargs): @@ -19,3 +39,487 @@ def make_inventory(hosts=("somehost", "anotherhost"), **kwargs): ), **kwargs, ) + + +class FakeState: + active = True + cwd = "/" + in_op = True + in_deploy = True + pipelining = False + is_executing = False + deploy_name = None + deploy_kwargs = None + + def __init__(self): + self.inventory = Inventory(([], {})) + self.config = Config() + + def get_temp_filename(*args): + return "_tempfile_" + + +def parse_value(value): + """ + Convert JSON types to more complex Python types because JSON is lacking. + """ + + if isinstance(value, str): + if value.startswith("datetime:"): + return datetime.fromisoformat(value[9:]) + if value.startswith("path:"): + return Path(value[5:]) + return value + + if isinstance(value, list): + if value and value[0] == "set:": + return set(parse_value(value) for value in value[1:]) + return [parse_value(value) for value in value] + + if isinstance(value, dict): + if set(value) == {"packageinfo"}: + fields = value["packageinfo"] + return PackageInfo( + name=fields["name"], + installed_versions=tuple(fields.get("installed_versions", ())), + available_version=fields.get("available_version"), + status=PackageStatus(fields.get("status", "installed")), + ) + return {key: parse_value(value) for key, value in value.items()} + + return value + + +class FakeFact: + def __init__(self, data): + self.data = parse_value(data) + + def __iter__(self): + return iter(self.data) + + def __getattr__(self, key): + return getattr(self.data, key) + + def __getitem__(self, key): + return self.data[key] + + def __setitem__(self, key, value): + self.data[key] = value + + def __contains__(self, key): + return key in self.data + + def __call__(self, *args, **kwargs): + item = self.data + + for arg in args: + if arg is None: + continue + + # Support for non-JSON-able fact arguments by turning them into JSON! + if isinstance(arg, list): + arg = json.dumps(arg) + + item = item.get(arg) + + return item + + def __str__(self): + return str(self.data) + + def __unicode__(self): + return self.data + + def __eq__(self, other_thing): + return self.data == other_thing + + def __ne__(self, other_thing): + return self.data != other_thing + + def get(self, key, default=None): + if key in self.data: + return self.data[key] + + return default + + +class FakeFacts: + def __init__(self, facts): + self.facts = {key: FakeFact(value) for key, value in facts.items()} + + def __getattr__(self, key): + return self.facts.get(key) + + def __setitem__(self, key, value): + self.facts[key] = value + + def _create(self, key, data=None, args=None): + self.facts[key][args[0]] = data + + def _delete(self, key, args=None): + self.facts[key].pop(args[0], None) + + +# TODO: remove after python2 removal, as only required because of different default ordering in 2/3 +def _sort_kwargs_str(string): + return ", ".join(sorted(string.split(", "))) + + +class FakeHost: + noop_description = None + + # Current context inside an @operation function + in_op = True + in_callback_op = False + current_op_hash = None + current_op_global_arguments = None + + # Current context inside a @deploy function + in_deploy = True + current_deploy_name = None + current_deploy_kwargs = None + current_deploy_data = None + + def __init__(self, state, name, facts, data): + self.state = state + self.name = name + self.fact = FakeFacts(facts) + self.data = data + self.connector_data = {} + + @property + def print_prefix(self): + return "" + + def noop(self, description): + self.noop_description = description + + def get_temp_filename(*args, **kwargs): + return "_tempfile_" + + def get_file( + self, + remote_filename, + filename_or_io, + remote_temp_filename=None, + print_output=False, + *arguments, + ): + return True + + def get_temp_dir_config(*args, **kwargs): + return "_tempdir_" + + @staticmethod + def _get_fact_key(fact_cls): + return "{}.{}".format(fact_cls.__module__.split(".")[-1], fact_cls.__name__) + + @staticmethod + def _check_fact_args(fact_cls, kwargs): + # Check that the arguments we're going to use to fake a fact are all actual arguments in + # the fact class, otherwise the test will hide a bug in the underlying operation. + real_args = getfullargspec(fact_cls.command).args + + for key in kwargs.keys(): + assert key in real_args, ( + f"Argument {key} is not a real argument in the `{fact_cls}.command` method" + ) + + def get_fact(self, fact_cls, *args, **kwargs): + fact_key = self._get_fact_key(fact_cls) + fact = getattr(self.fact, fact_key, None) + if fact is None: + raise KeyError(f"Missing test fact: {fact_key}") + + # This does the same thing that pyinfra.apit.facts._handle_fact_kwargs does + if args or kwargs: + # Merges args & kwargs into a single kwargs dictionary + kwargs = getcallargs(fact_cls().command, *args, **kwargs) + + if kwargs: + self._check_fact_args(fact_cls, kwargs) + kwargs_str = get_kwargs_str(kwargs) + if kwargs_str not in fact: + raise KeyError(f"Missing test fact key: {fact_key} -> {kwargs_str}") + return fact.get(kwargs_str) + return fact + + +class FakeFile: + _read = False + _data = None + + def __init__(self, name, data=None): + self._name = name + self._data = data + + def read(self, *args, **kwargs): + if self._read is False: + self._read = True + + if self._data: + return self._data + return "_test_data_" + + return "" + + def readlines(self, *args, **kwargs): + if self._read is False: + self._read = True + + if self._data: + return self._data.split() + return ["_test_data_"] + + return [] + + def seek(self, *args, **kwargs): + pass + + def close(self, *args, **kwargs): + pass + + def __enter__(self, *args, **kwargs): + return self + + def __exit__(self, *args, **kwargs): + pass + + +class patch_files: + def __init__(self, local_files): + directories, files, files_data, symlinks = self._parse_local_files(local_files) + + self._files = files + self._files_data = files_data + self._directories = directories + self._symlinks = symlinks # dict mapping path -> link_target + + @staticmethod + def _parse_local_files(local_files, prefix=FakeState.cwd): + files = [] + files_data = {} + directories = {} + symlinks = {} + + prefix = path.normpath(prefix) + + for filename, file_data in local_files.get("files", {}).items(): + filepath = path.join(prefix, filename) + files.append(filepath) + files_data[filepath] = file_data + + # Parse symlinks - these are stored as {"name": "target"} + for linkname, link_target in local_files.get("links", {}).items(): + linkpath = path.join(prefix, linkname) + symlinks[linkpath] = link_target + + for dirname, dir_files in local_files.get("dirs", {}).items(): + sub_dirname = path.join(prefix, dirname) + sub_directories, sub_files, sub_files_data, sub_symlinks = ( + patch_files._parse_local_files( + dir_files, + sub_dirname, + ) + ) + + files.extend(sub_files) + files_data.update(sub_files_data) + symlinks.update(sub_symlinks) + + directories[sub_dirname] = { + "files": list(dir_files.get("files", {}).keys()), + "dirs": list(dir_files.get("dirs", {}).keys()), + "links": list(dir_files.get("links", {}).keys()), + } + directories.update(sub_directories) + + return directories, files, files_data, symlinks + + def __enter__(self): + self.patches = [ + patch("pyinfra.operations.files.os.path.exists", self.exists), + patch("pyinfra.operations.files.os.path.isfile", self.isfile), + patch("pyinfra.operations.files.os.path.isdir", self.isdir), + patch("pyinfra.operations.files.os.path.islink", self.islink), + patch("pyinfra.operations.files.os.readlink", self.readlink), + patch("pyinfra.operations.files.os.walk", self.walk), + patch("pyinfra.operations.files.os.stat", self.stat), + patch("pyinfra.operations.files.os.makedirs", lambda path: True), + patch("pyinfra.api.util.stat", self.stat), + # Builtin patches + patch("pyinfra.operations.files.open", self.get_file, create=True), + patch("pyinfra.operations.server.open", self.get_file, create=True), + patch("pyinfra.api.util.open", self.get_file, create=True), + ] + + for patched in self.patches: + patched.start() + + def __exit__(self, type_, value, traceback): + for patched in self.patches: + patched.stop() + + def get_file(self, filename, *args): + if self.isfile(filename): + normalized_path = path.normpath(filename) + return FakeFile(normalized_path, self._files_data.get(normalized_path)) + + raise OSError(f"Missing FakeFile: {filename}") + + def exists(self, filename, *args): + return self.isfile(filename) or self.isdir(filename) or self.islink(filename) + + def isfile(self, filename, *args): + normalized_path = path.normpath(filename) + return normalized_path in self._files + + def isdir(self, dirname, *args): + normalized_path = path.normpath(dirname) + return normalized_path in self._directories + + def islink(self, pathname, *args): + normalized_path = path.normpath(pathname) + return normalized_path in self._symlinks + + def readlink(self, pathname, *args): + normalized_path = path.normpath(pathname) + if normalized_path in self._symlinks: + return self._symlinks[normalized_path] + raise OSError(f"No such file or directory: {pathname}") + + def stat(self, pathname): + try: + fileinfo = copy.deepcopy(self._files_data[pathname]) + if not fileinfo: + fileinfo = dict() + except KeyError: + fileinfo = dict() + + if self.isfile(pathname): + default_mode = 33188 # 644 file + elif self.isdir(pathname): + default_mode = 16877 # 755 directory + else: + raise OSError(f"No such file or directory: {pathname}") + + default_timeval = datetime.fromisoformat("2008-08-09T13:21:44").timestamp() + defaults = dict( + mode=default_mode, ino=64321, dev=64556, nlink=1, uid=1001, gid=1001, size=10240 + ) + + if "mode" in fileinfo.keys(): + if isinstance(fileinfo["mode"], str): + perms = int(fileinfo["mode"], 8) + else: + # this assumes the mode was provided as an integer whose digits are really octal + perms = int(str(fileinfo["mode"]), 8) + + if self.isfile(pathname): + fileinfo["mode"] = 0o100000 + perms + else: + fileinfo["mode"] = 0o40000 + perms + else: + fileinfo["mode"] = defaults["mode"] + + for field in ["dev", "nlink", "uid", "gid", "size"]: + if field in fileinfo.keys(): + if isinstance(fileinfo[field], str): + fileinfo[field] = int(fileinfo[field]) + else: + fileinfo[field] = defaults[field] + + # support both "ino" and "inode" as keys for st_ino + if "ino" in fileinfo.keys(): + if isinstance(fileinfo["ino"], str): + fileinfo["ino"] = int(fileinfo["ino"]) + elif "inode" in fileinfo.keys(): + if isinstance(fileinfo["inode"], str): + fileinfo["ino"] = int(fileinfo["inode"]) + else: + fileinfo["ino"] = fileinfo["inode"] + else: + fileinfo["ino"] = defaults["ino"] + + for timefield in ["atime", "mtime", "ctime"]: + if timefield in fileinfo.keys(): + if not isinstance(fileinfo[timefield], (int, float, str)): + raise TypeError("Parameter {0} must have type float, int or str", timefield) + + if isinstance(fileinfo[timefield], str): + if fileinfo[timefield].startswith("datetime:"): + timestr = fileinfo[timefield].removeprefix("datetime:") + dt = datetime.fromisoformat(timestr.strip()) + if not dt.tzinfo: + dt = dt.replace(tzinfo=timezone.utc) + fileinfo[timefield] = dt.timestamp() + elif re.match("^-?[0-9]+$", fileinfo[timefield].strip()): + fileinfo[timefield] = int(fileinfo[timefield].strip()) + elif re.match("^-?[0-9]+(\\.[0-9]*)?$", fileinfo[timefield].strip()): + fileinfo[timefield] = float(fileinfo[timefield].strip()) + else: + raise ValueError( + "Invalid argument: {0} for {1}", fileinfo[timefield], timefield + ) + else: + fileinfo[timefield] = default_timeval + + return os.stat_result( + tuple( + fileinfo[field] + for field in [ + "mode", + "ino", + "dev", + "nlink", + "uid", + "gid", + "size", + "atime", + "mtime", + "ctime", + ] + ) + ) + + def walk(self, dirname, topdown=True, onerror=None, followlinks=False): + if not self.isdir(dirname): + return + + normalized_path = path.normpath(dirname) + dir_definition = self._directories[normalized_path] + child_dirs = list(dir_definition.get("dirs", [])) + child_files = list(dir_definition.get("files", [])) + child_links = dir_definition.get("links", []) + + # os.walk reports symlinks in filenames (for file symlinks) or dirnames (for dir symlinks) + # We add all links to filenames since os.walk includes symlinks to files in filenames + # and symlinks to directories in dirnames (but won't traverse them if followlinks=False) + for link_name in child_links: + # For simplicity, we add all symlinks to filenames since that's what our sync code + # handles. The sync code then checks os.path.islink() to identify them. + child_files.append(link_name) + + yield dirname, child_dirs, child_files + + for child in child_dirs: + full_child = path.join(dirname, child) + # Don't traverse symlinked directories when followlinks=False + if not followlinks and self.islink(full_child): + continue + yield from self.walk(full_child, topdown, onerror, followlinks) + + +def create_host(state, name=None, facts=None, data=None): + """ + Creates a FakeHost object with attached fact data. + """ + + real_facts = {} + facts = facts or {} + data = data or {} + + for name, fact_data in facts.items(): + real_facts[name] = fact_data + + return FakeHost(state, name, facts=real_facts, data=data)