add hatchling support for manual build mode (#35)

* hatchling support wip, package scanner, first steps with testing

* implement rest of HatchlingProject

* reorganize code

* add isolated testing mode for hatch

* create more explicit build backend selection

* move away from bild_backend to more stable build_backend and add tests

* Fix nitpicks

---------

Co-authored-by: Daniel Fangl <daniel.fangl@localstack.cloud>

Author: thrau

.gitignore

@@ -136,5 +136,8 @@ venv.bak/
 # don't ignore build package
 !plux/build
 
+!tests/build
+plux.ini
+
 # Ignore dynamically generated version.py
-plux/version.py
+plux/version.py

plux/build/config.py

@@ -5,9 +5,13 @@
 
 import dataclasses
 import enum
+import logging
 import os
 import sys
 from importlib.util import find_spec
+from typing import Any
+
+LOG = logging.getLogger(__name__)
 
 
 class EntrypointBuildMode(enum.Enum):
@@ -24,6 +28,17 @@ class EntrypointBuildMode(enum.Enum):
     BUILD_HOOK = "build-hook"
 
 
+class BuildBackend(enum.Enum):
+    """
+    The build backend integration to use. Currently, we support setuptools and hatchling. If set to ``auto``, there
+    is an algorithm to detect the build backend automatically from the config.
+    """
+
+    AUTO = "auto"
+    SETUPTOOLS = "setuptools"
+    HATCHLING = "hatchling"
+
+
 @dataclasses.dataclass
 class PluxConfiguration:
     """
@@ -47,13 +62,17 @@ class PluxConfiguration:
     entrypoint_static_file: str = "plux.ini"
     """The name of the entrypoint ini file if entrypoint_build_mode is set to MANUAL."""
 
+    build_backend: BuildBackend = BuildBackend.AUTO
+    """The build backend to use. If set to ``auto``, the build backend will be detected automatically from the config."""
+
     def merge(
         self,
         path: str = None,
         exclude: list[str] = None,
         include: list[str] = None,
         entrypoint_build_mode: EntrypointBuildMode = None,
         entrypoint_static_file: str = None,
+        build_backend: BuildBackend = None,
     ) -> "PluxConfiguration":
         """
         Merges or overwrites the given values into the current configuration and returns a new configuration object.
@@ -69,6 +88,7 @@ def merge(
             entrypoint_static_file=entrypoint_static_file
             if entrypoint_static_file is not None
             else self.entrypoint_static_file,
+            build_backend=build_backend if build_backend is not None else self.build_backend,
         )
 
 
@@ -81,8 +101,7 @@ def read_plux_config_from_workdir(workdir: str = None) -> PluxConfiguration:
     :return: A plux configuration object
     """
     try:
-        pyproject_file = os.path.join(workdir or os.getcwd(), "pyproject.toml")
-        return parse_pyproject_toml(pyproject_file)
+        return parse_pyproject_toml(workdir or os.getcwd())
     except FileNotFoundError:
         return PluxConfiguration()
 
@@ -96,18 +115,7 @@ def parse_pyproject_toml(path: str | os.PathLike[str]) -> PluxConfiguration:
     :return: A plux configuration object containing the parsed values.
     :raises FileNotFoundError: If the file does not exist.
     """
-    if find_spec("tomllib"):
-        from tomllib import load as load_toml
-    elif find_spec("tomli"):
-        from tomli import load as load_toml
-    else:
-        raise ImportError("Could not find a TOML parser. Please install either tomllib or tomli.")
-
-    # read the file
-    if not os.path.exists(path):
-        raise FileNotFoundError(f"No pyproject.toml found at {path}")
-    with open(path, "rb") as file:
-        pyproject_config = load_toml(file)
+    pyproject_config = load_pyproject_toml(path)
 
     # find the [tool.plux] section
     tool_table = pyproject_config.get("tool", {})
@@ -127,4 +135,102 @@ def parse_pyproject_toml(path: str | os.PathLike[str]) -> PluxConfiguration:
         # will raise a ValueError exception if the mode is invalid
         kwargs["entrypoint_build_mode"] = EntrypointBuildMode(mode)
 
+    # parse build_backend
+    if build_backend := kwargs.get("build_backend"):
+        # will raise a ValueError exception if the build backend is invalid
+        kwargs["build_backend"] = BuildBackend(build_backend)
+
     return PluxConfiguration(**kwargs)
+
+
+def determine_build_backend_from_pyproject_config(pyproject_config: dict[str, Any]) -> BuildBackend | None:
+    """
+    Determine the build backend to use based on the pyproject.toml configuration.
+    """
+    build_backend = pyproject_config.get("build-system", {}).get("build-backend", "")
+    if build_backend.startswith("setuptools."):
+        return BuildBackend.SETUPTOOLS
+    if build_backend.startswith("hatchling."):
+        return BuildBackend.HATCHLING
+    else:
+        return None
+
+
+def load_pyproject_toml(pyproject_file_or_workdir: str | os.PathLike[str] = None) -> dict[str, Any]:
+    """
+    Loads a pyproject.toml file from the given path or the current working directory. Uses tomli or tomllib to parse.
+
+    :param pyproject_file_or_workdir: Path to the pyproject.toml file or the directory containing it. Defaults to the current working directory.
+    :return: The parsed pyproject.toml file as a dictionary.
+    """
+    if pyproject_file_or_workdir is None:
+        pyproject_file_or_workdir = os.getcwd()
+    if os.path.isfile(pyproject_file_or_workdir):
+        pyproject_file = pyproject_file_or_workdir
+    else:
+        pyproject_file = os.path.join(pyproject_file_or_workdir, "pyproject.toml")
+
+    if find_spec("tomllib"):
+        from tomllib import load as load_toml
+    elif find_spec("tomli"):
+        from tomli import load as load_toml
+    else:
+        raise ImportError("Could not find a TOML parser. Please install either tomllib or tomli.")
+
+    # read the file
+    if not os.path.exists(pyproject_file):
+        raise FileNotFoundError(f"No .toml file found at {pyproject_file}")
+    with open(pyproject_file, "rb") as file:
+        pyproject_config = load_toml(file)
+
+    return pyproject_config
+
+
+def determine_build_backend_from_config(workdir: str) -> BuildBackend:
+    """
+    Algorithm to determine the build backend to use based on the given workdir. First, it checks the pyproject.toml to
+    see whether there's a [tool.plux] build_backend =... is configured directly. If not found, it checks the
+    ``build-backend`` attribute in the pyproject.toml. Then, as a fallback, it tries to import both setuptools and
+    hatchling, and uses the first one that works
+    """
+    # parse config to get build backend
+    plux_config = read_plux_config_from_workdir(workdir)
+
+    if plux_config.build_backend != BuildBackend.AUTO:
+        # first, check if the user configured one
+        return plux_config.build_backend
+
+    # otherwise, try to determine it from the build-backend attribute in the pyproject.toml
+    try:
+        backend = determine_build_backend_from_pyproject_config(load_pyproject_toml(workdir))
+        if backend is not None:
+            return backend
+    except FileNotFoundError:
+        pass
+
+    # if that also fails, just try to import both build backends and return the first one that works
+    try:
+        import setuptools  # noqa
+
+        try:
+            # Try import here again to log proper warning if both are present in the environment
+            import hatchling
+
+            LOG.warning(
+                "Both setuptools and hatchling build backends available. Please manually choose a build-backend in the plux config. Defaulting to setuptools."
+            )
+        except ImportError:
+            pass
+
+        return BuildBackend.SETUPTOOLS
+    except ImportError:
+        pass
+
+    try:
+        import hatchling  # noqa
+
+        return BuildBackend.HATCHLING
+    except ImportError:
+        pass
+
+    raise ValueError("No supported build backend found. Plux needs either setuptools or hatchling to work.")

plux/build/discovery.py

@@ -5,11 +5,12 @@
 import importlib
 import inspect
 import logging
+import os
+import pkgutil
 import typing as t
 from fnmatch import fnmatchcase
+from pathlib import Path
 from types import ModuleType
-import os
-import pkgutil
 
 from plux import PluginFinder, PluginSpecResolver, PluginSpec
 
@@ -56,6 +57,107 @@ def path(self) -> str:
         raise NotImplementedError
 
 
+class SimplePackageFinder(PackageFinder):
+    """
+    A package finder that uses a heuristic to find python packages within a given path. It iterates over all
+    subdirectories in the path and returns every directory that contains a ``__init__.py`` file. It will include the
+    root package in the list of results, so if your tree looks like this::
+
+        mypkg
+        ├── __init__.py
+        ├── subpkg1
+        │   ├── __init__.py
+        │   └── nested_subpkg1
+        │       └── __init__.py
+        └── subpkg2
+            └── __init__.py
+
+    and you instantiate SimplePackageFinder("mypkg"), it will return::
+
+        [
+            "mypkg",
+            "mypkg.subpkg1",
+            "mypkg.subpkg2",
+            "mypkg.subpkg1.nested_subpkg1,
+        ]
+
+    If the root is not a package, say if you have a ``src/`` layout, and you pass "src/mypkg" as ``path`` it will omit
+    everything in the preceding path that's not a package.
+    """
+
+    DEFAULT_EXCLUDES = {"__pycache__"}
+
+    def __init__(self, path: str):
+        self._path = path
+
+    @property
+    def path(self) -> str:
+        return self._path
+
+    def find_packages(self) -> t.Iterable[str]:
+        """
+        Find all Python packages in the given path.
+
+        Returns a list of package names in the format "pkg", "pkg.subpkg", etc.
+        """
+        path = self.path
+        if not os.path.isdir(path):
+            return []
+
+        result = []
+
+        # Get the absolute path to handle relative paths correctly
+        abs_path = os.path.abspath(path)
+
+        # Check if the root directory is a package
+        root_is_package = self._looks_like_package(abs_path)
+
+        # Walk through the directory tree
+        for root, dirs, files in os.walk(abs_path):
+            # Skip directories that don't look like packages
+            if not self._looks_like_package(root):
+                continue
+
+            # Determine the base directory for relative path calculation
+            # If the root is not a package, we use the root directory itself as the base
+            # This ensures we don't include the root directory name in the package names
+            if root_is_package:
+                base_dir = os.path.dirname(abs_path)
+            else:
+                base_dir = abs_path
+
+            # Convert the path to a module name
+            rel_path = os.path.relpath(root, base_dir)
+            if rel_path == ".":
+                # If we're at the root and it's a package, use the directory name
+                rel_path = os.path.basename(abs_path)
+
+            # skip excludes TODO: should re-use Filter API
+            if os.path.basename(rel_path).strip(os.pathsep) in self.DEFAULT_EXCLUDES:
+                continue
+
+            # Skip invalid package names (those containing dots in the path)
+            if "." in os.path.basename(rel_path):
+                continue
+
+            module_name = self._path_to_module(rel_path)
+            result.append(module_name)
+
+        # Sort the results for consistent output
+        return sorted(result)
+
+    def _looks_like_package(self, path: str) -> bool:
+        return os.path.exists(os.path.join(path, "__init__.py"))
+
+    @staticmethod
+    def _path_to_module(path: str):
+        """
+        Convert a path to a Python module to its module representation
+        Example: plux/core/test -> plux.core.test
+        """
+        return ".".join(Path(path).with_suffix("").parts)
+
+
 class PluginFromPackageFinder(PluginFinder):
     """
     Finds Plugins from packages that are resolved by the given ``PackageFinder``. Under the hood this uses a