Make wheel builds reproducible

CHANGES/577.packaging.rst

@@ -0,0 +1,23 @@
+The :pep:`517` build backend now supports a new ``build-inplace``
+config setting (and ``FROZENLIST_BUILD_INPLACE`` environment variable)
+for controlling whether to build the project in-tree or in a
+temporary directory. It only affects wheels and is set up to build
+in a temporary directory by default. It does not affect editable
+wheel builds; they will keep being built in-tree regardless.
+
+Here's an example of using this setting:
+
+.. code-block:: console
+
+   $ python -m build --config-setting=build-inplace=true
+
+Additionally, when building wheels in an automatically created
+temporary directory, the build backend now normalizes the
+respective file system path to a deterministic source checkout
+directory by injecting the ``-ffile-prefix-map`` compiler option
+into the ``CFLAGS`` environment variable, as suggested by known
+`reproducible build practices
+<https://reproducible-builds.org/docs/build-path/>`__.
+
+The effect is that downstreams will get reproducible build results
+-- by :user:`bdraco`.

packaging/pep517_backend/_backend.py

@@ -79,6 +79,12 @@
 PURE_PYTHON_ENV_VAR = 'FROZENLIST_NO_EXTENSIONS'
 """Environment variable name toggle used to opt out of making C-exts."""
 
+BUILD_INPLACE_CONFIG_SETTING = 'build-inplace'
+"""Config setting name toggle for building C-exts in-place."""
+
+BUILD_INPLACE_ENV_VAR = 'FROZENLIST_BUILD_INPLACE'
+"""Environment variable name toggle for building C-exts in-place."""
+
 IS_CPYTHON = _system_implementation.name == "cpython"
 """A flag meaning that the current interpreter implementation is CPython."""
 
@@ -134,6 +140,19 @@ def _include_cython_line_tracing(
     )
 
 
+def _build_inplace(
+        config_settings: _ConfigDict | None = None,
+        *,
+        default: bool = False,
+) -> bool:
+    return _get_setting_value(
+        config_settings,
+        BUILD_INPLACE_CONFIG_SETTING,
+        BUILD_INPLACE_ENV_VAR,
+        default=default,
+    )
+
+
 @contextmanager
 def patched_distutils_cmd_install() -> Iterator[None]:
     """Make `install_lib` of `install` cmd always use `platlib`.
@@ -216,7 +235,7 @@ def _exclude_dir_path(
 
 
 @contextmanager
-def _in_temporary_directory(src_dir: Path) -> Iterator[None]:
+def _in_temporary_directory(src_dir: Path) -> Iterator[Path]:
     with TemporaryDirectory(prefix='.tmp-frozenlist-pep517-') as tmp_dir:
         tmp_dir_path = Path(tmp_dir)
         root_tmp_dir_path = tmp_dir_path.parent
@@ -231,7 +250,7 @@ def _in_temporary_directory(src_dir: Path) -> Iterator[None]:
                 symlinks=True,
             )
             os.chdir(tmp_src_dir)
-            yield
+            yield tmp_src_dir
 
 
 @contextmanager
@@ -284,15 +303,21 @@ def maybe_prebuild_c_extensions(
             stacklevel=999,
         )
 
+    original_src_dir = Path.cwd().resolve()
     build_dir_ctx = (
         nullcontext() if build_inplace
-        else _in_temporary_directory(src_dir=Path.cwd().resolve())
+        else _in_temporary_directory(src_dir=original_src_dir)
     )
-    with build_dir_ctx:
+    with build_dir_ctx as tmp_build_dir:
         config = _get_local_cython_config()
 
         cythonize_args = _make_cythonize_cli_args_from_config(config, cython_line_tracing_requested)
-        with _patched_cython_env(config['env'], cython_line_tracing_requested):
+        with _patched_cython_env(
+                config['env'],
+                cython_line_tracing_requested,
+                original_source_directory=original_src_dir,
+                temporary_build_directory=tmp_build_dir,
+        ):
             _cythonize_cli_cmd(cythonize_args)
         with patched_distutils_cmd_install():
             with patched_dist_has_ext_modules():
@@ -316,7 +341,7 @@ def build_wheel(
     """
     with maybe_prebuild_c_extensions(
             line_trace_cython_when_unset=False,
-            build_inplace=False,
+            build_inplace=_build_inplace(config_settings, default=False),
             config_settings=config_settings,
     ):
         return _setuptools_build_wheel(
@@ -341,9 +366,17 @@ def build_editable(
     :param metadata_directory: :file:`.dist-info` directory path.
 
     """
+    mandatory_build_inplace = True
+    if not _build_inplace(config_settings, default=mandatory_build_inplace):
+        _warn_that(
+            'Editable builds require C-extensions to be produced in-tree',
+            RuntimeWarning,
+            stacklevel=999,
+        )
+
     with maybe_prebuild_c_extensions(
             line_trace_cython_when_unset=True,
-            build_inplace=True,
+            build_inplace=mandatory_build_inplace,
             config_settings=config_settings,
     ):
         return _setuptools_build_editable(

packaging/pep517_backend/_cython_configuration.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import sys
 from collections.abc import Iterator
 from contextlib import contextmanager
 from pathlib import Path
@@ -103,23 +104,63 @@ def make_cythonize_cli_args_from_config(config: Config, cython_line_tracing_requ
 
 
 @contextmanager
-def patched_env(env: dict[str, str], cython_line_tracing_requested: bool) -> Iterator[None]:
+def patched_env(
+    env: dict[str, str],
+    cython_line_tracing_requested: bool,
+    *,
+    original_source_directory: Path | None = None,
+    temporary_build_directory: Path | None = None,
+) -> Iterator[None]:
     """Temporary set given env vars.
 
     :param env: tmp env vars to set
     :type env: dict
 
     :yields: None
     """
+    extra_cflags: list[str] = []
+    if cython_line_tracing_requested:
+        extra_cflags.append('-DCYTHON_TRACE_NOGIL=1')  # Implies CYTHON_TRACE=1
+    # When building in a temporary directory, rewrite the random tmp dir
+    # path back to the original source directory so the compiled artifacts
+    # are reproducible. `-ffile-prefix-map` is a GCC/Clang flag and is not
+    # understood by MSVC, so skip it on Windows. Validation runs before we
+    # touch ``os.environ`` so a configuration error cannot leave the process
+    # environment half-mutated.
+    # Ref: https://github.com/aio-libs/frozenlist/issues/577
+    if temporary_build_directory is not None and sys.platform != 'win32':
+        if original_source_directory is None:
+            raise ValueError(
+                'original_source_directory is required '
+                'when temporary_build_directory is set',
+            )
+        tmp_path = str(temporary_build_directory)
+        src_path = str(original_source_directory)
+        # `CFLAGS`/`CXXFLAGS` are split on whitespace by the compiler driver,
+        # so a build path containing a space would silently tokenise into
+        # multiple arguments and break the mapping. Fail loudly instead of
+        # producing a broken-but-quiet reproducibility result.
+        if ' ' in tmp_path or ' ' in src_path:
+            raise ValueError(
+                'Build paths must not contain whitespace for '
+                '`-ffile-prefix-map` to apply cleanly; got '
+                f'temporary_build_directory={tmp_path!r}, '
+                f'original_source_directory={src_path!r}',
+            )
+        extra_cflags.append(f'-ffile-prefix-map={tmp_path}={src_path}')
+
     orig_env = os.environ.copy()
     expanded_env = {name: expandvars(var_val) for name, var_val in env.items()}
     os.environ.update(expanded_env)
-
-    if cython_line_tracing_requested:
-        os.environ['CFLAGS'] = ' '.join((
-            os.getenv('CFLAGS', ''),
-            '-DCYTHON_TRACE_NOGIL=1',  # Implies CYTHON_TRACE=1
-        )).strip()
+    if extra_cflags:
+        # The Cython extension compiles as C++ (``# distutils: language = c++``),
+        # so setuptools' ``customize_compiler`` uses ``CXXFLAGS`` rather than
+        # ``CFLAGS`` for the compile step. Set both so the flags also apply
+        # when downstream forks switch the language.
+        for env_var in ('CFLAGS', 'CXXFLAGS'):
+            os.environ[env_var] = ' '.join(
+                (os.getenv(env_var, ''), *extra_cflags),
+            ).strip()
     try:
         yield
     finally:

pytest.ini

@@ -63,6 +63,11 @@ markers =
 
 minversion = 3.8.2
 
+# Make the in-tree PEP 517 backend importable for tests without a per-test
+# ``sys.path`` hack. ``testpaths = tests/`` keeps ``--doctest-modules`` from
+# collecting modules under ``packaging/``.
+pythonpath = packaging
+
 # Optimize pytest's lookup by restricting potentially deep dir tree scan:
 norecursedirs =
   build