Skip to content

Commit 845a282

Browse files
jlarkin09claudecursoragent
committed
feat(sdist): add helpers to normalize git clones and tarballs into valid sdists
Git clones and non-standard tarballs (e.g. GitHub release assets) lack PKG-INFO and standardized directory naming that PEP 517 build backends expect. This causes setuptools-scm failures during the PREPARE_BUILD phase when get_requires_for_build_wheel is invoked. Add a new sdist module with make_sdist_directory() and repack_as_sdist() that normalize source directories before build dependency resolution. Bump stub PKG-INFO from Metadata-Version 1.0 to 2.2 and integrate into the git clone prepare_source path and default_build_sdist. Closes: #554 Signed-off-by: James Larkin <jlarkin@redhat.com> Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 029327a commit 845a282

3 files changed

Lines changed: 383 additions & 39 deletions

File tree

‎src/fromager/sdist.py‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
"""Helpers to normalize arbitrary source directories into valid sdist layout.
2+
3+
Git clones and non-standard tarballs (e.g. GitHub release assets) lack the
4+
``PKG-INFO`` file and standardized directory naming that PEP 517 build
5+
backends expect. These helpers bridge that gap *before* the build backend
6+
is available -- only the package name and version are required.
7+
"""
8+
9+
from __future__ import annotations
10+
11+
import logging
12+
import pathlib
13+
import shutil
14+
import tarfile
15+
16+
from packaging.version import Version
17+
18+
from . import dependencies, overrides, tarballs
19+
20+
logger = logging.getLogger(__name__)
21+
22+
PKG_INFO_TEMPLATE = """\
23+
Metadata-Version: 2.2
24+
Name: {name}
25+
Version: {version}
26+
Summary: {summary}
27+
"""
28+
29+
30+
def _write_pkg_info(
31+
directory: pathlib.Path,
32+
name: str,
33+
version: Version,
34+
) -> pathlib.Path:
35+
"""Write a stub ``PKG-INFO`` into *directory* if one does not exist.
36+
37+
Returns the path to the ``PKG-INFO`` file.
38+
"""
39+
pkg_info_file = directory / "PKG-INFO"
40+
if not pkg_info_file.is_file():
41+
logger.info("writing stub PKG-INFO in %s", directory)
42+
pkg_info_file.write_text(
43+
PKG_INFO_TEMPLATE.format(
44+
name=name,
45+
version=str(version),
46+
summary=dependencies.STUB_PKG_INFO_SUMMARY,
47+
)
48+
)
49+
return pkg_info_file
50+
51+
52+
def make_sdist_directory(
53+
source_dir: pathlib.Path,
54+
name: str,
55+
version: Version,
56+
*,
57+
build_dir: pathlib.Path | None = None,
58+
) -> pathlib.Path:
59+
"""Normalize *source_dir* into a valid sdist directory layout.
60+
61+
The directory is renamed to ``{normalized_name}-{version}`` (using
62+
:func:`~fromager.overrides.pkgname_to_override_module`) and a stub
63+
``PKG-INFO`` is written if missing. When *build_dir* differs from
64+
the source root a second ``PKG-INFO`` is placed there for
65+
``setuptools-scm`` compatibility.
66+
67+
Args:
68+
source_dir: Path to the source directory (git clone or unpacked
69+
tarball).
70+
name: Distribution name (e.g. ``req.name``).
71+
version: Package version.
72+
build_dir: Optional non-standard build directory inside
73+
*source_dir*. Receives its own ``PKG-INFO`` copy.
74+
75+
Returns:
76+
Path to the (possibly renamed) source directory.
77+
"""
78+
normalized_name = overrides.pkgname_to_override_module(name)
79+
expected_name = f"{normalized_name}-{version}"
80+
81+
if source_dir.name != expected_name:
82+
old_source_dir = source_dir
83+
desired = source_dir.parent / expected_name
84+
logger.info(
85+
"renaming source directory %s -> %s",
86+
source_dir.name,
87+
expected_name,
88+
)
89+
try:
90+
shutil.move(str(source_dir), str(desired))
91+
except Exception as err:
92+
raise RuntimeError(
93+
f"Could not rename {source_dir} to {desired}: {err}"
94+
) from err
95+
source_dir = desired
96+
97+
# Rebase build_dir so it tracks the renamed parent directory.
98+
if build_dir is not None and build_dir.is_relative_to(old_source_dir):
99+
build_dir = source_dir / build_dir.relative_to(old_source_dir)
100+
101+
_write_pkg_info(source_dir, name, version)
102+
103+
if build_dir is not None and build_dir != source_dir:
104+
_write_pkg_info(build_dir, name, version)
105+
106+
return source_dir
107+
108+
109+
def repack_as_sdist(
110+
source_dir: pathlib.Path,
111+
name: str,
112+
version: Version,
113+
output_dir: pathlib.Path,
114+
*,
115+
build_dir: pathlib.Path | None = None,
116+
) -> pathlib.Path:
117+
"""Repack *source_dir* into a standards-compliant sdist tarball.
118+
119+
Calls :func:`make_sdist_directory` to normalize the layout first,
120+
then creates a reproducible ``{name}-{version}.tar.gz`` in
121+
*output_dir*.
122+
123+
Args:
124+
source_dir: Path to the source directory.
125+
name: Distribution name.
126+
version: Package version.
127+
output_dir: Directory where the tarball is written.
128+
build_dir: Optional non-standard build subdirectory. When set
129+
the tarball is rooted at *build_dir* (matching
130+
:func:`~fromager.sources.default_build_sdist` behavior).
131+
132+
Returns:
133+
Path to the created ``.tar.gz`` file.
134+
"""
135+
old_source_dir = source_dir
136+
source_dir = make_sdist_directory(source_dir, name, version, build_dir=build_dir)
137+
138+
# Rebase build_dir after a potential rename inside make_sdist_directory.
139+
if build_dir is not None and source_dir != old_source_dir:
140+
build_dir = source_dir / build_dir.relative_to(old_source_dir)
141+
142+
tar_root = build_dir if build_dir is not None else source_dir
143+
normalized_name = overrides.pkgname_to_override_module(name)
144+
sdist_filename = output_dir / f"{normalized_name}-{version}.tar.gz"
145+
146+
if sdist_filename.exists():
147+
sdist_filename.unlink()
148+
149+
with tarfile.open(sdist_filename, "x:gz", format=tarfile.PAX_FORMAT) as tar:
150+
tarballs.tar_reproducible(
151+
tar=tar,
152+
basedir=tar_root,
153+
prefix=tar_root.parent,
154+
)
155+
156+
logger.info("created sdist archive %s", sdist_filename)
157+
return sdist_filename

‎src/fromager/sources.py‎

Lines changed: 31 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
packagesettings,
3030
pyproject,
3131
resolver,
32+
sdist,
3233
tarballs,
3334
vendor_rust,
3435
)
@@ -540,6 +541,15 @@ def prepare_source(
540541
source_root_dir=source_root_dir,
541542
version=version,
542543
)
544+
source_root_dir = sdist.make_sdist_directory(
545+
source_root_dir,
546+
req.name,
547+
version,
548+
)
549+
pbi = ctx.package_build_info(req)
550+
build_dir = pbi.build_dir(source_root_dir)
551+
if build_dir != source_root_dir:
552+
sdist._write_pkg_info(build_dir, req.name, version)
543553
else:
544554
logger.info(f"preparing source for {req} from {source_filename}")
545555
prepare_source_details = overrides.find_and_invoke(
@@ -698,31 +708,27 @@ def default_build_sdist(
698708
build_env: build_environment.BuildEnvironment,
699709
build_dir: pathlib.Path,
700710
) -> pathlib.Path:
701-
# It seems like the "correct" way to do this would be to run the
702-
# PEP 517 API in the source tree we have modified. However, quite
703-
# a few packages assume their source distribution is being built
704-
# from a source code repository checkout and those throw an error
705-
# when we use the interface to try to rebuild the sdist. Since we
706-
# know what we have is an exploded tarball, we just tar it back
707-
# up.
708-
#
709-
# For cases where the PEP 517 approach works, use
710-
# pep517_build_sdist().
711+
"""Rebuild an sdist by re-tarring a previously unpacked source tree.
712+
713+
For cases where the PEP 517 approach works, use
714+
:func:`pep517_build_sdist` instead. Many packages assume the sdist
715+
is built from a repository checkout and error out when the PEP 517
716+
interface is used, so this function simply tars the tree back up.
717+
"""
718+
sdist.make_sdist_directory(
719+
sdist_root_dir,
720+
req.name,
721+
version,
722+
build_dir=build_dir,
723+
)
711724
sdist_filename = ctx.sdists_builds / f"{req.name}-{version}.tar.gz"
712725
if sdist_filename.exists():
713726
sdist_filename.unlink()
714-
ensure_pkg_info(
715-
ctx=ctx,
716-
req=req,
717-
version=version,
718-
sdist_root_dir=sdist_root_dir,
719-
build_dir=build_dir,
720-
)
721727
# The format argument is specified based on
722728
# https://peps.python.org/pep-0517/#build-sdist.
723-
with tarfile.open(sdist_filename, "x:gz", format=tarfile.PAX_FORMAT) as sdist:
729+
with tarfile.open(sdist_filename, "x:gz", format=tarfile.PAX_FORMAT) as sdist_tar:
724730
tarballs.tar_reproducible(
725-
tar=sdist,
731+
tar=sdist_tar,
726732
basedir=build_dir,
727733
prefix=build_dir.parent,
728734
)
@@ -754,14 +760,6 @@ def pep517_build_sdist(
754760
return ctx.sdists_builds / sdist_filename
755761

756762

757-
PKG_INFO_CONTENT = """\
758-
Metadata-Version: 1.0
759-
Name: {name}
760-
Version: {version}
761-
Summary: {summary}
762-
"""
763-
764-
765763
def ensure_pkg_info(
766764
*,
767765
ctx: context.WorkContext,
@@ -772,11 +770,11 @@ def ensure_pkg_info(
772770
) -> bool:
773771
"""Ensure that sdist has a PKG-INFO file.
774772
775-
Returns True if PKG-INFO is present, False if file is missing. The
776-
function also updates build_dir if package has a non-standard build
777-
directory. Every sdist must have a PKG-INFO file in the first directory.
778-
The additional PKG-INFO file in build_dir is required for projects
779-
with a non-standard layout and projects using setuptools-scm.
773+
Delegates to :func:`fromager.sdist._write_pkg_info` to create stub
774+
files when missing.
775+
776+
Returns True if PKG-INFO was already present in all directories,
777+
False if any file had to be created.
780778
"""
781779
had_pkg_info = True
782780
directories = [sdist_root_dir]
@@ -788,13 +786,7 @@ def ensure_pkg_info(
788786
logger.warning(
789787
f"PKG-INFO file is missing from {directory}, creating stub file"
790788
)
791-
pkg_info_file.write_text(
792-
PKG_INFO_CONTENT.format(
793-
name=req.name,
794-
version=str(version),
795-
summary=dependencies.STUB_PKG_INFO_SUMMARY,
796-
)
797-
)
789+
sdist._write_pkg_info(directory, req.name, version)
798790
had_pkg_info = False
799791
return had_pkg_info
800792

0 commit comments

Comments
 (0)