Skip to content

Commit 4c2965f

Browse files
jlarkin09claudecursoragent
committed
feat(sdist): add helpers to normalize git clones and tarballs into valid sdists
Git clones and non-standard tarballs (e.g. GitHub release assets) lack PKG-INFO and standardized directory naming that PEP 517 build backends expect. This causes setuptools-scm failures during the PREPARE_BUILD phase when get_requires_for_build_wheel is invoked. Add a new sdist module with make_sdist_directory() and repack_as_sdist() that normalize source directories before build dependency resolution. Bump stub PKG-INFO from Metadata-Version 1.0 to 2.2 and integrate into the git clone prepare_source path and default_build_sdist. Closes: #554 Signed-off-by: James Larkin <jlarkin@redhat.com> Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 029327a commit 4c2965f

3 files changed

Lines changed: 350 additions & 39 deletions

File tree

‎src/fromager/sdist.py‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
"""Helpers to normalize arbitrary source directories into valid sdist layout.
2+
3+
Git clones and non-standard tarballs (e.g. GitHub release assets) lack the
4+
``PKG-INFO`` file and standardized directory naming that PEP 517 build
5+
backends expect. These helpers bridge that gap *before* the build backend
6+
is available -- only the package name and version are required.
7+
"""
8+
9+
from __future__ import annotations
10+
11+
import logging
12+
import pathlib
13+
import shutil
14+
import tarfile
15+
16+
from packaging.version import Version
17+
18+
from . import dependencies, overrides, tarballs
19+
20+
logger = logging.getLogger(__name__)
21+
22+
PKG_INFO_TEMPLATE = """\
23+
Metadata-Version: 2.2
24+
Name: {name}
25+
Version: {version}
26+
Summary: {summary}
27+
"""
28+
29+
30+
def _write_pkg_info(
31+
directory: pathlib.Path,
32+
name: str,
33+
version: Version,
34+
) -> pathlib.Path:
35+
"""Write a stub ``PKG-INFO`` into *directory* if one does not exist.
36+
37+
Returns the path to the ``PKG-INFO`` file.
38+
"""
39+
pkg_info_file = directory / "PKG-INFO"
40+
if not pkg_info_file.is_file():
41+
logger.info("writing stub PKG-INFO in %s", directory)
42+
pkg_info_file.write_text(
43+
PKG_INFO_TEMPLATE.format(
44+
name=name,
45+
version=str(version),
46+
summary=dependencies.STUB_PKG_INFO_SUMMARY,
47+
)
48+
)
49+
return pkg_info_file
50+
51+
52+
def make_sdist_directory(
53+
source_dir: pathlib.Path,
54+
name: str,
55+
version: Version,
56+
*,
57+
build_dir: pathlib.Path | None = None,
58+
) -> pathlib.Path:
59+
"""Normalize *source_dir* into a valid sdist directory layout.
60+
61+
The directory is renamed to ``{normalized_name}-{version}`` (using
62+
:func:`~fromager.overrides.pkgname_to_override_module`) and a stub
63+
``PKG-INFO`` is written if missing. When *build_dir* differs from
64+
the source root a second ``PKG-INFO`` is placed there for
65+
``setuptools-scm`` compatibility.
66+
67+
Args:
68+
source_dir: Path to the source directory (git clone or unpacked
69+
tarball).
70+
name: Distribution name (e.g. ``req.name``).
71+
version: Package version.
72+
build_dir: Optional non-standard build directory inside
73+
*source_dir*. Receives its own ``PKG-INFO`` copy.
74+
75+
Returns:
76+
Path to the (possibly renamed) source directory.
77+
"""
78+
normalized_name = overrides.pkgname_to_override_module(name)
79+
expected_name = f"{normalized_name}-{version}"
80+
81+
if source_dir.name != expected_name:
82+
desired = source_dir.parent / expected_name
83+
logger.info(
84+
"renaming source directory %s -> %s",
85+
source_dir.name,
86+
expected_name,
87+
)
88+
try:
89+
shutil.move(str(source_dir), str(desired))
90+
except Exception as err:
91+
raise RuntimeError(
92+
f"Could not rename {source_dir} to {desired}: {err}"
93+
) from err
94+
source_dir = desired
95+
96+
_write_pkg_info(source_dir, name, version)
97+
98+
if build_dir is not None and build_dir != source_dir:
99+
_write_pkg_info(build_dir, name, version)
100+
101+
return source_dir
102+
103+
104+
def repack_as_sdist(
105+
source_dir: pathlib.Path,
106+
name: str,
107+
version: Version,
108+
output_dir: pathlib.Path,
109+
*,
110+
build_dir: pathlib.Path | None = None,
111+
) -> pathlib.Path:
112+
"""Repack *source_dir* into a standards-compliant sdist tarball.
113+
114+
Calls :func:`make_sdist_directory` to normalize the layout first,
115+
then creates a reproducible ``{name}-{version}.tar.gz`` in
116+
*output_dir*.
117+
118+
Args:
119+
source_dir: Path to the source directory.
120+
name: Distribution name.
121+
version: Package version.
122+
output_dir: Directory where the tarball is written.
123+
build_dir: Optional non-standard build subdirectory. When set
124+
the tarball is rooted at *build_dir* (matching
125+
:func:`~fromager.sources.default_build_sdist` behavior).
126+
127+
Returns:
128+
Path to the created ``.tar.gz`` file.
129+
"""
130+
source_dir = make_sdist_directory(source_dir, name, version, build_dir=build_dir)
131+
132+
tar_root = build_dir if build_dir is not None else source_dir
133+
normalized_name = overrides.pkgname_to_override_module(name)
134+
sdist_filename = output_dir / f"{normalized_name}-{version}.tar.gz"
135+
136+
if sdist_filename.exists():
137+
sdist_filename.unlink()
138+
139+
with tarfile.open(sdist_filename, "x:gz", format=tarfile.PAX_FORMAT) as tar:
140+
tarballs.tar_reproducible(
141+
tar=tar,
142+
basedir=tar_root,
143+
prefix=tar_root.parent,
144+
)
145+
146+
logger.info("created sdist archive %s", sdist_filename)
147+
return sdist_filename

‎src/fromager/sources.py‎

Lines changed: 29 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
packagesettings,
3030
pyproject,
3131
resolver,
32+
sdist,
3233
tarballs,
3334
vendor_rust,
3435
)
@@ -540,6 +541,13 @@ def prepare_source(
540541
source_root_dir=source_root_dir,
541542
version=version,
542543
)
544+
pbi = ctx.package_build_info(req)
545+
source_root_dir = sdist.make_sdist_directory(
546+
source_root_dir,
547+
req.name,
548+
version,
549+
build_dir=pbi.build_dir(source_root_dir),
550+
)
543551
else:
544552
logger.info(f"preparing source for {req} from {source_filename}")
545553
prepare_source_details = overrides.find_and_invoke(
@@ -698,31 +706,27 @@ def default_build_sdist(
698706
build_env: build_environment.BuildEnvironment,
699707
build_dir: pathlib.Path,
700708
) -> pathlib.Path:
701-
# It seems like the "correct" way to do this would be to run the
702-
# PEP 517 API in the source tree we have modified. However, quite
703-
# a few packages assume their source distribution is being built
704-
# from a source code repository checkout and those throw an error
705-
# when we use the interface to try to rebuild the sdist. Since we
706-
# know what we have is an exploded tarball, we just tar it back
707-
# up.
708-
#
709-
# For cases where the PEP 517 approach works, use
710-
# pep517_build_sdist().
709+
"""Rebuild an sdist by re-tarring a previously unpacked source tree.
710+
711+
For cases where the PEP 517 approach works, use
712+
:func:`pep517_build_sdist` instead. Many packages assume the sdist
713+
is built from a repository checkout and error out when the PEP 517
714+
interface is used, so this function simply tars the tree back up.
715+
"""
716+
sdist.make_sdist_directory(
717+
sdist_root_dir,
718+
req.name,
719+
version,
720+
build_dir=build_dir,
721+
)
711722
sdist_filename = ctx.sdists_builds / f"{req.name}-{version}.tar.gz"
712723
if sdist_filename.exists():
713724
sdist_filename.unlink()
714-
ensure_pkg_info(
715-
ctx=ctx,
716-
req=req,
717-
version=version,
718-
sdist_root_dir=sdist_root_dir,
719-
build_dir=build_dir,
720-
)
721725
# The format argument is specified based on
722726
# https://peps.python.org/pep-0517/#build-sdist.
723-
with tarfile.open(sdist_filename, "x:gz", format=tarfile.PAX_FORMAT) as sdist:
727+
with tarfile.open(sdist_filename, "x:gz", format=tarfile.PAX_FORMAT) as sdist_tar:
724728
tarballs.tar_reproducible(
725-
tar=sdist,
729+
tar=sdist_tar,
726730
basedir=build_dir,
727731
prefix=build_dir.parent,
728732
)
@@ -754,14 +758,6 @@ def pep517_build_sdist(
754758
return ctx.sdists_builds / sdist_filename
755759

756760

757-
PKG_INFO_CONTENT = """\
758-
Metadata-Version: 1.0
759-
Name: {name}
760-
Version: {version}
761-
Summary: {summary}
762-
"""
763-
764-
765761
def ensure_pkg_info(
766762
*,
767763
ctx: context.WorkContext,
@@ -772,11 +768,11 @@ def ensure_pkg_info(
772768
) -> bool:
773769
"""Ensure that sdist has a PKG-INFO file.
774770
775-
Returns True if PKG-INFO is present, False if file is missing. The
776-
function also updates build_dir if package has a non-standard build
777-
directory. Every sdist must have a PKG-INFO file in the first directory.
778-
The additional PKG-INFO file in build_dir is required for projects
779-
with a non-standard layout and projects using setuptools-scm.
771+
Delegates to :func:`fromager.sdist._write_pkg_info` to create stub
772+
files when missing.
773+
774+
Returns True if PKG-INFO was already present in all directories,
775+
False if any file had to be created.
780776
"""
781777
had_pkg_info = True
782778
directories = [sdist_root_dir]
@@ -788,13 +784,7 @@ def ensure_pkg_info(
788784
logger.warning(
789785
f"PKG-INFO file is missing from {directory}, creating stub file"
790786
)
791-
pkg_info_file.write_text(
792-
PKG_INFO_CONTENT.format(
793-
name=req.name,
794-
version=str(version),
795-
summary=dependencies.STUB_PKG_INFO_SUMMARY,
796-
)
797-
)
787+
sdist._write_pkg_info(directory, req.name, version)
798788
had_pkg_info = False
799789
return had_pkg_info
800790

0 commit comments

Comments
 (0)