",
+)
+ASF_DEVELOPED_NOTICE = re.compile(
+ r"This product includes software developed at\n"
+ r"The Apache Software Foundation \(https?://www\.apache\.org/\)\."
+)
+
+
+def repository_root() -> Path:
+ return Path(
+ subprocess.check_output(
+ ["git", "rev-parse", "--show-toplevel"], text=True
+ ).strip()
+ )
+
+
+def report_specs() -> list[Report]:
+ reports = [
+ Report(
+ component="python",
+ manifest="bindings/python/Cargo.toml",
+ target=target,
+ output=(f"bindings/python/licenses/{target}/THIRD-PARTY-LICENSES.html"),
+ )
+ for target in PYTHON_TARGETS
+ ]
+ reports.extend(
+ Report(
+ component="go",
+ manifest="bindings/c/Cargo.toml",
+ target=target,
+ output=f"bindings/go/{filename}",
+ )
+ for target, filename in GO_REPORTS.items()
+ )
+ return reports
+
+
+def verify_cargo_about(root: Path) -> None:
+ output = subprocess.check_output(
+ ["cargo", "about", "--version"], cwd=root, text=True
+ ).strip()
+ actual = output.rsplit(" ", 1)[-1]
+ if actual != CARGO_ABOUT_VERSION:
+ raise RuntimeError(
+ f"cargo-about {CARGO_ABOUT_VERSION} is required, found {output!r}"
+ )
+
+
+def cargo_metadata(root: Path, report: Report) -> dict:
+ output = subprocess.check_output(
+ [
+ "cargo",
+ "metadata",
+ "--locked",
+ "--format-version",
+ "1",
+ "--manifest-path",
+ report.manifest,
+ "--filter-platform",
+ report.target,
+ ],
+ cwd=root,
+ text=True,
+ )
+ metadata = json.loads(output)
+ tree = subprocess.check_output(
+ [
+ "cargo",
+ "tree",
+ "--locked",
+ "--manifest-path",
+ report.manifest,
+ "--target",
+ report.target,
+ "--edges",
+ "normal",
+ "--prefix",
+ "none",
+ "--format",
+ "{p}",
+ ],
+ cwd=root,
+ text=True,
+ )
+ packages = set()
+ for line in tree.splitlines():
+ match = re.match(r"^([A-Za-z0-9_-]+) v([^ ]+)", line)
+ if match is None:
+ raise RuntimeError(f"unexpected cargo tree package line: {line!r}")
+ packages.add(match.groups())
+ metadata["normal_packages"] = packages
+ return metadata
+
+
+def generate_base_report(root: Path, report: Report, output: Path) -> str:
+ subprocess.run(
+ [
+ "cargo",
+ "about",
+ "generate",
+ "--frozen",
+ "--fail",
+ "--config",
+ str(root / "about.toml"),
+ "--manifest-path",
+ report.manifest,
+ "--target",
+ report.target,
+ "--output-file",
+ str(output),
+ str(root / "about.hbs"),
+ ],
+ cwd=root,
+ check=True,
+ )
+ return output.read_text(encoding="utf-8")
+
+
+def resolved_packages(metadata: dict) -> list[dict]:
+ normal_packages = metadata["normal_packages"]
+ return [
+ package
+ for package in metadata["packages"]
+ if (package["name"], package["version"]) in normal_packages
+ ]
+
+
+def package_by_name(
+ metadata: dict, crate_name: str, required: bool = True
+) -> dict | None:
+ matches = [
+ package
+ for package in resolved_packages(metadata)
+ if package["name"] == crate_name
+ ]
+ if not matches and not required:
+ return None
+ if len(matches) != 1:
+ versions = [package["version"] for package in matches]
+ raise RuntimeError(
+ f"expected exactly one resolved {crate_name} package, found {versions}"
+ )
+ return matches[0]
+
+
+def package_features(metadata: dict, package: dict) -> set[str]:
+ matches = [
+ node["features"]
+ for node in metadata["resolve"]["nodes"]
+ if node["id"] == package["id"]
+ ]
+ if len(matches) != 1:
+ raise RuntimeError(f"could not resolve features for {package['name']}")
+ return set(matches[0])
+
+
+def correction_html(metadata: dict, correction: LicenseCorrection) -> str:
+ used_by = []
+ for crate_name in correction.crates:
+ package = package_by_name(metadata, crate_name)
+ repository = package.get("repository") or (
+ f"https://crates.io/crates/{crate_name}"
+ )
+ used_by.append(
+ f' '
+ f"{html.escape(crate_name)} {html.escape(package['version'])}"
+ )
+
+ license_package = package_by_name(metadata, correction.license_crate)
+ license_file = (
+ Path(license_package["manifest_path"]).parent / correction.license_path
+ )
+ if not license_file.is_file():
+ raise RuntimeError(f"corrected license file is missing: {license_file}")
+ license_text = license_file.read_text(encoding="utf-8")
+
+ return "\n".join(
+ [
+ ' ',
+ f' '
+ f"{html.escape(correction.license_name)}
",
+ " Used by
",
+ ' ",
+ f' {html.escape(license_text)}',
+ " ",
+ ]
+ )
+
+
+def replace_placeholder_entry(
+ report_text: str,
+ metadata: dict,
+ marker: str,
+ corrections: tuple[LicenseCorrection, ...],
+) -> str:
+ if report_text.count(marker) != 1:
+ raise RuntimeError(
+ f"expected exactly one license placeholder {marker!r}, "
+ f"found {report_text.count(marker)}"
+ )
+ marker_index = report_text.index(marker)
+ entry_start = report_text.rfind(' ', 0, marker_index)
+ entry_end = report_text.find(" ", marker_index)
+ if entry_start == -1 or entry_end == -1:
+ raise RuntimeError(f"could not locate license entry for {marker!r}")
+ entry_end += len(" ")
+ placeholder_entry = report_text[entry_start:entry_end]
+
+ actual_crates = set(
+ re.findall(
+ r'([A-Za-z0-9_-]+) [^<]+',
+ placeholder_entry,
+ )
+ )
+ expected_crates = {
+ crate_name for correction in corrections for crate_name in correction.crates
+ }
+ if actual_crates != expected_crates:
+ raise RuntimeError(
+ f"placeholder dependency set changed for {marker!r}: expected "
+ f"{sorted(expected_crates)}, found {sorted(actual_crates)}"
+ )
+
+ replacement = "\n".join(
+ correction_html(metadata, correction) for correction in corrections
+ )
+ return report_text[:entry_start] + replacement + report_text[entry_end:]
+
+
+def bundled_component_html(
+ root: Path, report: Report, report_text: str, metadata: dict
+) -> str:
+ items = []
+ for component in BUNDLED_COMPONENTS:
+ if component.components and report.component not in component.components:
+ continue
+ if component.targets and report.target not in component.targets:
+ continue
+ package = package_by_name(metadata, component.crate, required=False)
+ if package is None:
+ if component.required:
+ raise RuntimeError(
+ f"required bundled component crate is missing: {component.crate}"
+ )
+ continue
+ if component.crate_version and package["version"] != component.crate_version:
+ raise RuntimeError(
+ f"expected {component.crate} {component.crate_version}, "
+ f"found {package['version']}"
+ )
+ missing_features = set(component.required_features) - package_features(
+ metadata, package
+ )
+ if missing_features:
+ raise RuntimeError(
+ f"{component.crate} is missing required features: "
+ f"{sorted(missing_features)}"
+ )
+ marker = f">{component.crate} {package['version']}"
+ if marker not in report_text:
+ if component.required:
+ raise RuntimeError(
+ f"required bundled component is absent from report: {component.crate}"
+ )
+ # Cargo metadata may include workspace-unified features.
+ continue
+ if component.license_from_repository:
+ license_file = root / component.license_path
+ else:
+ license_file = (
+ Path(package["manifest_path"]).parent / component.license_path
+ )
+ if not license_file.is_file():
+ raise RuntimeError(f"bundled license file is missing: {license_file}")
+ license_text = license_file.read_text(encoding="utf-8")
+ repository = package.get("repository") or (
+ f"https://crates.io/crates/{component.crate}"
+ )
+ items.append(
+ "\n".join(
+ [
+ ' ',
+ f' '
+ f"{html.escape(component.license_name)}
",
+ " Bundled component
",
+ ' ",
+ f' {html.escape(license_text)}',
+ " ",
+ ]
+ )
+ )
+
+ if not items:
+ return ""
+ return "\n".join(
+ [
+ "",
+ " Licenses for bundled native and source components
",
+ " ",
+ " These components ship with the native artifact but have",
+ " license files outside their Rust package metadata.",
+ "
",
+ ' ",
+ ]
+ )
+
+
+def complete_report(
+ root: Path, base_report: str, report: Report, metadata: dict
+) -> str:
+ result = replace_placeholder_entry(
+ base_report, metadata, ALLOC_PLACEHOLDER, ALLOC_CORRECTIONS
+ )
+ mit_corrections = COMMON_MIT_CORRECTIONS
+ if report.component == "python":
+ mit_corrections += PYTHON_MIT_CORRECTIONS
+ result = replace_placeholder_entry(
+ result, metadata, MIT_PLACEHOLDER, mit_corrections
+ )
+
+ for placeholder in LICENSE_PLACEHOLDERS:
+ if placeholder in result:
+ raise RuntimeError(
+ f"generated report still contains license placeholder {placeholder!r}"
+ )
+
+ description = (
+ "\n Artifact: "
+ f"{html.escape(report.component)}
"
+ "\n Rust target: "
+ f"{html.escape(report.target)}
"
+ "\n Root manifest: "
+ f"{html.escape(report.manifest)}
"
+ )
+ first_paragraph_end = result.find("")
+ if first_paragraph_end == -1:
+ raise RuntimeError("about.hbs output has no introductory paragraph")
+ first_paragraph_end += len("")
+ result = result[:first_paragraph_end] + description + result[first_paragraph_end:]
+
+ closing_main = result.rfind(" ")
+ if closing_main == -1:
+ raise RuntimeError("about.hbs output has no closing main element")
+ result = (
+ result[:closing_main]
+ + bundled_component_html(root, report, result, metadata)
+ + "\n"
+ + result[closing_main:]
+ )
+ return "\n".join(line.rstrip() for line in result.rstrip().splitlines()) + "\n"
+
+
+def binary_license(apache_license: str, heading: str, reports: list[str]) -> str:
+ appendix = [
+ "",
+ "=" * 79,
+ "BUNDLED THIRD-PARTY COMPONENTS",
+ "=" * 79,
+ "",
+ heading,
+ "The component inventory, copyright notices, and complete license texts",
+ "are provided in:",
+ "",
+ ]
+ appendix.extend(f" {report}" for report in reports)
+ return apache_license.rstrip() + "\n" + "\n".join(appendix) + "\n"
+
+
+def notice_paragraphs(text: str) -> list[str]:
+ normalized = "\n".join(line.rstrip() for line in text.strip().splitlines())
+ return [part.strip() for part in re.split(r"\n\s*\n", normalized) if part.strip()]
+
+
+def binary_notice(root: Path, metadata_by_target: list[dict]) -> str:
+ paragraphs = notice_paragraphs((root / "NOTICE").read_text(encoding="utf-8"))
+ seen_paragraphs = set(paragraphs)
+ seen_packages = set()
+ for metadata in metadata_by_target:
+ for package in resolved_packages(metadata):
+ package_key = (
+ package["name"],
+ package["version"],
+ package["manifest_path"],
+ )
+ if package_key in seen_packages:
+ continue
+ seen_packages.add(package_key)
+ package_dir = Path(package["manifest_path"]).parent
+ notice_files = sorted(
+ path
+ for path in package_dir.iterdir()
+ if path.is_file()
+ and path.name.lower() in {"notice", "notice.txt", "notice.md"}
+ )
+ for notice_file in notice_files:
+ notice_text = notice_file.read_text(encoding="utf-8")
+ for paragraph in notice_paragraphs(notice_text):
+ if ASF_DEVELOPED_NOTICE.fullmatch(paragraph):
+ continue
+ if paragraph in seen_paragraphs:
+ continue
+ seen_paragraphs.add(paragraph)
+ paragraphs.append(paragraph)
+
+ return "\n\n".join(paragraphs) + "\n"
+
+
+def verify_python_source_legal_files(root: Path) -> None:
+ for name in ("LICENSE", "NOTICE"):
+ source = root / name
+ copy = root / "bindings/python" / name
+ if source.read_bytes() != copy.read_bytes():
+ raise RuntimeError(f"restore bindings/python/{name} before generation")
+ source_report = root / "bindings/python/THIRD-PARTY-LICENSES.html"
+ report_text = source_report.read_text(encoding="utf-8")
+ for marker in (
+ "Python source distribution licenses",
+ "contains no compiled native library",
+ ):
+ if marker not in report_text:
+ raise RuntimeError(f"Python source license report is missing {marker!r}")
+
+
+def generate_reports(
+ root: Path, reports: list[Report]
+) -> tuple[dict[Report, str], dict[Report, dict]]:
+ verify_cargo_about(root)
+ result = {}
+ metadata_by_report = {}
+ with tempfile.TemporaryDirectory(prefix="paimon-rust-license-reports-") as temp:
+ temp_root = Path(temp)
+ for index, report in enumerate(reports):
+ print(f"generating {report.component} license report for {report.target}")
+ base = generate_base_report(
+ root, report, temp_root / f"report-{index}.html"
+ )
+ metadata = cargo_metadata(root, report)
+ metadata_by_report[report] = metadata
+ result[report] = complete_report(root, base, report, metadata)
+ return result, metadata_by_report
+
+
+def python_binary_files(
+ root: Path, target: str, report: str, metadata: dict
+) -> dict[str, str]:
+ apache_license = (root / "LICENSE").read_text(encoding="utf-8")
+ return {
+ "LICENSE": binary_license(
+ apache_license,
+ f"This binary wheel bundles the Rust native library for {target}.",
+ ["THIRD-PARTY-LICENSES.html"],
+ ),
+ "NOTICE": binary_notice(root, [metadata]),
+ "THIRD-PARTY-LICENSES.html": report,
+ }
+
+
+def python_target_files(root: Path, target: str, output_dir: Path) -> dict[Path, str]:
+ if target not in PYTHON_TARGETS:
+ raise ValueError(f"unsupported Python release target: {target}")
+ verify_python_source_legal_files(root)
+ report = next(
+ item
+ for item in report_specs()
+ if item.component == "python" and item.target == target
+ )
+ reports, metadata = generate_reports(root, [report])
+ return {
+ output_dir / name: content
+ for name, content in python_binary_files(
+ root, target, reports[report], metadata[report]
+ ).items()
+ }
+
+
+def python_verification_files(root: Path, output_dir: Path) -> dict[Path, str]:
+ verify_python_source_legal_files(root)
+ reports = [report for report in report_specs() if report.component == "python"]
+ generated, metadata = generate_reports(root, reports)
+ result = {}
+ for report in reports:
+ target_dir = output_dir / report.target
+ result.update(
+ {
+ target_dir / name: content
+ for name, content in python_binary_files(
+ root,
+ report.target,
+ generated[report],
+ metadata[report],
+ ).items()
+ }
+ )
+ return result
+
+
+def go_target_files(root: Path, target: str, output_dir: Path) -> dict[Path, str]:
+ if target not in GO_REPORTS:
+ raise ValueError(f"unsupported Go release target: {target}")
+ report = next(
+ item
+ for item in report_specs()
+ if item.component == "go" and item.target == target
+ )
+ generated, _ = generate_reports(root, [report])
+ return {output_dir / GO_REPORTS[target]: generated[report]}
+
+
+def go_release_files(root: Path, output_dir: Path) -> dict[Path, str]:
+ reports = [report for report in report_specs() if report.component == "go"]
+ generated, metadata = generate_reports(root, reports)
+ apache_license = (root / "LICENSE").read_text(encoding="utf-8")
+ result = {
+ output_dir / GO_REPORTS[report.target]: generated[report] for report in reports
+ }
+ result[output_dir / "LICENSE"] = binary_license(
+ apache_license,
+ "This Go module bundles Rust native libraries for four release targets.",
+ list(GO_REPORTS.values()),
+ )
+ result[output_dir / "NOTICE"] = binary_notice(
+ root, [metadata[report] for report in reports]
+ )
+ return result
+
+
+def generated_files(root: Path) -> dict[Path, str]:
+ verify_python_source_legal_files(root)
+ reports = report_specs()
+ generated, metadata = generate_reports(root, reports)
+ result = {root / report.output: generated[report] for report in reports}
+
+ apache_license = (root / "LICENSE").read_text(encoding="utf-8")
+ python_reports = [report for report in reports if report.component == "python"]
+ for report in python_reports:
+ license_dir = root / "bindings/python/licenses" / report.target
+ files = python_binary_files(
+ root, report.target, generated[report], metadata[report]
+ )
+ for name in ("LICENSE", "NOTICE"):
+ result[license_dir / name] = files[name]
+
+ go_reports = list(GO_REPORTS.values())
+ result[root / "bindings/go/LICENSE"] = binary_license(
+ apache_license,
+ "This Go module bundles Rust native libraries for four release targets.",
+ go_reports,
+ )
+ result[root / "bindings/go/NOTICE"] = binary_notice(
+ root,
+ [
+ metadata[report]
+ for report in reports
+ if report.component == "go"
+ ],
+ )
+ return result
+
+
+def check_generated_files(files: dict[Path, str], root: Path) -> None:
+ relative_paths = [str(path.relative_to(root)) for path in files]
+ tracked = subprocess.check_output(
+ ["git", "ls-files", "--", *relative_paths], cwd=root, text=True
+ ).splitlines()
+ if tracked:
+ raise RuntimeError(
+ "generated binary legal files must not be tracked: "
+ + ", ".join(tracked)
+ )
+
+ python_paths = [
+ relative
+ for relative in relative_paths
+ if relative.startswith("bindings/python/licenses/")
+ ]
+ for relative in python_paths:
+ attribute = subprocess.check_output(
+ ["git", "check-attr", "export-ignore", "--", relative],
+ cwd=root,
+ text=True,
+ ).strip()
+ if not attribute.endswith(": set"):
+ raise RuntimeError(f"generated binary legal file is not export-ignored: {relative}")
+
+ print(f"validated {len(files)} generated binary legal files")
+
+
+def write_files(files: dict[Path, str], root: Path) -> None:
+ for path, content in files.items():
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(content.encode("utf-8"))
+ display_path = path.relative_to(root) if path.is_relative_to(root) else path
+ print(f"generated {display_path}")
+
+
+def verify_existing_go_reports(files: dict[Path, str]) -> None:
+ report_names = set(GO_REPORTS.values())
+ for path, expected in files.items():
+ if path.name not in report_names:
+ continue
+ if not path.is_file():
+ raise RuntimeError(f"missing generated Go report: {path}")
+ if path.read_bytes() != expected.encode("utf-8"):
+ raise RuntimeError(f"generated Go report differs from release input: {path}")
+
+
+def stage_python(root: Path, target: str) -> None:
+ write_files(python_target_files(root, target, root / "bindings/python"), root)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument(
+ "--check",
+ action="store_true",
+ help="validate reproducible binary legal output without writing it",
+ )
+ group.add_argument(
+ "--stage-python",
+ metavar="TARGET",
+ help="stage one target's legal files for a maturin wheel build",
+ )
+ group.add_argument(
+ "--generate-python-reports",
+ metavar="DIRECTORY",
+ help="generate all target legal files for final wheel verification",
+ )
+ group.add_argument(
+ "--generate-go-report",
+ nargs=2,
+ metavar=("TARGET", "DIRECTORY"),
+ help="generate one target report for a Go build job",
+ )
+ group.add_argument(
+ "--generate-go-release",
+ metavar="DIRECTORY",
+ help="verify Go target reports and generate aggregate legal files",
+ )
+ args = parser.parse_args()
+ root = repository_root()
+
+ try:
+ if args.stage_python:
+ stage_python(root, args.stage_python)
+ return 0
+ if args.generate_python_reports:
+ files = python_verification_files(
+ root, Path(args.generate_python_reports).resolve()
+ )
+ write_files(files, root)
+ return 0
+ if args.generate_go_report:
+ target, directory = args.generate_go_report
+ files = go_target_files(root, target, Path(directory).resolve())
+ write_files(files, root)
+ return 0
+ if args.generate_go_release:
+ files = go_release_files(root, Path(args.generate_go_release).resolve())
+ verify_existing_go_reports(files)
+ write_files(
+ {
+ path: content
+ for path, content in files.items()
+ if path.name in {"LICENSE", "NOTICE"}
+ },
+ root,
+ )
+ return 0
+ files = generated_files(root)
+ except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as error:
+ print(f"release license generation failed: {error}", file=sys.stderr)
+ return 1
+
+ if args.check:
+ try:
+ check_generated_files(files, root)
+ except (OSError, RuntimeError, subprocess.CalledProcessError) as error:
+ print(f"release license generation failed: {error}", file=sys.stderr)
+ return 1
+ return 0
+ write_files(files, root)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/test_release_licenses.py b/scripts/tests/test_release_licenses.py
new file mode 100644
index 00000000..1742030e
--- /dev/null
+++ b/scripts/tests/test_release_licenses.py
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import tempfile
+import unittest
+from pathlib import Path
+
+from scripts.release_licenses import binary_notice, resolved_packages
+
+
+class ResolvedPackagesTest(unittest.TestCase):
+ def test_uses_normal_cargo_tree_packages(self) -> None:
+ metadata = {
+ "normal_packages": {("root", "1.0.0"), ("required", "2.0.0")},
+ "packages": [
+ {"name": "root", "version": "1.0.0"},
+ {"name": "required", "version": "2.0.0"},
+ {"name": "optional", "version": "3.0.0"},
+ ],
+ }
+ self.assertEqual(
+ [package["name"] for package in resolved_packages(metadata)],
+ ["root", "required"],
+ )
+
+ def test_notice_uses_only_normal_dependencies(self) -> None:
+ with tempfile.TemporaryDirectory() as temp:
+ root = Path(temp)
+ (root / "NOTICE").write_text("Root notice\n", encoding="utf-8")
+ packages = []
+ for name, notice in (
+ ("required", "Required dependency notice\n"),
+ ("optional", "Optional dependency notice\n"),
+ ):
+ package_dir = root / name
+ package_dir.mkdir()
+ manifest = package_dir / "Cargo.toml"
+ manifest.write_text("[package]\n", encoding="utf-8")
+ (package_dir / "NOTICE").write_text(notice, encoding="utf-8")
+ packages.append(
+ {
+ "name": name,
+ "version": "1.0.0",
+ "manifest_path": str(manifest),
+ }
+ )
+
+ metadata = {
+ "normal_packages": {("required", "1.0.0")},
+ "packages": packages,
+ }
+ notice = binary_notice(root, [metadata])
+ self.assertIn("Required dependency notice", notice)
+ self.assertNotIn("Optional dependency notice", notice)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/tests/test_verify_python_wheels.py b/scripts/tests/test_verify_python_wheels.py
new file mode 100644
index 00000000..3b47cbe7
--- /dev/null
+++ b/scripts/tests/test_verify_python_wheels.py
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import tarfile
+import unittest
+
+from scripts.verify_python_wheels import (
+ VerificationError,
+ index_tar_members,
+ normalized_relative_path,
+)
+
+
+class ArchivePathTest(unittest.TestCase):
+ def test_rejects_non_canonical_paths(self) -> None:
+ for value in ("root/./LICENSE", "root//LICENSE", "C:/LICENSE"):
+ with self.subTest(value=value):
+ with self.assertRaises(VerificationError):
+ normalized_relative_path(value, "archive member")
+
+ def test_rejects_tar_links(self) -> None:
+ member = tarfile.TarInfo("root/LICENSE")
+ member.type = tarfile.SYMTYPE
+ member.linkname = "NOTICE"
+ with self.assertRaises(VerificationError):
+ index_tar_members([member], "sdist")
+
+ def test_rejects_duplicate_tar_members(self) -> None:
+ first = tarfile.TarInfo("root/LICENSE")
+ second = tarfile.TarInfo("root/LICENSE")
+ with self.assertRaises(VerificationError):
+ index_tar_members([first, second], "sdist")
+
+ def test_rejects_portable_tar_collisions(self) -> None:
+ upper = tarfile.TarInfo("root/LICENSE")
+ lower = tarfile.TarInfo("root/license")
+ with self.assertRaises(VerificationError):
+ index_tar_members([upper, lower], "sdist")
+
+ def test_indexes_regular_tar_members(self) -> None:
+ license_member = tarfile.TarInfo("root/LICENSE")
+ notice_member = tarfile.TarInfo("root/NOTICE")
+ indexed = index_tar_members([license_member, notice_member], "sdist")
+ self.assertEqual(set(indexed), {"root/LICENSE", "root/NOTICE"})
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/verify_go_release_artifacts.py b/scripts/verify_go_release_artifacts.py
new file mode 100644
index 00000000..fc8c8e13
--- /dev/null
+++ b/scripts/verify_go_release_artifacts.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Verify Go native release artifacts and their target-specific licenses."""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Artifact:
+ library: str
+ report: str
+ format_markers: tuple[str, ...]
+ architecture_markers: tuple[str, ...]
+
+
+ARTIFACTS = {
+ "x86_64-unknown-linux-gnu": Artifact(
+ library="libpaimon_c.linux.amd64.so.zst",
+ report="THIRD-PARTY-LICENSES.linux.amd64.html",
+ format_markers=("elf 64-bit", "shared object"),
+ architecture_markers=("x86-64", "x86_64"),
+ ),
+ "aarch64-unknown-linux-gnu": Artifact(
+ library="libpaimon_c.linux.arm64.so.zst",
+ report="THIRD-PARTY-LICENSES.linux.arm64.html",
+ format_markers=("elf 64-bit", "shared object"),
+ architecture_markers=("aarch64", "arm64"),
+ ),
+ "x86_64-apple-darwin": Artifact(
+ library="libpaimon_c.darwin.amd64.dylib.zst",
+ report="THIRD-PARTY-LICENSES.darwin.amd64.html",
+ format_markers=("mach-o 64-bit", "dynamically linked shared library"),
+ architecture_markers=("x86_64", "x86-64"),
+ ),
+ "aarch64-apple-darwin": Artifact(
+ library="libpaimon_c.darwin.arm64.dylib.zst",
+ report="THIRD-PARTY-LICENSES.darwin.arm64.html",
+ format_markers=("mach-o 64-bit", "dynamically linked shared library"),
+ architecture_markers=("arm64", "aarch64"),
+ ),
+}
+
+LICENSE_PLACEHOLDERS = (
+ "<year>",
+ "<owner>",
+ "<copyright holders>",
+ "",
+ "",
+ "",
+)
+AVRO_NOTICE_MARKERS = (
+ "Apache Avro",
+ "Flavien Raynaud",
+ "donated to the Apache Avro project in 2020.",
+)
+
+
+def repository_root() -> Path:
+ return Path(__file__).resolve().parent.parent
+
+
+def require_file(path: Path) -> None:
+ if not path.is_file() or path.stat().st_size == 0:
+ raise ValueError(f"missing or empty release file: {path}")
+
+
+def verify_avro_notice(content: str, description: str) -> None:
+ for marker in AVRO_NOTICE_MARKERS:
+ if marker not in content:
+ raise ValueError(f"{description} is missing {marker!r}")
+
+
+def verify_report(report: Path, target: str) -> None:
+ require_file(report)
+ report_text = report.read_text(encoding="utf-8")
+ required_markers = (
+ "Artifact: go",
+ f"Rust target: {target}",
+ "Root manifest: bindings/c/Cargo.toml",
+ )
+ for marker in required_markers:
+ if marker not in report_text:
+ raise ValueError(f"{report} is missing {marker!r}")
+ for placeholder in LICENSE_PLACEHOLDERS:
+ if placeholder in report_text:
+ raise ValueError(f"{report} contains placeholder {placeholder!r}")
+
+
+def verify_library(library: Path, artifact: Artifact) -> None:
+ require_file(library)
+ with tempfile.TemporaryDirectory(prefix="paimon-go-release-") as temp_dir:
+ unpacked = Path(temp_dir) / library.name.removesuffix(".zst")
+ with unpacked.open("wb") as destination:
+ result = subprocess.run(
+ ["zstd", "-d", "-q", "-c", str(library)],
+ stdout=destination,
+ stderr=subprocess.PIPE,
+ text=True,
+ check=False,
+ )
+ if result.returncode != 0:
+ raise ValueError(f"cannot decompress {library}: {result.stderr.strip()}")
+ require_file(unpacked)
+ description = subprocess.check_output(
+ ["file", "-b", str(unpacked)], text=True
+ ).strip()
+
+ normalized = description.lower()
+ if any(marker not in normalized for marker in artifact.format_markers):
+ raise ValueError(f"unexpected binary format for {library}: {description}")
+ if not any(marker in normalized for marker in artifact.architecture_markers):
+ raise ValueError(f"unexpected binary architecture for {library}: {description}")
+
+
+def verify_target(root: Path, artifacts_dir: Path, target: str) -> None:
+ artifact = ARTIFACTS[target]
+ library = artifacts_dir / artifact.library
+ report = artifacts_dir / artifact.report
+ verify_library(library, artifact)
+ verify_report(report, target)
+ print(f"verified {artifact.library}: {target}")
+
+
+def verify_release_legal_files(root: Path, artifacts_dir: Path) -> None:
+ license_path = artifacts_dir / "LICENSE"
+ require_file(license_path)
+ license_content = license_path.read_text(encoding="utf-8")
+ apache_license = (root / "LICENSE").read_text(encoding="utf-8").rstrip()
+ if not license_content.startswith(apache_license + "\n"):
+ raise ValueError("binary LICENSE does not begin with the Apache License")
+ for artifact in ARTIFACTS.values():
+ if artifact.report not in license_content:
+ raise ValueError(f"binary LICENSE is missing {artifact.report}")
+
+ notice_path = artifacts_dir / "NOTICE"
+ require_file(notice_path)
+ notice = notice_path.read_text(encoding="utf-8")
+ verify_avro_notice(notice, "NOTICE")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "--artifacts-dir",
+ required=True,
+ type=Path,
+ help="directory containing downloaded or locally built release assets",
+ )
+ parser.add_argument(
+ "--target",
+ choices=tuple(ARTIFACTS),
+ help="verify one matrix target; omit to verify the complete release set",
+ )
+ args = parser.parse_args()
+
+ root = repository_root()
+ artifacts_dir = args.artifacts_dir.resolve()
+ try:
+ if args.target:
+ verify_target(root, artifacts_dir, args.target)
+ else:
+ verify_release_legal_files(root, artifacts_dir)
+ for target in ARTIFACTS:
+ verify_target(root, artifacts_dir, target)
+ except (OSError, UnicodeError, ValueError, subprocess.SubprocessError) as error:
+ print(f"Go release artifact verification failed: {error}", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/verify_python_wheels.py b/scripts/verify_python_wheels.py
new file mode 100644
index 00000000..a01242f3
--- /dev/null
+++ b/scripts/verify_python_wheels.py
@@ -0,0 +1,654 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Verify Python release archives and their legal metadata."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import csv
+import hashlib
+import io
+import re
+import stat
+import sys
+import tarfile
+import tomllib
+import unicodedata
+import zipfile
+from email import policy
+from email.message import Message
+from email.parser import BytesParser
+from pathlib import Path, PurePosixPath
+
+
+ROOT = Path(__file__).resolve().parents[1]
+PYTHON_DIR = ROOT / "bindings/python"
+LEGAL_FILES = ("LICENSE", "NOTICE", "THIRD-PARTY-LICENSES.html")
+TARGETS = {
+ "x86_64-unknown-linux-gnu",
+ "aarch64-unknown-linux-gnu",
+ "x86_64-apple-darwin",
+ "aarch64-apple-darwin",
+ "x86_64-pc-windows-msvc",
+}
+PLACEHOLDERS = (
+ b"<year>",
+ b"<owner>",
+ b"<copyright holders>",
+ b"",
+ b"",
+ b"",
+)
+AVRO_NOTICE_MARKERS = (
+ b"Apache Avro",
+ b"Flavien Raynaud",
+ b"donated to the Apache Avro project in 2020.",
+)
+XZ_LICENSE_ANCHOR = b'id="bundled-xz-utils-0bsd"'
+
+
+class VerificationError(RuntimeError):
+ """A release archive failed verification."""
+
+
+def require(condition: bool, message: str) -> None:
+ if not condition:
+ raise VerificationError(message)
+
+
+def normalized_relative_path(
+ value: str, description: str, allow_trailing_slash: bool = False
+) -> PurePosixPath:
+ require(bool(value), f"empty {description}")
+ require("\\" not in value, f"invalid {description}: {value!r}")
+ require(
+ re.match(r"^[A-Za-z]:", value) is None,
+ f"Windows drive {description}: {value!r}",
+ )
+ candidate = value[:-1] if allow_trailing_slash and value.endswith("/") else value
+ path = PurePosixPath(candidate)
+ require(not path.is_absolute(), f"absolute {description}: {value!r}")
+ require(".." not in path.parts, f"unsafe {description}: {value!r}")
+ canonical = path.as_posix()
+ require(
+ value == canonical
+ or (allow_trailing_slash and value == f"{canonical}/"),
+ f"non-canonical {description}: {value!r}",
+ )
+ return path
+
+
+def portable_path_key(path: PurePosixPath) -> str:
+ return unicodedata.normalize("NFC", path.as_posix()).casefold()
+
+
+def index_tar_members(
+ members: list[tarfile.TarInfo], description: str
+) -> dict[str, tarfile.TarInfo]:
+ result = {}
+ portable_names = set()
+ for member in members:
+ require(
+ member.isfile() or member.isdir(),
+ f"{description} has a link or special member: {member.name}",
+ )
+ normalized = normalized_relative_path(
+ member.name,
+ f"{description} member",
+ allow_trailing_slash=member.isdir(),
+ )
+ name = normalized.as_posix()
+ require(
+ name not in result,
+ f"{description} has duplicate normalized member: {name}",
+ )
+ portable_name = portable_path_key(normalized)
+ require(
+ portable_name not in portable_names,
+ f"{description} has a portable path collision: {name}",
+ )
+ result[name] = member
+ portable_names.add(portable_name)
+ return result
+
+
+def verify_record(
+ archive: zipfile.ZipFile,
+ names: list[str],
+ record_member: str,
+ description: str,
+) -> None:
+ try:
+ content = archive.read(record_member).decode("utf-8")
+ rows = list(csv.reader(io.StringIO(content, newline=""), strict=True))
+ except (UnicodeDecodeError, csv.Error) as error:
+ raise VerificationError(f"invalid RECORD in {description}: {error}") from error
+
+ entries = {}
+ for row in rows:
+ require(len(row) == 3, f"invalid RECORD row in {description}: {row}")
+ name, digest, size = row
+ normalized_relative_path(name, "RECORD path")
+ require(name not in entries, f"duplicate RECORD path in {description}: {name}")
+ entries[name] = (digest, size)
+
+ require(
+ set(entries) == set(names),
+ f"RECORD members differ in {description}",
+ )
+ record_dir = PurePosixPath(record_member).parent
+ unsigned = {
+ record_member,
+ str(record_dir / "RECORD.jws"),
+ str(record_dir / "RECORD.p7s"),
+ }
+ for name in names:
+ digest, size = entries[name]
+ if name in unsigned:
+ require(not digest and not size, f"signed RECORD row in {description}: {name}")
+ continue
+ data = archive.read(name)
+ encoded = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=")
+ require(
+ digest == f"sha256={encoded.decode()}",
+ f"RECORD hash differs in {description}: {name}",
+ )
+ require(
+ size.isdecimal() and int(size) == len(data),
+ f"RECORD size differs in {description}: {name}",
+ )
+
+
+def verify_legal_bytes(content: bytes, description: str) -> None:
+ require(b"\r" not in content, f"{description} contains CRLF line endings")
+ lowered = content.lower()
+ for placeholder in PLACEHOLDERS:
+ require(
+ placeholder not in lowered,
+ f"{description} contains license placeholder {placeholder!r}",
+ )
+
+
+def verify_avro_notice(content: bytes, description: str) -> None:
+ for marker in AVRO_NOTICE_MARKERS:
+ require(marker in content, f"{description} is missing {marker!r}")
+
+
+def verify_license_report(content: bytes, target: str, description: str) -> None:
+ markers = (
+ b"Artifact: python",
+ f"Rust target: {target}".encode(),
+ b"Root manifest: bindings/python/Cargo.toml",
+ XZ_LICENSE_ANCHOR,
+ )
+ for marker in markers:
+ require(marker in content, f"{description} is missing {marker!r}")
+
+
+def parse_metadata(content: bytes, description: str) -> Message:
+ try:
+ metadata = BytesParser(policy=policy.default).parsebytes(content)
+ except Exception as error:
+ raise VerificationError(f"invalid {description}: {error}") from error
+ require(
+ metadata.get("Metadata-Version") == "2.4",
+ f"{description} must use Metadata-Version 2.4",
+ )
+ require(
+ metadata.get("Name") == "pypaimon-rust",
+ f"{description} has an invalid package name",
+ )
+ require(bool(metadata.get("Version")), f"{description} has no version")
+ require(
+ metadata.get("License-Expression") == "Apache-2.0",
+ f"{description} has an invalid License-Expression",
+ )
+ license_files = metadata.get_all("License-File", [])
+ require(
+ sorted(license_files) == sorted(LEGAL_FILES),
+ f"{description} has invalid License-File entries: {license_files}",
+ )
+ return metadata
+
+
+def workspace_version() -> str:
+ with (ROOT / "Cargo.toml").open("rb") as source:
+ return str(tomllib.load(source)["workspace"]["package"]["version"])
+
+
+def release_pyproject(source: str, version: str, base_version: str) -> str:
+ if version == base_version:
+ return source
+ dynamic_version = 'dynamic = ["version"]\n'
+ fixed_version = f'version = "{version}"\n'
+ dynamic_count = source.count(dynamic_version)
+ fixed_count = source.count(fixed_version)
+ if dynamic_count == 1 and fixed_count == 0:
+ return source.replace(dynamic_version, fixed_version, 1)
+ require(
+ dynamic_count == 0 and fixed_count == 1,
+ "unexpected Python release version setting",
+ )
+ return source
+
+
+def expected_sdist_pyproject(version: str) -> bytes:
+ source = (PYTHON_DIR / "pyproject.toml").read_text(encoding="utf-8")
+ source = release_pyproject(source, version, workspace_version())
+ python_source = 'python-source = "python"\n'
+ dependency_groups = "\n[dependency-groups]\n"
+ require(source.count(python_source) == 1, "unexpected python-source setting")
+ require(
+ source.count(dependency_groups) == 1,
+ "unexpected dependency-groups section",
+ )
+ source = source.replace(python_source, "", 1)
+ rewrite = (
+ 'manifest-path = "bindings/python/Cargo.toml"\n'
+ + python_source
+ + dependency_groups
+ )
+ return source.replace(dependency_groups, rewrite, 1).encode()
+
+
+def target_from_wheel_name(path: Path) -> str:
+ parts = path.name.removesuffix(".whl").rsplit("-", 3)
+ require(len(parts) == 4, f"invalid wheel filename: {path.name}")
+ platform_tags = parts[-1].split(".")
+ targets = set()
+ for platform_tag in platform_tags:
+ if platform_tag == "win_amd64":
+ targets.add("x86_64-pc-windows-msvc")
+ elif platform_tag.startswith("macosx_") and platform_tag.endswith("_x86_64"):
+ targets.add("x86_64-apple-darwin")
+ elif platform_tag.startswith("macosx_") and platform_tag.endswith("_arm64"):
+ targets.add("aarch64-apple-darwin")
+ elif re.fullmatch(
+ r"(?:manylinux(?:\d{4}|_\d+_\d+)|linux)_x86_64", platform_tag
+ ):
+ targets.add("x86_64-unknown-linux-gnu")
+ elif re.fullmatch(
+ r"(?:manylinux(?:\d{4}|_\d+_\d+)|linux)_aarch64", platform_tag
+ ):
+ targets.add("aarch64-unknown-linux-gnu")
+ else:
+ raise VerificationError(
+ f"unsupported wheel platform tag {platform_tag!r}: {path.name}"
+ )
+ require(len(targets) == 1, f"wheel has mixed platform targets: {path.name}")
+ return targets.pop()
+
+
+def verify_native_header(content: bytes, target: str, description: str) -> None:
+ if target.endswith("linux-gnu"):
+ require(content[:4] == b"\x7fELF", f"{description} is not ELF")
+ require(len(content) >= 20 and content[4] == 2, f"{description} is not ELF64")
+ byte_order = "little" if content[5] == 1 else "big"
+ machine = int.from_bytes(content[18:20], byte_order)
+ expected = 62 if target.startswith("x86_64") else 183
+ require(machine == expected, f"{description} has ELF machine {machine}")
+ return
+
+ if target.endswith("windows-msvc"):
+ require(content[:2] == b"MZ" and len(content) >= 64, f"{description} is not PE")
+ pe_offset = int.from_bytes(content[60:64], "little")
+ require(
+ len(content) >= pe_offset + 6, f"{description} has a truncated PE header"
+ )
+ require(
+ content[pe_offset : pe_offset + 4] == b"PE\0\0", f"{description} is not PE"
+ )
+ machine = int.from_bytes(content[pe_offset + 4 : pe_offset + 6], "little")
+ require(machine == 0x8664, f"{description} has PE machine {machine:#x}")
+ return
+
+ magic = content[:4]
+ if magic == b"\xcf\xfa\xed\xfe":
+ byte_order = "little"
+ elif magic == b"\xfe\xed\xfa\xcf":
+ byte_order = "big"
+ else:
+ raise VerificationError(f"{description} is not 64-bit Mach-O")
+ require(len(content) >= 8, f"{description} has a truncated Mach-O header")
+ cpu_type = int.from_bytes(content[4:8], byte_order)
+ expected = 0x01000007 if target.startswith("x86_64") else 0x0100000C
+ require(cpu_type == expected, f"{description} has Mach-O CPU type {cpu_type:#x}")
+
+
+def is_native_library(name: str) -> bool:
+ filename = PurePosixPath(name).name.lower()
+ return filename.endswith((".so", ".pyd", ".dylib", ".dll")) or ".so." in filename
+
+
+def verify_wheel(path: Path) -> tuple[str, str]:
+ target = target_from_wheel_name(path)
+ expected_dir = PYTHON_DIR / "licenses" / target
+ if not expected_dir.is_dir():
+ expected_dir = PYTHON_DIR
+ filename_prefix = "pypaimon_rust-"
+ filename_head = path.name.removesuffix(".whl").rsplit("-", 3)[0]
+ require(
+ filename_head.startswith(filename_prefix),
+ f"invalid wheel distribution name: {path.name}",
+ )
+ filename_version = filename_head.removeprefix(filename_prefix)
+
+ try:
+ archive = zipfile.ZipFile(path)
+ except (OSError, zipfile.BadZipFile) as error:
+ raise VerificationError(f"cannot open wheel {path}: {error}") from error
+
+ with archive:
+ infos = [info for info in archive.infolist() if not info.is_dir()]
+ names = [info.filename for info in infos]
+ portable_names = set()
+ for info in infos:
+ mode = info.external_attr >> 16
+ file_type = stat.S_IFMT(mode)
+ require(
+ file_type in {0, stat.S_IFREG},
+ f"wheel has a link or special member: {info.filename}",
+ )
+ normalized = normalized_relative_path(info.filename, "wheel member")
+ key = portable_path_key(normalized)
+ require(
+ key not in portable_names,
+ f"wheel has a portable path collision: {path.name}:{info.filename}",
+ )
+ portable_names.add(key)
+ bad_member = archive.testzip()
+ require(bad_member is None, f"wheel has a corrupt member: {bad_member}")
+
+ metadata_members = [
+ name for name in names if name.endswith(".dist-info/METADATA")
+ ]
+ require(
+ len(metadata_members) == 1, f"wheel must contain one METADATA: {path.name}"
+ )
+ metadata_member = metadata_members[0]
+ metadata = parse_metadata(archive.read(metadata_member), metadata_member)
+ version = str(metadata["Version"])
+ require(
+ version == filename_version,
+ f"wheel filename and metadata versions differ: {path.name}",
+ )
+ dist_info = PurePosixPath(metadata_member).parent
+
+ wheel_members = [name for name in names if name == f"{dist_info}/WHEEL"]
+ require(len(wheel_members) == 1, f"wheel metadata is incomplete: {path.name}")
+ wheel_metadata = BytesParser(policy=policy.default).parsebytes(
+ archive.read(wheel_members[0])
+ )
+ require(
+ wheel_metadata.get("Root-Is-Purelib", "").lower() == "false",
+ f"wheel is incorrectly marked pure: {path.name}",
+ )
+ record_member = f"{dist_info}/RECORD"
+ require(record_member in names, f"wheel has no RECORD: {path.name}")
+ verify_record(archive, names, record_member, path.name)
+
+ license_report = None
+ for license_file in metadata.get_all("License-File", []):
+ relative = normalized_relative_path(license_file, "License-File entry")
+ members = (
+ str(relative),
+ str(dist_info / "licenses" / relative),
+ )
+ matches = [
+ name
+ for name in names
+ if PurePosixPath(name).name == relative.name
+ and relative.name in LEGAL_FILES
+ ]
+ require(
+ sorted(matches) == sorted(members),
+ f"wheel has invalid {relative.name} members: {matches}",
+ )
+ expected_path = expected_dir / relative
+ require(
+ expected_path.is_file(),
+ f"missing committed legal file: {expected_path}",
+ )
+ expected = expected_path.read_bytes()
+ for member in members:
+ actual = archive.read(member)
+ require(
+ actual == expected,
+ f"wheel {member} differs for {target}",
+ )
+ verify_legal_bytes(actual, f"{path.name}:{member}")
+ if relative.name == "NOTICE":
+ verify_avro_notice(actual, f"{path.name}:{member}")
+ if relative.name == "THIRD-PARTY-LICENSES.html":
+ license_report = actual
+ verify_license_report(actual, target, f"{path.name}:{member}")
+
+ native_members = [
+ name
+ for name in names
+ if len(PurePosixPath(name).parts) == 2
+ and PurePosixPath(name).parts[0] == "pypaimon_rust"
+ and PurePosixPath(name).name.startswith("pypaimon_rust.")
+ and PurePosixPath(name).suffix in {".so", ".pyd"}
+ ]
+ require(
+ len(native_members) == 1,
+ f"wheel must contain one native module: {path.name}",
+ )
+ native_member = native_members[0]
+ expected_suffix = ".pyd" if target.endswith("windows-msvc") else ".so"
+ require(
+ PurePosixPath(native_member).suffix == expected_suffix,
+ f"wheel has an invalid native module suffix: {native_member}",
+ )
+ with archive.open(native_member) as native_file:
+ verify_native_header(
+ native_file.read(65536), target, f"{path.name}:{native_member}"
+ )
+
+ extra_native_members = [
+ name for name in names if name != native_member and is_native_library(name)
+ ]
+ if target.endswith("linux-gnu"):
+ openssl_version = (
+ b"OpenSSL 1.1.1k FIPS"
+ if target.startswith("x86_64")
+ else b"OpenSSL 1.1.1w"
+ )
+ openssl_libraries = (
+ (r"pypaimon_rust\.libs/libcrypto-[^/]+\.so\.1\.1", openssl_version),
+ (r"pypaimon_rust\.libs/libssl-[^/]+\.so\.1\.1", None),
+ )
+ for pattern, version_marker in openssl_libraries:
+ matches = [
+ name for name in extra_native_members if re.fullmatch(pattern, name)
+ ]
+ require(
+ len(matches) == 1,
+ f"wheel must contain one {pattern}: {extra_native_members}",
+ )
+ with archive.open(matches[0]) as native_file:
+ content = native_file.read()
+ verify_native_header(
+ content,
+ target,
+ f"{path.name}:{matches[0]}",
+ )
+ if version_marker is not None:
+ require(
+ version_marker in content,
+ f"wheel libcrypto is missing {version_marker!r}",
+ )
+ require(
+ len(extra_native_members) == len(openssl_libraries),
+ f"wheel has unexpected native libraries: {extra_native_members}",
+ )
+ require(
+ license_report is not None
+ and b'id="bundled-openssl-1.1.1"' in license_report,
+ "Linux wheel license report is missing the OpenSSL anchor",
+ )
+ else:
+ require(
+ not extra_native_members,
+ f"wheel has unexpected native libraries: {extra_native_members}",
+ )
+
+ return target, version
+
+
+def verify_sdist(path: Path) -> str:
+ try:
+ archive = tarfile.open(path, mode="r:gz")
+ except (OSError, tarfile.TarError) as error:
+ raise VerificationError(f"cannot open sdist {path}: {error}") from error
+
+ with archive:
+ members_by_name = index_tar_members(
+ archive.getmembers(), f"sdist {path.name}"
+ )
+ names = list(members_by_name)
+ roots = {PurePosixPath(name).parts[0] for name in names}
+ require(len(roots) == 1, f"sdist must have one root directory: {path.name}")
+ root = roots.pop()
+
+ def read_member(name: str) -> bytes:
+ try:
+ member = members_by_name[name]
+ except KeyError as error:
+ raise VerificationError(
+ f"sdist is missing {name}: {path.name}"
+ ) from error
+ require(member.isfile(), f"sdist member is not a file: {name}")
+ extracted = archive.extractfile(member)
+ require(extracted is not None, f"cannot read sdist member: {name}")
+ return extracted.read()
+
+ metadata = parse_metadata(read_member(f"{root}/PKG-INFO"), f"{root}/PKG-INFO")
+ version = str(metadata["Version"])
+ require(root == f"pypaimon_rust-{version}", f"invalid sdist root: {root}")
+ require(
+ path.name == f"pypaimon_rust-{version}.tar.gz",
+ f"invalid sdist filename: {path.name}",
+ )
+ for license_file in metadata.get_all("License-File", []):
+ relative = normalized_relative_path(license_file, "License-File entry")
+ member = f"{root}/{relative}"
+ actual = read_member(member)
+ expected_path = PYTHON_DIR / relative
+ require(
+ expected_path.is_file(), f"missing source legal file: {expected_path}"
+ )
+ require(
+ actual == expected_path.read_bytes(),
+ f"sdist {relative} differs from source",
+ )
+ verify_legal_bytes(actual, f"{path.name}:{member}")
+
+ require(
+ read_member(f"{root}/Cargo.lock") == (ROOT / "Cargo.lock").read_bytes(),
+ "sdist Cargo.lock differs from source",
+ )
+ require(
+ read_member(f"{root}/pyproject.toml") == expected_sdist_pyproject(version),
+ "sdist pyproject.toml has an unexpected rewrite",
+ )
+ target_reports = [
+ name for name in names if "/bindings/python/licenses/" in name
+ ]
+ require(
+ not target_reports,
+ f"sdist contains target-specific license reports: {target_reports}",
+ )
+ native_members = [name for name in names if is_native_library(name)]
+ require(
+ not native_members, f"sdist contains native libraries: {native_members}"
+ )
+ return version
+
+
+def collect_artifacts(inputs: list[str]) -> list[Path]:
+ candidates = [Path(value) for value in inputs] or [PYTHON_DIR / "dist"]
+ artifacts = []
+ for candidate in candidates:
+ if candidate.is_dir():
+ artifacts.extend(sorted(candidate.iterdir()))
+ else:
+ artifacts.append(candidate)
+ artifacts = [
+ path
+ for path in artifacts
+ if path.name.endswith(".whl") or path.name.endswith(".tar.gz")
+ ]
+ unique = list(dict.fromkeys(path.resolve() for path in artifacts))
+ require(bool(unique), "no wheel or sdist artifacts found")
+ for path in unique:
+ require(path.is_file(), f"artifact is not a file: {path}")
+ return unique
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("artifacts", nargs="*", help="artifact files or directories")
+ parser.add_argument(
+ "--allow-partial",
+ action="store_true",
+ help="verify supplied artifacts without requiring the full release set",
+ )
+ args = parser.parse_args()
+
+ try:
+ artifacts = collect_artifacts(args.artifacts)
+ targets = {}
+ sdist_count = 0
+ versions = set()
+ for artifact in artifacts:
+ if artifact.name.endswith(".whl"):
+ target, version = verify_wheel(artifact)
+ require(target not in targets, f"duplicate wheel target: {target}")
+ targets[target] = artifact
+ versions.add(version)
+ print(f"verified {artifact.name} ({target})")
+ else:
+ versions.add(verify_sdist(artifact))
+ sdist_count += 1
+ print(f"verified {artifact.name} (sdist)")
+
+ require(len(versions) == 1, f"artifact versions differ: {sorted(versions)}")
+ require(sdist_count <= 1, f"expected at most one sdist, found {sdist_count}")
+ if not args.allow_partial:
+ missing = sorted(TARGETS - targets.keys())
+ require(not missing, f"missing wheel targets: {missing}")
+ require(sdist_count == 1, f"expected one sdist, found {sdist_count}")
+ print(f"verified {len(artifacts)} Python release artifacts")
+ return 0
+ except (
+ KeyError,
+ OSError,
+ VerificationError,
+ tarfile.TarError,
+ tomllib.TOMLDecodeError,
+ zipfile.BadZipFile,
+ ) as error:
+ print(f"Python release verification failed: {error}", file=sys.stderr)
+ return 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/third-party-licenses/README.md b/third-party-licenses/README.md
new file mode 100644
index 00000000..642d169a
--- /dev/null
+++ b/third-party-licenses/README.md
@@ -0,0 +1,34 @@
+
+
+# Bundled third-party licenses
+
+`openssl-1.1.1.LICENSE` covers OpenSSL 1.1.1k FIPS on x86_64 Linux and
+OpenSSL 1.1.1w on aarch64 Linux. Both versions use the same license text.
+
+The XZ Utils 5.8.3 license is read from the locked `liblzma-sys` crate at
+`xz/COPYING.0BSD`.
+
+Target-specific Python and Go reports are generated in their binary build jobs.
+They are intentionally not committed or included in the ASF source archive.
+
+Sources:
+
+-
+-
diff --git a/third-party-licenses/openssl-1.1.1.LICENSE b/third-party-licenses/openssl-1.1.1.LICENSE
new file mode 100644
index 00000000..5b5ccdc9
--- /dev/null
+++ b/third-party-licenses/openssl-1.1.1.LICENSE
@@ -0,0 +1,124 @@
+
+ LICENSE ISSUES
+ ==============
+
+ The OpenSSL toolkit stays under a double license, i.e. both the conditions of
+ the OpenSSL License and the original SSLeay license apply to the toolkit.
+ See below for the actual license texts.
+
+ OpenSSL License
+ ---------------
+
+/* ====================================================================
+ * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ * software must display the following acknowledgment:
+ * "This product includes software developed by the OpenSSL Project
+ * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ * endorse or promote products derived from this software without
+ * prior written permission. For written permission, please contact
+ * openssl-core@openssl.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ * nor may "OpenSSL" appear in their names without prior written
+ * permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ * acknowledgment:
+ * "This product includes software developed by the OpenSSL Project
+ * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com). This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+ Original SSLeay License
+ -----------------------
+
+/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
+ * All rights reserved.
+ *
+ * This package is an SSL implementation written
+ * by Eric Young (eay@cryptsoft.com).
+ * The implementation was written so as to conform with Netscapes SSL.
+ *
+ * This library is free for commercial and non-commercial use as long as
+ * the following conditions are aheared to. The following conditions
+ * apply to all code found in this distribution, be it the RC4, RSA,
+ * lhash, DES, etc., code; not just the SSL code. The SSL documentation
+ * included with this distribution is covered by the same copyright terms
+ * except that the holder is Tim Hudson (tjh@cryptsoft.com).
+ *
+ * Copyright remains Eric Young's, and as such any Copyright notices in
+ * the code are not to be removed.
+ * If this package is used in a product, Eric Young should be given attribution
+ * as the author of the parts of the library used.
+ * This can be in the form of a textual message at program startup or
+ * in documentation (online or textual) provided with the package.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * "This product includes cryptographic software written by
+ * Eric Young (eay@cryptsoft.com)"
+ * The word 'cryptographic' can be left out if the rouines from the library
+ * being used are not cryptographic related :-).
+ * 4. If you include any Windows specific code (or a derivative thereof) from
+ * the apps directory (application code) you must include an acknowledgement:
+ * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The licence and distribution terms for any publically available version or
+ * derivative of this code cannot be changed. i.e. this code cannot simply be
+ * copied and put under another distribution licence
+ * [including the GNU Public Licence.]
+ */