diff --git a/.licenserc.yaml b/.licenserc.yaml
index c3cac61b..e147040a 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -28,6 +28,7 @@ header:
- ".github/PULL_REQUEST_TEMPLATE.md"
- "crates/paimon/tests/**/*.json"
- "crates/paimon/testdata/**"
+ - "third-party-licenses/jieba-rs-0.10.3.LICENSE"
- "third-party-licenses/openssl-1.1.1.LICENSE"
- "**/go.sum"
- "**/DEPENDENCIES.*.tsv"
diff --git a/scripts/release_licenses.py b/scripts/release_licenses.py
index 8c6e6323..5785cf80 100644
--- a/scripts/release_licenses.py
+++ b/scripts/release_licenses.py
@@ -61,6 +61,8 @@ class LicenseCorrection:
license_path: str
license_name: str
anchor: str
+ license_from_repository: bool = False
+ expected_versions: tuple[tuple[str, str], ...] = ()
@dataclass(frozen=True)
@@ -122,6 +124,18 @@ class BundledComponent:
)
PYTHON_MIT_CORRECTIONS = (
+ LicenseCorrection(
+ crates=("jieba-macros", "jieba-rs"),
+ license_crate="jieba-rs",
+ license_path="third-party-licenses/jieba-rs-0.10.3.LICENSE",
+ license_name="MIT License (jieba-rs workspace)",
+ anchor="mit-jieba-rs-workspace",
+ license_from_repository=True,
+ expected_versions=(
+ ("jieba-macros", "0.10.3"),
+ ("jieba-rs", "0.10.3"),
+ ),
+ ),
LicenseCorrection(
crates=(
"ownedbytes",
@@ -342,14 +356,18 @@ def resolved_packages(metadata: dict) -> list[dict]:
]
-def package_by_name(
- metadata: dict, crate_name: str, required: bool = True
-) -> dict | None:
- matches = [
+def packages_by_name(metadata: dict, crate_name: str) -> list[dict]:
+ return [
package
for package in resolved_packages(metadata)
if package["name"] == crate_name
]
+
+
+def package_by_name(
+ metadata: dict, crate_name: str, required: bool = True
+) -> dict | None:
+ matches = packages_by_name(metadata, crate_name)
if not matches and not required:
return None
if len(matches) != 1:
@@ -371,25 +389,61 @@ def package_features(metadata: dict, package: dict) -> set[str]:
return set(matches[0])
-def correction_html(metadata: dict, correction: LicenseCorrection) -> str:
+def correction_html(
+ root: Path, metadata: dict, correction: LicenseCorrection
+) -> str:
+ expected_versions = dict(correction.expected_versions)
used_by = []
for crate_name in correction.crates:
- package = package_by_name(metadata, crate_name)
- repository = package.get("repository") or (
- f"https://crates.io/crates/{crate_name}"
- )
- used_by.append(
- f' '
- f"{html.escape(crate_name)} {html.escape(package['version'])}"
- )
+ packages = packages_by_name(metadata, crate_name)
+ if not packages:
+ raise RuntimeError(f"expected at least one resolved {crate_name} package")
+ for package in packages:
+ if correction.license_from_repository:
+ expected_version = expected_versions.get(crate_name)
+ if expected_version is None:
+ raise RuntimeError(
+ "repository-backed correction is missing an expected "
+ f"version for {crate_name}"
+ )
+ if package["version"] != expected_version:
+ raise RuntimeError(
+ f"expected {crate_name} {expected_version}, "
+ f"found {package['version']}"
+ )
+ repository = package.get("repository") or (
+ f"https://crates.io/crates/{crate_name}"
+ )
+ used_by.append(
+ f' '
+ f"{html.escape(crate_name)} {html.escape(package['version'])}"
+ )
- license_package = package_by_name(metadata, correction.license_crate)
- license_file = (
- Path(license_package["manifest_path"]).parent / correction.license_path
- )
- if not license_file.is_file():
- raise RuntimeError(f"corrected license file is missing: {license_file}")
- license_text = license_file.read_text(encoding="utf-8")
+ if correction.license_from_repository:
+ license_file = root / correction.license_path
+ if not license_file.is_file():
+ raise RuntimeError(f"corrected license file is missing: {license_file}")
+ license_text = license_file.read_text(encoding="utf-8")
+ else:
+ license_files = [
+ Path(package["manifest_path"]).parent / correction.license_path
+ for package in packages_by_name(metadata, correction.license_crate)
+ ]
+ if not license_files:
+ raise RuntimeError(
+ f"expected at least one resolved {correction.license_crate} package"
+ )
+ missing = [path for path in license_files if not path.is_file()]
+ if missing:
+ raise RuntimeError(f"corrected license file is missing: {missing[0]}")
+ license_texts = {
+ path.read_text(encoding="utf-8") for path in license_files
+ }
+ if len(license_texts) != 1:
+ raise RuntimeError(
+ f"corrected license files differ for {correction.license_crate}"
+ )
+ license_text = license_texts.pop()
return "\n".join(
[
@@ -407,6 +461,7 @@ def correction_html(metadata: dict, correction: LicenseCorrection) -> str:
def replace_placeholder_entry(
+ root: Path,
report_text: str,
metadata: dict,
marker: str,
@@ -441,7 +496,7 @@ def replace_placeholder_entry(
)
replacement = "\n".join(
- correction_html(metadata, correction) for correction in corrections
+ correction_html(root, metadata, correction) for correction in corrections
)
return report_text[:entry_start] + replacement + report_text[entry_end:]
@@ -538,13 +593,13 @@ def complete_report(
root: Path, base_report: str, report: Report, metadata: dict
) -> str:
result = replace_placeholder_entry(
- base_report, metadata, ALLOC_PLACEHOLDER, ALLOC_CORRECTIONS
+ root, base_report, metadata, ALLOC_PLACEHOLDER, ALLOC_CORRECTIONS
)
mit_corrections = COMMON_MIT_CORRECTIONS
if report.component == "python":
mit_corrections += PYTHON_MIT_CORRECTIONS
result = replace_placeholder_entry(
- result, metadata, MIT_PLACEHOLDER, mit_corrections
+ root, result, metadata, MIT_PLACEHOLDER, mit_corrections
)
for placeholder in LICENSE_PLACEHOLDERS:
diff --git a/scripts/tests/test_release_licenses.py b/scripts/tests/test_release_licenses.py
index 1742030e..2cfe1b8a 100644
--- a/scripts/tests/test_release_licenses.py
+++ b/scripts/tests/test_release_licenses.py
@@ -19,7 +19,156 @@
import unittest
from pathlib import Path
-from scripts.release_licenses import binary_notice, resolved_packages
+from scripts.release_licenses import (
+ LicenseCorrection,
+ PYTHON_MIT_CORRECTIONS,
+ binary_notice,
+ correction_html,
+ resolved_packages,
+)
+
+
+class LicenseCorrectionsTest(unittest.TestCase):
+ def test_python_corrections_cover_jieba_workspace(self) -> None:
+ corrections = [
+ correction
+ for correction in PYTHON_MIT_CORRECTIONS
+ if correction.crates == ("jieba-macros", "jieba-rs")
+ ]
+ self.assertEqual(len(corrections), 1)
+ correction = corrections[0]
+ self.assertEqual(correction.license_crate, "jieba-rs")
+ self.assertEqual(
+ correction.license_path,
+ "third-party-licenses/jieba-rs-0.10.3.LICENSE",
+ )
+ root = Path(__file__).resolve().parents[2]
+ self.assertTrue((root / correction.license_path).is_file())
+
+ def test_repository_license_correction_reads_checked_in_file(self) -> None:
+ with tempfile.TemporaryDirectory() as temp:
+ root = Path(temp)
+ license_path = root / "third-party-licenses" / "jieba.LICENSE"
+ license_path.parent.mkdir()
+ license_path.write_text("Jieba workspace license\n", encoding="utf-8")
+ metadata = {
+ "normal_packages": {
+ ("jieba-macros", "0.10.3"),
+ ("jieba-rs", "0.10.3"),
+ },
+ "packages": [
+ {
+ "name": name,
+ "version": "0.10.3",
+ "manifest_path": str(root / name / "Cargo.toml"),
+ "repository": "https://example.com/jieba-rs",
+ }
+ for name in ("jieba-macros", "jieba-rs")
+ ],
+ }
+ correction = LicenseCorrection(
+ crates=("jieba-macros", "jieba-rs"),
+ license_crate="jieba-rs",
+ license_path="third-party-licenses/jieba.LICENSE",
+ license_name="MIT License (jieba-rs workspace)",
+ anchor="mit-jieba-rs-workspace",
+ license_from_repository=True,
+ expected_versions=(
+ ("jieba-macros", "0.10.3"),
+ ("jieba-rs", "0.10.3"),
+ ),
+ )
+
+ result = correction_html(root, metadata, correction)
+
+ self.assertIn("Jieba workspace license", result)
+ self.assertIn("jieba-macros 0.10.3", result)
+ self.assertIn("jieba-rs 0.10.3", result)
+
+ def test_repository_license_correction_rejects_unverified_version(self) -> None:
+ root = Path(__file__).resolve().parents[2]
+ correction = next(
+ correction
+ for correction in PYTHON_MIT_CORRECTIONS
+ if correction.crates == ("jieba-macros", "jieba-rs")
+ )
+ metadata = {
+ "normal_packages": {
+ ("jieba-macros", "9.9.9"),
+ ("jieba-rs", "9.9.9"),
+ },
+ "packages": [
+ {
+ "name": name,
+ "version": "9.9.9",
+ "manifest_path": str(root / name / "Cargo.toml"),
+ "repository": "https://example.com/jieba-rs",
+ }
+ for name in ("jieba-macros", "jieba-rs")
+ ],
+ }
+
+ with self.assertRaisesRegex(
+ RuntimeError,
+ r"expected jieba-macros 0\.10\.3, found 9\.9\.9",
+ ):
+ correction_html(root, metadata, correction)
+
+ def test_correction_supports_multiple_versions_with_same_license(self) -> None:
+ with tempfile.TemporaryDirectory() as temp:
+ root = Path(temp)
+ packages = []
+ normal_packages = set()
+ for version in ("0.7.0", "0.11.0"):
+ package_dir = root / f"tantivy-common-{version}"
+ package_dir.mkdir()
+ manifest = package_dir / "Cargo.toml"
+ manifest.write_text("[package]\n", encoding="utf-8")
+ (package_dir / "LICENSE").write_text(
+ "Shared Tantivy license\n",
+ encoding="utf-8",
+ )
+ packages.append(
+ {
+ "name": "tantivy-common",
+ "version": version,
+ "manifest_path": str(manifest),
+ "repository": "https://example.com/tantivy",
+ }
+ )
+ normal_packages.add(("tantivy-common", version))
+ metadata = {
+ "normal_packages": normal_packages,
+ "packages": packages,
+ }
+ correction = LicenseCorrection(
+ crates=("tantivy-common",),
+ license_crate="tantivy-common",
+ license_path="LICENSE",
+ license_name="MIT License (Tantivy workspace)",
+ anchor="mit-tantivy-workspace",
+ )
+
+ result = correction_html(root, metadata, correction)
+
+ self.assertIn("tantivy-common 0.7.0", result)
+ self.assertIn("tantivy-common 0.11.0", result)
+ self.assertEqual(result.count("Shared Tantivy license"), 1)
+
+ def test_jieba_license_source_is_documented_and_header_exempt(self) -> None:
+ root = Path(__file__).resolve().parents[2]
+ license_path = "third-party-licenses/jieba-rs-0.10.3.LICENSE"
+ license_config = (root / ".licenserc.yaml").read_text(encoding="utf-8")
+ third_party_readme = (
+ root / "third-party-licenses" / "README.md"
+ ).read_text(encoding="utf-8")
+
+ self.assertIn(f'"{license_path}"', license_config)
+ self.assertIn(
+ "messense/jieba-rs/blob/"
+ "c62e0df1f9dcc2cc1e014711c5aa4561ae260538/LICENSE",
+ third_party_readme,
+ )
class ResolvedPackagesTest(unittest.TestCase):
diff --git a/third-party-licenses/README.md b/third-party-licenses/README.md
index 642d169a..449c9aa3 100644
--- a/third-party-licenses/README.md
+++ b/third-party-licenses/README.md
@@ -25,6 +25,11 @@ OpenSSL 1.1.1w on aarch64 Linux. Both versions use the same license text.
The XZ Utils 5.8.3 license is read from the locked `liblzma-sys` crate at
`xz/COPYING.0BSD`.
+`jieba-rs-0.10.3.LICENSE` is the license file from the root of the upstream
+`jieba-rs` workspace at commit
+`c62e0df1f9dcc2cc1e014711c5aa4561ae260538`. It covers `jieba-rs` 0.10.3 and
+`jieba-macros` 0.10.3.
+
Target-specific Python and Go reports are generated in their binary build jobs.
They are intentionally not committed or included in the ASF source archive.
@@ -32,3 +37,4 @@ Sources:
-
-
+-
diff --git a/third-party-licenses/jieba-rs-0.10.3.LICENSE b/third-party-licenses/jieba-rs-0.10.3.LICENSE
new file mode 100644
index 00000000..a9c2c612
--- /dev/null
+++ b/third-party-licenses/jieba-rs-0.10.3.LICENSE
@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2018 - 2019 messense
+Copyright (c) 2019 Paul Meng
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.