build: correct Jieba release license metadata

.licenserc.yaml

@@ -28,6 +28,7 @@ header:
     - ".github/PULL_REQUEST_TEMPLATE.md"
     - "crates/paimon/tests/**/*.json"
     - "crates/paimon/testdata/**"
+    - "third-party-licenses/jieba-rs-0.10.3.LICENSE"
     - "third-party-licenses/openssl-1.1.1.LICENSE"
     - "**/go.sum"
     - "**/DEPENDENCIES.*.tsv"

scripts/release_licenses.py

@@ -61,6 +61,8 @@ class LicenseCorrection:
     license_path: str
     license_name: str
     anchor: str
+    license_from_repository: bool = False
+    expected_versions: tuple[tuple[str, str], ...] = ()
 
 
 @dataclass(frozen=True)
@@ -122,6 +124,18 @@ class BundledComponent:
 )
 
 PYTHON_MIT_CORRECTIONS = (
+    LicenseCorrection(
+        crates=("jieba-macros", "jieba-rs"),
+        license_crate="jieba-rs",
+        license_path="third-party-licenses/jieba-rs-0.10.3.LICENSE",
+        license_name="MIT License (jieba-rs workspace)",
+        anchor="mit-jieba-rs-workspace",
+        license_from_repository=True,
+        expected_versions=(
+            ("jieba-macros", "0.10.3"),
+            ("jieba-rs", "0.10.3"),
+        ),
+    ),
     LicenseCorrection(
         crates=(
             "ownedbytes",
@@ -342,14 +356,18 @@ def resolved_packages(metadata: dict) -> list[dict]:
     ]
 
 
-def package_by_name(
-    metadata: dict, crate_name: str, required: bool = True
-) -> dict | None:
-    matches = [
+def packages_by_name(metadata: dict, crate_name: str) -> list[dict]:
+    return [
         package
         for package in resolved_packages(metadata)
         if package["name"] == crate_name
     ]
+
+
+def package_by_name(
+    metadata: dict, crate_name: str, required: bool = True
+) -> dict | None:
+    matches = packages_by_name(metadata, crate_name)
     if not matches and not required:
         return None
     if len(matches) != 1:
@@ -371,25 +389,61 @@ def package_features(metadata: dict, package: dict) -> set[str]:
     return set(matches[0])
 
 
-def correction_html(metadata: dict, correction: LicenseCorrection) -> str:
+def correction_html(
+    root: Path, metadata: dict, correction: LicenseCorrection
+) -> str:
+    expected_versions = dict(correction.expected_versions)
     used_by = []
     for crate_name in correction.crates:
-        package = package_by_name(metadata, crate_name)
-        repository = package.get("repository") or (
-            f"https://crates.io/crates/{crate_name}"
-        )
-        used_by.append(
-            f'                    <li><a href="{html.escape(repository, quote=True)}">'
-            f"{html.escape(crate_name)} {html.escape(package['version'])}</a></li>"
-        )
+        packages = packages_by_name(metadata, crate_name)
+        if not packages:
+            raise RuntimeError(f"expected at least one resolved {crate_name} package")
+        for package in packages:
+            if correction.license_from_repository:
+                expected_version = expected_versions.get(crate_name)
+                if expected_version is None:
+                    raise RuntimeError(
+                        "repository-backed correction is missing an expected "
+                        f"version for {crate_name}"
+                    )
+                if package["version"] != expected_version:
+                    raise RuntimeError(
+                        f"expected {crate_name} {expected_version}, "
+                        f"found {package['version']}"
+                    )
+            repository = package.get("repository") or (
+                f"https://crates.io/crates/{crate_name}"
+            )
+            used_by.append(
+                f'                    <li><a href="{html.escape(repository, quote=True)}">'
+                f"{html.escape(crate_name)} {html.escape(package['version'])}</a></li>"
+            )
 
-    license_package = package_by_name(metadata, correction.license_crate)
-    license_file = (
-        Path(license_package["manifest_path"]).parent / correction.license_path
-    )
-    if not license_file.is_file():
-        raise RuntimeError(f"corrected license file is missing: {license_file}")
-    license_text = license_file.read_text(encoding="utf-8")
+    if correction.license_from_repository:
+        license_file = root / correction.license_path
+        if not license_file.is_file():
+            raise RuntimeError(f"corrected license file is missing: {license_file}")
+        license_text = license_file.read_text(encoding="utf-8")
+    else:
+        license_files = [
+            Path(package["manifest_path"]).parent / correction.license_path
+            for package in packages_by_name(metadata, correction.license_crate)
+        ]
+        if not license_files:
+            raise RuntimeError(
+                f"expected at least one resolved {correction.license_crate} package"
+            )
+        missing = [path for path in license_files if not path.is_file()]
+        if missing:
+            raise RuntimeError(f"corrected license file is missing: {missing[0]}")
+        license_texts = {
+            path.read_text(encoding="utf-8") for path in license_files
+        }
+        if len(license_texts) != 1:
+            raise RuntimeError(
+                f"corrected license files differ for {correction.license_crate}"
+            )
+        license_text = license_texts.pop()
 
     return "\n".join(
         [
@@ -407,6 +461,7 @@ def correction_html(metadata: dict, correction: LicenseCorrection) -> str:
 
 
 def replace_placeholder_entry(
+    root: Path,
     report_text: str,
     metadata: dict,
     marker: str,
@@ -441,7 +496,7 @@ def replace_placeholder_entry(
         )
 
     replacement = "\n".join(
-        correction_html(metadata, correction) for correction in corrections
+        correction_html(root, metadata, correction) for correction in corrections
     )
     return report_text[:entry_start] + replacement + report_text[entry_end:]
 
@@ -538,13 +593,13 @@ def complete_report(
     root: Path, base_report: str, report: Report, metadata: dict
 ) -> str:
     result = replace_placeholder_entry(
-        base_report, metadata, ALLOC_PLACEHOLDER, ALLOC_CORRECTIONS
+        root, base_report, metadata, ALLOC_PLACEHOLDER, ALLOC_CORRECTIONS
     )
     mit_corrections = COMMON_MIT_CORRECTIONS
     if report.component == "python":
         mit_corrections += PYTHON_MIT_CORRECTIONS
     result = replace_placeholder_entry(
-        result, metadata, MIT_PLACEHOLDER, mit_corrections
+        root, result, metadata, MIT_PLACEHOLDER, mit_corrections
     )
 
     for placeholder in LICENSE_PLACEHOLDERS:

scripts/tests/test_release_licenses.py

@@ -19,7 +19,156 @@
 import unittest
 from pathlib import Path
 
-from scripts.release_licenses import binary_notice, resolved_packages
+from scripts.release_licenses import (
+    LicenseCorrection,
+    PYTHON_MIT_CORRECTIONS,
+    binary_notice,
+    correction_html,
+    resolved_packages,
+)
+
+
+class LicenseCorrectionsTest(unittest.TestCase):
+    def test_python_corrections_cover_jieba_workspace(self) -> None:
+        corrections = [
+            correction
+            for correction in PYTHON_MIT_CORRECTIONS
+            if correction.crates == ("jieba-macros", "jieba-rs")
+        ]
+        self.assertEqual(len(corrections), 1)
+        correction = corrections[0]
+        self.assertEqual(correction.license_crate, "jieba-rs")
+        self.assertEqual(
+            correction.license_path,
+            "third-party-licenses/jieba-rs-0.10.3.LICENSE",
+        )
+        root = Path(__file__).resolve().parents[2]
+        self.assertTrue((root / correction.license_path).is_file())
+
+    def test_repository_license_correction_reads_checked_in_file(self) -> None:
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            license_path = root / "third-party-licenses" / "jieba.LICENSE"
+            license_path.parent.mkdir()
+            license_path.write_text("Jieba workspace license\n", encoding="utf-8")
+            metadata = {
+                "normal_packages": {
+                    ("jieba-macros", "0.10.3"),
+                    ("jieba-rs", "0.10.3"),
+                },
+                "packages": [
+                    {
+                        "name": name,
+                        "version": "0.10.3",
+                        "manifest_path": str(root / name / "Cargo.toml"),
+                        "repository": "https://example.com/jieba-rs",
+                    }
+                    for name in ("jieba-macros", "jieba-rs")
+                ],
+            }
+            correction = LicenseCorrection(
+                crates=("jieba-macros", "jieba-rs"),
+                license_crate="jieba-rs",
+                license_path="third-party-licenses/jieba.LICENSE",
+                license_name="MIT License (jieba-rs workspace)",
+                anchor="mit-jieba-rs-workspace",
+                license_from_repository=True,
+                expected_versions=(
+                    ("jieba-macros", "0.10.3"),
+                    ("jieba-rs", "0.10.3"),
+                ),
+            )
+
+            result = correction_html(root, metadata, correction)
+
+            self.assertIn("Jieba workspace license", result)
+            self.assertIn("jieba-macros 0.10.3", result)
+            self.assertIn("jieba-rs 0.10.3", result)
+
+    def test_repository_license_correction_rejects_unverified_version(self) -> None:
+        root = Path(__file__).resolve().parents[2]
+        correction = next(
+            correction
+            for correction in PYTHON_MIT_CORRECTIONS
+            if correction.crates == ("jieba-macros", "jieba-rs")
+        )
+        metadata = {
+            "normal_packages": {
+                ("jieba-macros", "9.9.9"),
+                ("jieba-rs", "9.9.9"),
+            },
+            "packages": [
+                {
+                    "name": name,
+                    "version": "9.9.9",
+                    "manifest_path": str(root / name / "Cargo.toml"),
+                    "repository": "https://example.com/jieba-rs",
+                }
+                for name in ("jieba-macros", "jieba-rs")
+            ],
+        }
+
+        with self.assertRaisesRegex(
+            RuntimeError,
+            r"expected jieba-macros 0\.10\.3, found 9\.9\.9",
+        ):
+            correction_html(root, metadata, correction)
+
+    def test_correction_supports_multiple_versions_with_same_license(self) -> None:
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            packages = []
+            normal_packages = set()
+            for version in ("0.7.0", "0.11.0"):
+                package_dir = root / f"tantivy-common-{version}"
+                package_dir.mkdir()
+                manifest = package_dir / "Cargo.toml"
+                manifest.write_text("[package]\n", encoding="utf-8")
+                (package_dir / "LICENSE").write_text(
+                    "Shared Tantivy license\n",
+                    encoding="utf-8",
+                )
+                packages.append(
+                    {
+                        "name": "tantivy-common",
+                        "version": version,
+                        "manifest_path": str(manifest),
+                        "repository": "https://example.com/tantivy",
+                    }
+                )
+                normal_packages.add(("tantivy-common", version))
+            metadata = {
+                "normal_packages": normal_packages,
+                "packages": packages,
+            }
+            correction = LicenseCorrection(
+                crates=("tantivy-common",),
+                license_crate="tantivy-common",
+                license_path="LICENSE",
+                license_name="MIT License (Tantivy workspace)",
+                anchor="mit-tantivy-workspace",
+            )
+
+            result = correction_html(root, metadata, correction)
+
+            self.assertIn("tantivy-common 0.7.0", result)
+            self.assertIn("tantivy-common 0.11.0", result)
+            self.assertEqual(result.count("Shared Tantivy license"), 1)
+
+    def test_jieba_license_source_is_documented_and_header_exempt(self) -> None:
+        root = Path(__file__).resolve().parents[2]
+        license_path = "third-party-licenses/jieba-rs-0.10.3.LICENSE"
+        license_config = (root / ".licenserc.yaml").read_text(encoding="utf-8")
+        third_party_readme = (
+            root / "third-party-licenses" / "README.md"
+        ).read_text(encoding="utf-8")
+
+        self.assertIn(f'"{license_path}"', license_config)
+        self.assertIn(
+            "messense/jieba-rs/blob/"
+            "c62e0df1f9dcc2cc1e014711c5aa4561ae260538/LICENSE",
+            third_party_readme,
+        )
 
 
 class ResolvedPackagesTest(unittest.TestCase):

third-party-licenses/README.md

@@ -25,10 +25,16 @@ OpenSSL 1.1.1w on aarch64 Linux. Both versions use the same license text.
 The XZ Utils 5.8.3 license is read from the locked `liblzma-sys` crate at
 `xz/COPYING.0BSD`.
 
+`jieba-rs-0.10.3.LICENSE` is the license file from the root of the upstream
+`jieba-rs` workspace at commit
+`c62e0df1f9dcc2cc1e014711c5aa4561ae260538`. It covers `jieba-rs` 0.10.3 and
+`jieba-macros` 0.10.3.
+
 Target-specific Python and Go reports are generated in their binary build jobs.
 They are intentionally not committed or included in the ASF source archive.
 
 Sources:
 
 - <https://github.com/openssl/openssl/blob/OpenSSL_1_1_1k/LICENSE>
 - <https://github.com/openssl/openssl/blob/OpenSSL_1_1_1w/LICENSE>
+- <https://github.com/messense/jieba-rs/blob/c62e0df1f9dcc2cc1e014711c5aa4561ae260538/LICENSE>

third-party-licenses/jieba-rs-0.10.3.LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2018 - 2019 messense
+Copyright (c) 2019 Paul Meng
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.