Merge pull request #984 from PolicyEngine/feat/publication-candidate-changelog-snapshot

Snapshot publication candidate changelog fragments

Author: anth-volk

.github/bump_version.py

@@ -2,6 +2,8 @@
 
 import json
 import re
+import os
+import shutil
 import sys
 from pathlib import Path
 
@@ -18,6 +20,8 @@
 VERSION_RE = re.compile(r'^version\s*=\s*"([^"]+)"', re.MULTILINE)
 SEMVER_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)(?:rc(\d+))?$")
 PUBLICATION_SCOPE_PATH = Path(".github/publication_scope.json")
+PUBLICATION_CANDIDATES_DIR = Path(".github/publication_candidates")
+CHANGELOG_KEEP_FILE = ".gitkeep"
 
 
 def get_current_version(pyproject_path: Path) -> str:
@@ -33,9 +37,7 @@ def get_current_version(pyproject_path: Path) -> str:
 
 
 def infer_bump(changelog_dir: Path) -> str:
-    fragments = [
-        f for f in changelog_dir.iterdir() if f.is_file() and f.name != ".gitkeep"
-    ]
+    fragments = changelog_fragments(changelog_dir)
     if not fragments:
         print("No changelog fragments found", file=sys.stderr)
         sys.exit(1)
@@ -62,13 +64,56 @@ def bump_version(version: str, bump: str) -> str:
 
 
 def write_publication_scope(path: Path, payload: dict[str, str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
     print(f"  Updated {path}")
 
 
+def changelog_fragments(changelog_dir: Path) -> list[Path]:
+    return sorted(
+        f
+        for f in changelog_dir.iterdir()
+        if f.is_file() and f.name != CHANGELOG_KEEP_FILE
+    )
+
+
+def snapshot_changelog_fragments(
+    *,
+    run_id: str,
+    changelog_dir: Path,
+    publication_candidates_dir: Path,
+) -> Path:
+    fragments = changelog_fragments(changelog_dir)
+    if not run_id:
+        print(
+            "US_DATA_RUN_ID is required to snapshot changelog fragments",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    if not fragments:
+        print("No changelog fragments found", file=sys.stderr)
+        sys.exit(1)
+
+    snapshot_dir = publication_candidates_dir / run_id / "changelog.d"
+    if snapshot_dir.exists() and changelog_fragments(snapshot_dir):
+        print(
+            f"Candidate changelog snapshot already exists: {snapshot_dir}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    snapshot_dir.mkdir(parents=True, exist_ok=True)
+    for fragment in fragments:
+        destination = snapshot_dir / fragment.name
+        shutil.copy2(fragment, destination)
+        fragment.unlink()
+        print(f"  Snapshotted {fragment} -> {destination}")
+    return snapshot_dir
+
+
 def main():
     pyproject = _REPO_ROOT / "pyproject.toml"
     changelog_dir = _REPO_ROOT / "changelog.d"
+    run_id = os.environ.get("US_DATA_RUN_ID", "")
 
     current = get_current_version(pyproject)
     bump = infer_bump(changelog_dir)
@@ -80,14 +125,25 @@ def main():
     print(f"Release bump: {bump}")
     print(f"Would release as at build time: {would_release_as}")
 
+    snapshot_changelog_fragments(
+        run_id=run_id,
+        changelog_dir=changelog_dir,
+        publication_candidates_dir=_REPO_ROOT / PUBLICATION_CANDIDATES_DIR,
+    )
+    payload = {
+        "run_id": run_id,
+        "base_release_version": current,
+        "release_bump": bump,
+        "candidate_scope": candidate_scope,
+        "would_release_as_at_build_time": would_release_as,
+    }
     write_publication_scope(
         _REPO_ROOT / PUBLICATION_SCOPE_PATH,
-        {
-            "base_release_version": current,
-            "release_bump": bump,
-            "candidate_scope": candidate_scope,
-            "would_release_as_at_build_time": would_release_as,
-        },
+        payload,
+    )
+    write_publication_scope(
+        _REPO_ROOT / PUBLICATION_CANDIDATES_DIR / run_id / PUBLICATION_SCOPE_PATH.name,
+        payload,
     )
 
 

.github/scripts/resolve_run_context.py

@@ -57,15 +57,27 @@ def _pyproject_version() -> str:
         return tomllib.load(file)["project"]["version"]
 
 
-def _publication_scope() -> dict[str, str]:
+def _publication_scope(env: Mapping[str, str] | None = None) -> dict[str, str]:
+    env = env or os.environ
+    run_id = env.get(RUN_ID_ENV, "")
+    if run_id:
+        candidate_path = (
+            _REPO_ROOT
+            / ".github"
+            / "publication_candidates"
+            / run_id
+            / "publication_scope.json"
+        )
+        if candidate_path.exists():
+            return json.loads(candidate_path.read_text())
     path = _REPO_ROOT / ".github" / "publication_scope.json"
     if not path.exists():
         return {}
     return json.loads(path.read_text())
 
 
 def _base_release_version(env: Mapping[str, str]) -> str:
-    scope = _publication_scope()
+    scope = _publication_scope(env)
     value = (
         env.get(BASE_RELEASE_VERSION_ENV)
         or env.get("BASE_RELEASE_VERSION", "")
@@ -77,7 +89,7 @@ def _base_release_version(env: Mapping[str, str]) -> str:
 
 
 def _release_bump(env: Mapping[str, str]) -> str:
-    scope = _publication_scope()
+    scope = _publication_scope(env)
     value = (
         env.get(RELEASE_BUMP_ENV)
         or env.get("RELEASE_BUMP", "")
@@ -94,7 +106,7 @@ def _candidate_version(
     base_release_version: str = "",
     release_bump: str = "",
 ) -> str:
-    scope = _publication_scope()
+    scope = _publication_scope(env)
     version = (
         env.get(CANDIDATE_SCOPE_ENV)
         or env.get(CANDIDATE_VERSION_ENV)

.github/scripts/restore_publication_changelog.py

@@ -0,0 +1,93 @@
+"""Restore candidate-scoped changelog fragments for final promotion."""
+
+from __future__ import annotations
+
+import filecmp
+import os
+import shutil
+import sys
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+ROOT_CHANGELOG_DIR = REPO_ROOT / "changelog.d"
+PUBLICATION_CANDIDATES_DIR = REPO_ROOT / ".github" / "publication_candidates"
+CHANGELOG_KEEP_FILE = ".gitkeep"
+
+
+def _fragments(path: Path) -> list[Path]:
+    if not path.exists():
+        return []
+    return sorted(
+        item
+        for item in path.iterdir()
+        if item.is_file() and item.name != CHANGELOG_KEEP_FILE
+    )
+
+
+def _validate_root_fragments_match_snapshot(
+    *,
+    root_fragments: list[Path],
+    snapshot_fragments: list[Path],
+) -> None:
+    snapshot_by_name = {fragment.name: fragment for fragment in snapshot_fragments}
+    root_by_name = {fragment.name: fragment for fragment in root_fragments}
+    extra = sorted(set(root_by_name).difference(snapshot_by_name))
+    missing = sorted(set(snapshot_by_name).difference(root_by_name))
+    changed = sorted(
+        name
+        for name in set(root_by_name).intersection(snapshot_by_name)
+        if not filecmp.cmp(root_by_name[name], snapshot_by_name[name], shallow=False)
+    )
+    if extra or missing or changed:
+        details = []
+        if extra:
+            details.append(f"extra root fragments: {', '.join(extra)}")
+        if missing:
+            details.append(f"missing root fragments: {', '.join(missing)}")
+        if changed:
+            details.append(f"changed root fragments: {', '.join(changed)}")
+        raise RuntimeError(
+            "Root changelog fragments do not match the candidate snapshot; "
+            + "; ".join(details)
+        )
+
+
+def restore_candidate_changelog(run_id: str) -> Path:
+    if not run_id:
+        raise RuntimeError("US_DATA_RUN_ID is required to restore changelog fragments.")
+
+    snapshot_dir = PUBLICATION_CANDIDATES_DIR / run_id / "changelog.d"
+    snapshot_fragments = _fragments(snapshot_dir)
+    if not snapshot_fragments:
+        raise RuntimeError(
+            f"No candidate changelog fragments found for run {run_id}: {snapshot_dir}"
+        )
+
+    ROOT_CHANGELOG_DIR.mkdir(parents=True, exist_ok=True)
+    root_fragments = _fragments(ROOT_CHANGELOG_DIR)
+    if root_fragments:
+        _validate_root_fragments_match_snapshot(
+            root_fragments=root_fragments,
+            snapshot_fragments=snapshot_fragments,
+        )
+
+    for fragment in snapshot_fragments:
+        destination = ROOT_CHANGELOG_DIR / fragment.name
+        shutil.copy2(fragment, destination)
+        print(f"Restored {destination} from {fragment}")
+
+    return snapshot_dir
+
+
+def main() -> None:
+    try:
+        snapshot_dir = restore_candidate_changelog(os.environ.get("US_DATA_RUN_ID", ""))
+    except Exception as exc:
+        print(str(exc), file=sys.stderr)
+        sys.exit(1)
+    print(f"Restored candidate changelog fragments from {snapshot_dir}")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/local_area_promote.yaml

@@ -46,8 +46,8 @@ jobs:
 
       - uses: astral-sh/setup-uv@v8.1.0
 
-      - name: Install Modal CLI
-        run: pip install modal
+      - name: Install promotion CLI deps
+        run: pip install modal towncrier
 
       - name: Resolve run context
         id: run-context
@@ -58,13 +58,15 @@ jobs:
 
       - name: Finalize package version
         run: |
+          python .github/scripts/restore_publication_changelog.py
           python .github/scripts/finalize_package_version.py
+          towncrier build --yes --version "$US_DATA_RELEASE_VERSION"
           uv lock
 
       - name: Commit final package version
         uses: EndBug/add-and-commit@v10
         with:
-          add: "pyproject.toml uv.lock"
+          add: "pyproject.toml uv.lock CHANGELOG.md changelog.d"
           message: Finalize package version
 
       - name: Build final wheel

.github/workflows/push.yaml

@@ -62,7 +62,7 @@ jobs:
           folder: docs/_build/html
           clean: true
 
-  # ── Publication candidate scope + changelog on ordinary pushes ──
+  # ── Publication candidate scope + changelog snapshot on ordinary pushes ──
   versioning:
     name: Versioning
     runs-on: ubuntu-latest
@@ -87,11 +87,11 @@ jobs:
         with:
           python-version: "3.14"
       - uses: astral-sh/setup-uv@v8.1.0
-      - run: pip install towncrier
-      - name: Bump version and build changelog
+      - name: Snapshot candidate changelog fragments
+        env:
+          US_DATA_RUN_ID: ${{ needs.run-context.outputs.run_id }}
         run: |
           python .github/bump_version.py
-          towncrier build --yes --version "$(python .github/scripts/fetch_publication_scope.py would_release_as_at_build_time)"
       - name: Generate pipeline documentation artifacts
         run: uv run --no-sync --with pyyaml python scripts/extract_pipeline_docs.py
       - name: Update lockfile

changelog.d/983.changed.md

@@ -0,0 +1 @@
+Snapshot publication candidate changelog fragments for final release promotion.