scheduled docs generation #105
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: scheduled docs generation | |
| on: | |
| schedule: | |
| - cron: '0 5 * * *' | |
| workflow_dispatch: | |
| push: | |
| # Pages deployment requires pages:write and id-token:write. | |
| # contents:write is no longer needed since we no longer push to gh-pages directly. | |
| permissions: | |
| contents: read | |
| pages: write | |
| id-token: write | |
| # Prevent overlapping scheduled runs from deploying to GitHub Pages simultaneously. | |
| concurrency: | |
| group: pages-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| get-versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| versions: ${{ steps.get-versions.outputs.versions }} | |
| steps: | |
| - name: Get supported Python versions and translations | |
| id: get-versions | |
| run: | | |
| python - <<'PY' >> "$GITHUB_OUTPUT" | |
| import json | |
| import tomllib | |
| from urllib.request import urlopen | |
| with urlopen("https://peps.python.org/api/release-cycle.json", timeout=30) as response: | |
| release_cycle = json.load(response) | |
| versions = [ | |
| { | |
| "branch": release.get("branch") or version, | |
| "python_version": "3.12" if version == "3.10" else "3", | |
| } | |
| for version, release in release_cycle.items() | |
| if release.get("status") not in {"end-of-life", "planned"} | |
| ] | |
| with urlopen( | |
| "https://raw.githubusercontent.com/python/docsbuild-scripts/main/config.toml", | |
| timeout=30, | |
| ) as response: | |
| config = tomllib.loads(response.read().decode("utf-8")) | |
| defaults = config.get("defaults", {}) | |
| default_in_prod = defaults.get("in_prod", True) | |
| default_sphinxopts = defaults.get("sphinxopts", []) | |
| languages = [] | |
| for language, language_config in config.get("languages", {}).items(): | |
| if not language_config.get("in_prod", default_in_prod): | |
| continue | |
| languages.append( | |
| { | |
| "language": language, | |
| "sphinxopts_json": json.dumps( | |
| language_config.get("sphinxopts", default_sphinxopts), | |
| ensure_ascii=False, | |
| ), | |
| } | |
| ) | |
| matrix = [{**version, **language} for version in versions for language in languages] | |
| print(f"versions={json.dumps(matrix, ensure_ascii=False)}") | |
| PY | |
| build: | |
| needs: get-versions | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.get-versions.outputs.versions) }} | |
| # Build jobs run concurrently and upload artifacts per version. The single | |
| # deploy job below collects all artifacts and publishes once, avoiding | |
| # concurrent git pushes/commit conflicts to gh-pages. | |
| uses: ./.github/workflows/build.yaml | |
| with: | |
| reference: ${{ matrix.branch }} | |
| python_version: ${{ matrix.python_version }} | |
| language: ${{ matrix.language }} | |
| sphinxopts_json: ${{ matrix.sphinxopts_json }} | |
| publish: ${{ 'false' }} | |
| deploy: | |
| # A single deploy job runs after all build matrix jobs complete. | |
| # Artifacts from every concurrent build job are merged here and published | |
| # once via actions/deploy-pages, eliminating concurrent gh-pages push conflicts. | |
| needs: build | |
| if: ${{ !cancelled() && needs.build.result != 'failure' && github.event_name != 'push' }} | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| - name: Configure Pages | |
| uses: actions/configure-pages@v5 | |
| - name: Checkout existing gh-pages content | |
| id: checkout-pages | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: gh-pages | |
| path: _site | |
| continue-on-error: true | |
| - name: Prepare site directory | |
| run: | | |
| # Remove git metadata; safe even if checkout above did not succeed | |
| rm -rf _site/.git | |
| - name: Download all build artifacts | |
| # Collect artifacts uploaded by all concurrent build matrix jobs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts/ | |
| - name: Copy new archives into site directory | |
| run: | | |
| # Copy generated archives (zip, tar.bz2, epub) into _site/<lang>/<major_minor>/, | |
| # excluding PDF build logs which are for debugging only. | |
| # Extract major.minor and language from filenames like: | |
| # python-3.14.0-docs-html.zip (English zip) | |
| # python-3.14.0-docs.epub (English epub) | |
| # python-3.14.0-fr-docs-html.zip (French zip) | |
| # python-3.14.0-fr-docs.epub (French epub) | |
| python - <<'PY' | |
| import re | |
| import shutil | |
| from pathlib import Path | |
| artifacts = Path("artifacts") | |
| site = Path("_site") | |
| name_re = re.compile( | |
| r"^python-(\d+\.\d+)[\d.]*(?:-(?!docs|pdf)([a-zA-Z][a-zA-Z0-9_]*))?-(?:docs|pdf)" | |
| ) | |
| for f in artifacts.rglob("*"): | |
| if not f.is_file(): | |
| continue | |
| name = f.name | |
| if not (name.endswith(".zip") or name.endswith(".tar.bz2") or name.endswith(".epub")): | |
| continue | |
| if re.search(r"-pdf-logs\.zip$", name): | |
| continue | |
| m = name_re.match(name) | |
| if not m: | |
| continue | |
| major_minor = m.group(1) | |
| lang = (m.group(2) or "en").replace("_", "-").lower() | |
| dest = site / lang / major_minor | |
| dest.mkdir(parents=True, exist_ok=True) | |
| shutil.copy2(f, dest / name) | |
| PY | |
| - name: Symlink 3 to stable version | |
| run: | | |
| # Create a relative symlink _site/en/3 -> <stable> (e.g. 3 -> 3.14), | |
| # pointing to the first "bugfix" (stable) version from release-cycle JSON. | |
| stable=$(curl -sf https://peps.python.org/api/release-cycle.json | \ | |
| jq -r '[to_entries[] | select(.value.status == "bugfix")] | first | .key') | |
| if [ -z "$stable" ]; then | |
| echo "Error: no stable (bugfix) version found in release-cycle JSON" >&2 | |
| exit 1 | |
| fi | |
| # Remove existing 3 directory or symlink before creating new symlink | |
| rm -rf _site/en/3 | |
| ln -s "$stable" _site/en/3 | |
| - name: Symlink bare version paths to en/ for backwards compatibility | |
| run: | | |
| # Create _site/<major_minor> -> en/<major_minor> symlinks so that old | |
| # URLs like /3.14/ continue to work after content moved to /en/3.14/. | |
| python - <<'PY' | |
| import re | |
| from pathlib import Path | |
| site = Path("_site") | |
| en_dir = site / "en" | |
| version_pattern = re.compile(r"^\d+\.\d+$") | |
| for version_dir in en_dir.iterdir(): | |
| if not version_dir.is_dir() or not version_pattern.match(version_dir.name): | |
| continue | |
| link = site / version_dir.name | |
| if not link.exists() and not link.is_symlink(): | |
| link.symlink_to(Path("en") / version_dir.name) | |
| PY | |
| - name: Generate per-version directory listing | |
| run: | | |
| python - <<'PY' | |
| import re | |
| from datetime import datetime, timezone | |
| from html import escape | |
| from pathlib import Path | |
| from urllib.parse import quote | |
| root = Path("_site") | |
| version_pattern = re.compile(r"^\d+\.\d+$") | |
| for lang_dir in sorted(p for p in root.iterdir() if p.is_dir()): | |
| for version_dir in sorted( | |
| p for p in lang_dir.iterdir() if p.is_dir() and version_pattern.match(p.name) | |
| ): | |
| files = sorted( | |
| p for p in version_dir.iterdir() if p.is_file() and p.name != "index.html" | |
| ) | |
| rows = [] | |
| for file_path in files: | |
| stat = file_path.stat() | |
| timestamp = datetime.fromtimestamp(stat.st_mtime, timezone.utc).strftime( | |
| "%Y-%m-%d %H:%M:%S UTC" | |
| ) | |
| rows.append( | |
| f'<tr><td><a href="{quote(file_path.name)}">{escape(file_path.name)}</a></td><td>{timestamp}</td><td>{stat.st_size}</td></tr>' | |
| ) | |
| relative_path = f"/{version_dir.relative_to(root).as_posix()}/" | |
| html = [ | |
| "<!doctype html>", | |
| '<html lang="en">', | |
| '<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Directory listing</title></head>', | |
| "<body>", | |
| f"<h1>Path: {escape(relative_path)}</h1>", | |
| "<table>", | |
| '<thead><tr><th scope="col">Filename</th><th scope="col">Timestamp (UTC)</th><th scope="col">Size (bytes)</th></tr></thead>', | |
| "<tbody>", | |
| *rows, | |
| "</tbody>", | |
| "</table>", | |
| "</body>", | |
| "</html>", | |
| ] | |
| (version_dir / "index.html").write_text("\n".join(html) + "\n", encoding="utf-8") | |
| PY | |
| - name: Upload Pages artifact | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: _site | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v4 |