From 856c60f0c1c47278593f3dd68d7ebcaad946ab00 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 May 2026 10:25:32 +0100 Subject: [PATCH 01/17] Refactor integration testing system to build a dashboard and run in CI --- .github/workflows/integration.yml | 87 ++++++ .gitignore | 5 + README.md | 108 +++++++- build_dashboard.py | 146 +++++++++++ packages.yaml | 76 ++++++ run_integration.py | 423 ++++++++++++++++++++++++++++++ status.py | 50 ++++ templates/_base.html | 12 + templates/_style.css | 29 ++ templates/cell.html | 25 ++ templates/index.html | 49 ++++ templates/variant.html | 42 +++ 12 files changed, 1040 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/integration.yml create mode 100644 build_dashboard.py create mode 100644 packages.yaml create mode 100644 run_integration.py create mode 100644 status.py create mode 100644 templates/_base.html create mode 100644 templates/_style.css create mode 100644 templates/cell.html create mode 100644 templates/index.html create mode 100644 templates/variant.html diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..a5858a3 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,87 @@ +name: integration-matrix + +on: + schedule: + - cron: '0 6 * * 0' # Sundays at 06:00 UTC + workflow_dispatch: + +permissions: + contents: write # to push gh-pages + +concurrency: + group: integration-matrix + cancel-in-progress: false + +jobs: + variant: + strategy: + fail-fast: false + matrix: + variant: [stable, latest, dev] + runs-on: ubuntu-latest + timeout-minutes: 240 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Cache ~/.cache/uv + uses: actions/cache@v4 + with: + path: ~/.cache/uv + key: uv-${{ runner.os }}-${{ matrix.variant }}-${{ hashFiles('packages.yaml') }} + restore-keys: | + uv-${{ runner.os }}-${{ matrix.variant }}- + uv-${{ runner.os }}- + + - name: Pre-install Python 3.12 + run: uv python install 3.12 + + - name: Install runtime deps + run: uv pip install --system packaging pyyaml + + - name: Run ${{ matrix.variant }} variant + run: python run_integration.py --variant ${{ matrix.variant }} + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.variant }} + path: results/${{ matrix.variant }}.json + + dashboard: + needs: variant + if: always() + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Install runtime deps + run: uv pip install --system jinja2 + + - name: Download all variant results + uses: actions/download-artifact@v4 + with: + pattern: results-* + merge-multiple: true + path: results/ + + - name: List results + run: ls -la results/ + + - name: Build dashboard + run: python build_dashboard.py + + - name: Publish to gh-pages + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./site + force_orphan: true + commit_message: Update integration dashboard diff --git a/.gitignore b/.gitignore index e31e2f6..40a230f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ .tox .hypothesis result_images +.tmp/ +results/ +site/ +__pycache__/ +*.pyc diff --git a/README.md b/README.md index 65f1d8e..6d182a9 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,103 @@ Integration testing for the Astropy ecosystem ============================================= -[![Integration Status](https://github.com/astropy/astropy-integration-testing/workflows/astropy_rc_basic/badge.svg)](https://github.com/astropy/astropy-integration-testing/actions) +[![Integration matrix](https://github.com/astropy/astropy-integration-testing/actions/workflows/integration.yml/badge.svg)](https://github.com/astropy/astropy-integration-testing/actions/workflows/integration.yml) -This repository is a way to do integration testing -across the Astropy ecosystem to ensure that the core and coordinated packages -work well together. +Cross-ecosystem integration tests for the Astropy core and coordinated +packages. Individual packages should still test against dev/pre-release +astropy in their own CI; the goal here is to catch issues that only +appear when many packages are installed together. -The tests here only do basic testings for those packages on Linux. -Individual packages should still do the due diligence to test against -dev and/or pre-release versions of `astropy` on their own to be sure. +The dashboard is published to +[astropy.github.io/astropy-integration-testing](https://astropy.github.io/astropy-integration-testing/) +after each scheduled run. -To run these tests on GitHub Action as a maintainer of this repo: +How it works +------------ -1. Goto Actions tab. -2. Select `astropy_rc_basic` job. -3. Click "Run workflow" dropdown and then the green "Run workflow" button. -4. A new run should kick off after a few seconds. Monitor the logs of this run. +Three CI jobs run on a schedule (and on `workflow_dispatch`), one per +astropy variant: + +| Variant | Astropy | Each package | +|----------|-----------------------------------------------------|----------------------------------------| +| `stable` | Latest non-pre-release on PyPI | Latest non-pre-release on PyPI | +| `latest` | Latest including pre-releases (`--prerelease=allow`)| Latest including pre-releases | +| `dev` | Latest dev wheel from the astropy/simple channel | `git+` (HEAD of main branch) | + +Within each job, a single shared venv is built and packages are +installed one at a time in a deterministic order (coordinated first, +alphabetical within each tier). If a package can't be installed +alongside the existing venv (e.g. it pins `astropy<7` but we already +installed astropy 8), it's skipped and recorded; the rest of the venv +is untouched. After installs, `pytest --pyargs ` runs for each +package that installed successfully. + +A fourth job downloads the three result JSONs and publishes the +dashboard to `gh-pages`. + +What's in the repo +------------------ + +| File | Purpose | +|-------------------------------------|------------------------------------------------------| +| `packages.yaml` | The list of packages tested (one block per package). | +| `run_integration.py` | Runs one variant: resolve specs, install, test, write `results/.json`. | +| `build_dashboard.py` | Reads `results/*.json`, renders `site/`. | +| `status.py` | Shared status vocabulary (used by both scripts). | +| `templates/` | HTML/CSS for the dashboard. | +| `.github/workflows/integration.yml` | The new matrix workflow (variant x3 + dashboard). | +| `tox.ini`, `sunpy_pytest.ini`, `.github/workflows/integration_testing.yml` | Legacy tox setup, kept for now. | + +Running locally +--------------- + +```bash +pip install jinja2 packaging pyyaml +# uv is required; see https://docs.astral.sh/uv/ + +# Run one variant. Each variant takes 30-90 min depending on package count. +python run_integration.py --variant stable + +# Or a single package, to iterate faster: +python run_integration.py --variant stable --packages reproject + +# Or a tier subset (default: all tiers run): +python run_integration.py --variant stable --tiers coordinated,other + +# Build the dashboard from whatever results/.json files exist: +python build_dashboard.py + +# Preview locally: +python -m http.server -d site 8000 +``` + +Results land in `results/.json`; the dashboard in `site/`. +Both directories are gitignored. + +Adding or disabling a package +----------------------------- + +Edit `packages.yaml`. Each entry takes: + +- `pypi_name` (the package's name on PyPI; also used as the row label) +- `tier` (label used for ordering and the `--tiers` filter; conventional + values are `coordinated`, `affiliated`, `other`) +- `module` (the top-level Python module name, for `pytest --pyargs`) +- `repo_url` (for the `dev` variant install) +- `install_extras` (list, e.g. `[test, all]`) +- `extra_deps` (optional list of extra packages to add to the install) +- `pytest_args` (optional list passed through to pytest; use `-k "not foo"` to skip tests) + +Every entry runs by default. Use `--tiers ` on the runner to +restrict to a tier subset (e.g. `--tiers coordinated`). + +Triggering a run from GitHub +---------------------------- + +1. Actions tab -> `integration-matrix` workflow. +2. "Run workflow" dropdown -> green button. +3. Three variant jobs run in parallel; the `dashboard` job waits for + them and publishes to `gh-pages`. + +The legacy `astropy_rc_basic` (tox-based) workflow is still present +and triggerable separately. diff --git a/build_dashboard.py b/build_dashboard.py new file mode 100644 index 0000000..e9c2621 --- /dev/null +++ b/build_dashboard.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +"""Build the static integration-matrix dashboard. + +Reads the per-variant JSONs from results/ (stable.json, latest.json, +dev.json - any missing variant just shows as missing cells) and emits: + + site/index.html Nx3 matrix + site/cells/__.html per-(package, variant) detail page + site/variants/.html per-variant detail with full freeze + +Publishing to gh-pages is done from CI via a published action; this +script only builds locally. +""" + +import argparse +import json +import shutil +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +import status + + +VARIANTS = status.VARIANTS + + +def _variant_meta(name, data): + if not data: + return {"name": name, "has_data": False} + return { + "name": name, + "has_data": True, + "astropy_version": data["astropy"]["version"], + "extra_index_urls": data["astropy"].get("extra_index_urls") or [], + "python_version": data["python_version"], + "started_at": data["started_at"], + "finished_at": data["finished_at"], + } + + +def _ordered_packages(by_variant): + """Return package names in the order they first appear across variants.""" + seen = [] + seen_set = set() + for v in VARIANTS: + if not by_variant[v]: + continue + for entry in by_variant[v]["packages"]: + if entry["name"] not in seen_set: + seen.append(entry["name"]) + seen_set.add(entry["name"]) + return seen + + +def _make_rows(by_variant, names): + rows = [] + for name in names: + cells = {} + for v in VARIANTS: + data = by_variant[v] + entry = None + if data: + entry = next((e for e in data["packages"] if e["name"] == name), None) + if entry is None: + cells[v] = {"status": "missing", "label": "-", + "detail_filename": "", "resolved_version": ""} + else: + cells[v] = { + **status.cell_badge(entry), + "detail_filename": f"{name}__{v}.html", + "resolved_version": entry.get("resolved_version", ""), + } + rows.append({"name": name, "cells": cells}) + return rows + + +def build(results_dir, output_dir, templates_dir): + results_dir = Path(results_dir) + output_dir = Path(output_dir) + if output_dir.exists(): + shutil.rmtree(output_dir) + (output_dir / "cells").mkdir(parents=True) + (output_dir / "variants").mkdir(parents=True) + + env = Environment( + loader=FileSystemLoader(str(templates_dir)), + autoescape=select_autoescape(["html"]), + ) + index_tpl = env.get_template("index.html") + cell_tpl = env.get_template("cell.html") + variant_tpl = env.get_template("variant.html") + + by_variant = { + v: (json.loads((results_dir / f"{v}.json").read_text()) + if (results_dir / f"{v}.json").exists() else None) + for v in VARIANTS + } + + names = _ordered_packages(by_variant) + rows = _make_rows(by_variant, names) + variants_meta = [_variant_meta(v, by_variant[v]) for v in VARIANTS] + + (output_dir / "index.html").write_text( + index_tpl.render(variants=variants_meta, rows=rows) + ) + n_pages = 1 + + for v in VARIANTS: + data = by_variant[v] + if not data: + continue + for entry in data["packages"]: + (output_dir / "cells" / f"{entry['name']}__{v}.html").write_text( + cell_tpl.render( + entry=entry, + variant=v, + status=status.cell_badge(entry), + astropy_version=data["astropy"]["version"], + ) + ) + n_pages += 1 + (output_dir / "variants" / f"{v}.html").write_text( + variant_tpl.render( + variant=v, + data=data, + install_badge=status.INSTALL_BADGE, + test_badge=status.TEST_BADGE, + ) + ) + n_pages += 1 + + print(f"Wrote {n_pages} pages to {output_dir}/") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--results-dir", default="results") + ap.add_argument("--output", default="site") + ap.add_argument("--templates-dir", default="templates") + args = ap.parse_args() + build(args.results_dir, args.output, args.templates_dir) + + +if __name__ == "__main__": + main() diff --git a/packages.yaml b/packages.yaml new file mode 100644 index 0000000..44c8d32 --- /dev/null +++ b/packages.yaml @@ -0,0 +1,76 @@ +# Packages tested by run_integration.py. +# +# One block per package = one row in the dashboard. For each astropy +# variant (stable / latest / dev), the runner builds a shared venv, +# installs astropy first then each package in turn (skipping any that +# the resolver can't satisfy alongside the existing set), then runs +# `pytest --pyargs ` for each one that installed. +# +# `tier` is purely a category label used for ordering and the optional +# `--tiers` CLI filter. Conventions: `coordinated` for the official +# coordinated set, `affiliated` for affiliated packages, `other` for +# anything else worth testing. + +packages: + - pypi_name: asdf_astropy + tier: coordinated + module: asdf_astropy + repo_url: https://github.com/astropy/asdf-astropy.git + install_extras: [test] + + - pypi_name: astropy_healpix + tier: coordinated + module: astropy_healpix + repo_url: https://github.com/astropy/astropy-healpix.git + install_extras: [test] + + - pypi_name: astroquery + tier: coordinated + module: astroquery + repo_url: https://github.com/astropy/astroquery.git + install_extras: [test, all] + + - pypi_name: ccdproc + tier: coordinated + module: ccdproc + repo_url: https://github.com/astropy/ccdproc.git + install_extras: [test, all] + extra_deps: [psutil] + + - pypi_name: photutils + tier: coordinated + module: photutils + repo_url: https://github.com/astropy/photutils.git + install_extras: [test, all] + + - pypi_name: regions + tier: coordinated + module: regions + repo_url: https://github.com/astropy/regions.git + install_extras: [test, all] + + - pypi_name: reproject + tier: coordinated + module: reproject + repo_url: https://github.com/astropy/reproject.git + install_extras: [test, all] + extra_deps: [gwcs] + + - pypi_name: specreduce + tier: coordinated + module: specreduce + repo_url: https://github.com/astropy/specreduce.git + install_extras: [test] + + - pypi_name: specutils + tier: coordinated + module: specutils + repo_url: https://github.com/astropy/specutils.git + install_extras: [all, test] + + - pypi_name: sunpy + tier: other + module: sunpy + repo_url: https://github.com/sunpy/sunpy.git + install_extras: [tests, all] + pytest_args: ["-c", "sunpy_pytest.ini"] diff --git a/run_integration.py b/run_integration.py new file mode 100644 index 0000000..af42ce6 --- /dev/null +++ b/run_integration.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python +"""Run one variant of the astropy ecosystem integration matrix. + +Creates a single shared venv, installs astropy first then each package +from packages.yaml in order, recording per-package install outcome +(installed / skipped / install-fail / no-spec). Then runs +`pytest --pyargs ` for each successfully-installed package. +Writes results/.json with the full venv freeze and per-package +data. + +Usage: + python run_integration.py --variant stable + python run_integration.py --variant latest + python run_integration.py --variant dev + python run_integration.py --variant stable --packages reproject,sunpy + python run_integration.py --variant stable --tiers coordinated +""" + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.request +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path + + +try: + import yaml + from packaging.version import Version, InvalidVersion +except ImportError: + sys.exit("This script requires 'pyyaml' and 'packaging'. " + "Install with: pip install pyyaml packaging") + +import status + + +PYPI_JSON_URL = "https://pypi.org/pypi/{name}/json" +ASTROPY_NIGHTLY_INDEX = "https://pypi.anaconda.org/astropy/simple" +PYTHON_VERSION = "3.12" + + +def _http_json(url, timeout=20): + with urllib.request.urlopen(url, timeout=timeout) as r: + return json.load(r) + + +def _http_text(url, timeout=20): + with urllib.request.urlopen(url, timeout=timeout) as r: + return r.read().decode("utf-8", "replace") + + +def _version_key(v): + try: + return Version(v) + except InvalidVersion: + return Version("0") + + +def _is_prerelease(v): + try: + return Version(v).is_prerelease + except InvalidVersion: + return True + + +def _yanked(files): + return all(f.get("yanked") for f in files) if files else True + + +def latest_pypi(name, include_prereleases): + info = _http_json(PYPI_JSON_URL.format(name=name)) + versions = [] + for ver, files in info["releases"].items(): + if _yanked(files): + continue + if not include_prereleases and _is_prerelease(ver): + continue + versions.append(ver) + if not versions and not include_prereleases: + # Fallback: package has only pre-releases on PyPI + return latest_pypi(name, include_prereleases=True) + if not versions: + return None + return sorted(versions, key=_version_key)[-1] + + +def latest_astropy_nightly(): + """Return the latest astropy version from the astropy/simple channel.""" + html = _http_text(f"{ASTROPY_NIGHTLY_INDEX}/astropy/") + versions = set() + for href in re.findall(r'href="([^"]+\.whl)"', html): + fname = href.rsplit("/", 1)[-1] + parts = fname[: -len(".whl")].split("-") + if len(parts) >= 2 and parts[0].lower() == "astropy": + versions.add(parts[1]) + if not versions: + sys.exit("No astropy nightly wheels found on the astropy/simple channel.") + return sorted(versions, key=_version_key)[-1] + + +def _extras_suffix(pkg): + extras = pkg.get("install_extras") or [] + return "[" + ",".join(extras) + "]" if extras else "" + + +def resolve_specs(packages, variant): + """Resolve the astropy spec and per-package install specs for the variant.""" + if variant == "dev": + astropy_ver = latest_astropy_nightly() + astropy = { + "install": f"astropy=={astropy_ver}", + "version": astropy_ver, + "extra_index_urls": [ASTROPY_NIGHTLY_INDEX], + "prerelease_strategy": "if-necessary-or-explicit", + } + + def pkg_spec(pkg): + repo = pkg.get("repo_url") + if not repo: + return None, None + return f"{pkg['pypi_name']}{_extras_suffix(pkg)} @ git+{repo}", None + + else: + include_pre = (variant == "latest") + astropy_ver = latest_pypi("astropy", include_prereleases=include_pre) + astropy = { + "install": f"astropy=={astropy_ver}", + "version": astropy_ver, + "extra_index_urls": [], + "prerelease_strategy": "allow" if include_pre else "if-necessary-or-explicit", + } + + def pkg_spec(pkg): + ver = latest_pypi(pkg["pypi_name"], include_prereleases=include_pre) + if not ver: + return None, None + return f"{pkg['pypi_name']}{_extras_suffix(pkg)}=={ver}", ver + + pkg_specs = [] + for pkg in packages: + spec, target = pkg_spec(pkg) + pkg_specs.append((pkg, spec, target)) + return astropy, pkg_specs + + +def _resolver_conflict(stderr): + """True if the install stderr looks like a uv resolver conflict.""" + s = stderr.lower() + keywords = ( + "no solution found", + "incompatible", + "conflict", + "could not find a version", + "no matching distribution", + "no version of", + ) + return any(k in s for k in keywords) + + +def ensure_python(version): + proc = subprocess.run(["uv", "python", "find", version], + capture_output=True, text=True, timeout=60) + if proc.returncode == 0: + return proc.stdout.strip() + inst = subprocess.run(["uv", "python", "install", version], + capture_output=True, text=True, timeout=600) + if inst.returncode != 0: + sys.exit(f"uv python install {version}: {inst.stderr.strip()}") + proc = subprocess.run(["uv", "python", "find", version], + capture_output=True, text=True, timeout=60) + if proc.returncode != 0: + sys.exit(f"uv python find {version}: {proc.stderr.strip()}") + return proc.stdout.strip() + + +def _venv_python_version(python): + proc = subprocess.run( + [python, "-c", "import sys; print('.'.join(str(x) for x in sys.version_info[:3]))"], + capture_output=True, text=True, timeout=30, + ) + return proc.stdout.strip() if proc.returncode == 0 else "" + + +def _pkg_version(python, name): + proc = subprocess.run( + [python, "-c", + "import importlib.metadata as md, sys; print(md.version(sys.argv[1]))", + name], + capture_output=True, text=True, timeout=30, + ) + return proc.stdout.strip() if proc.returncode == 0 else "" + + +def _freeze(python): + proc = subprocess.run(["uv", "pip", "freeze", "--python", python], + capture_output=True, text=True, timeout=60) + out = {} + if proc.returncode == 0: + for line in proc.stdout.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + if "==" in line: + n, v = line.split("==", 1) + out[n.lower()] = v + return out + + +def _load_packages(path): + raw = yaml.safe_load(Path(path).read_text()) or {} + return list(raw.get("packages", [])) + + +# Ordering of tiers when installing/displaying. Unknown tiers sort last. +TIER_RANK = {"coordinated": 0, "affiliated": 1, "other": 2} + + +def _install_order(packages): + """Coordinated before affiliated before other, alphabetical within each tier.""" + return sorted(packages, key=lambda p: ( + TIER_RANK.get(p.get("tier", "coordinated"), 9), + p["pypi_name"].lower(), + )) + + +def _run_install(install_cmd, timeout): + """Wrap subprocess.run with TimeoutExpired catch. Returns (rc, stderr_or_msg).""" + try: + proc = subprocess.run(install_cmd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + return None, "timeout during install" + return proc.returncode, (proc.stderr or proc.stdout)[-3000:] + + +def run_variant(variant, packages, repo_root, results_dir, timeouts): + astropy, pkg_specs = resolve_specs(packages, variant) + print(f"\n=== Variant: {variant} ===") + print(f" astropy: {astropy['install']}") + for pkg, spec, target in pkg_specs: + print(f" {pkg['pypi_name']:<20} {spec or '(no install spec)'}") + + result = { + "variant": variant, + "started_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "finished_at": "", + "astropy": { + "install_spec": astropy["install"], + "version": astropy["version"], + "extra_index_urls": astropy["extra_index_urls"], + "prerelease_strategy": astropy["prerelease_strategy"], + }, + "python_version": "", + "fatal_error": "", + "installed_deps": {}, + "packages": [], + } + out_path = results_dir / f"{variant}.json" + + Path(repo_root, ".tmp").mkdir(exist_ok=True) + tmpdir = tempfile.mkdtemp(prefix=f"int-{variant}-", + dir=str(Path(repo_root, ".tmp").resolve())) + + try: + py_path = ensure_python(PYTHON_VERSION) + venv = os.path.join(tmpdir, "venv") + venv_proc = subprocess.run(["uv", "venv", venv, "-p", py_path, "-q"], + capture_output=True, text=True, timeout=120) + if venv_proc.returncode != 0: + result["fatal_error"] = f"uv venv: {(venv_proc.stderr or venv_proc.stdout)[-500:]}" + return result, out_path + python = os.path.join(venv, "bin", "python") + result["python_version"] = _venv_python_version(python) + + common = ["uv", "pip", "install", "--python", python, "-q"] + for url in astropy["extra_index_urls"]: + common += ["--extra-index-url", url] + common += [f"--prerelease={astropy['prerelease_strategy']}"] + + print("\nInstalling astropy + pytest...") + rc, err = _run_install(common + [astropy["install"], "pytest", "pytest-timeout"], + timeouts["install"]) + if rc != 0: + print(" FATAL: astropy install failed") + result["fatal_error"] = err + return result, out_path + result["astropy"]["version"] = _pkg_version(python, "astropy") or result["astropy"]["version"] + + installed_pkgs = [] + for pkg, install_spec, target_version in pkg_specs: + entry = { + "name": pkg["pypi_name"], + "tier": pkg.get("tier", "coordinated"), + "module": pkg.get("module", pkg["pypi_name"]), + "install_spec": install_spec, + "target_version": target_version, + "resolved_version": "", + "install_status": "", # installed | skipped | install-fail | no-spec + "install_error": "", + "test_status": "", # pass | fail | no-tests | timeout | not-run + "tests_passed": None, + "test_output": "", + "duration": 0, + } + if install_spec is None: + entry["install_status"] = status.NO_SPEC + entry["install_error"] = "no install spec (missing repo_url, or no PyPI release)" + result["packages"].append(entry) + continue + + print(f"\nInstalling {pkg['pypi_name']}...") + install_cmd = common + [install_spec] + (pkg.get("extra_deps") or []) + rc, err = _run_install(install_cmd, timeouts["install"]) + if rc == 0: + entry["install_status"] = status.INSTALLED + entry["resolved_version"] = _pkg_version(python, pkg["pypi_name"]) + print(f" installed at {entry['resolved_version']}") + installed_pkgs.append((pkg, entry)) + else: + entry["install_error"] = err + if _resolver_conflict(err): + entry["install_status"] = status.SKIPPED + print(" skipped (resolver conflict)") + else: + entry["install_status"] = status.INSTALL_FAIL + print(" install failed") + result["packages"].append(entry) + + result["installed_deps"] = _freeze(python) + + for pkg, entry in installed_pkgs: + print(f"\nTesting {pkg['pypi_name']}...") + module = entry["module"] + cmd = [python, "-m", "pytest", "--pyargs", module] + cmd += pkg.get("pytest_args", []) + cmd += ["--timeout=120", "-q", "--tb=line", "--no-header"] + + env = {**os.environ, "MPLBACKEND": "Agg", "DISPLAY": ""} + start = time.time() + try: + proc = subprocess.run(cmd, capture_output=True, text=True, + timeout=timeouts["test"], cwd=repo_root, env=env) + except subprocess.TimeoutExpired: + entry["test_status"] = status.TIMEOUT + entry["test_output"] = "timeout" + entry["duration"] = round(time.time() - start, 1) + print(f" timeout after {entry['duration']}s") + continue + entry["duration"] = round(time.time() - start, 1) + out = proc.stdout[-50000:] + if proc.stderr: + out += "\n--- stderr ---\n" + proc.stderr[-5000:] + entry["test_output"] = out + + no_module = ( + proc.returncode == 4 + and "module or package not found" in (proc.stdout + proc.stderr) + ) + if proc.returncode == 5 or no_module: + entry["test_status"] = status.NO_TESTS + elif proc.returncode == 0: + entry["test_status"] = status.PASS + entry["tests_passed"] = True + else: + entry["test_status"] = status.FAIL + entry["tests_passed"] = False + print(f" {entry['test_status']} in {entry['duration']}s") + + finally: + result["finished_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds") + out_path.write_text(json.dumps(result, indent=2)) + shutil.rmtree(tmpdir, ignore_errors=True) + + return result, out_path + + +def _counts(result): + install = Counter(e["install_status"] for e in result["packages"]) + test = Counter(e["test_status"] for e in result["packages"] if e["test_status"]) + return {"install": dict(install), "test": dict(test)} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--config", default="packages.yaml") + ap.add_argument("--results-dir", default="results") + ap.add_argument("--variant", choices=status.VARIANTS, required=True) + ap.add_argument("--packages", help="Comma-separated subset of package names to run") + ap.add_argument("--tiers", + help="Comma-separated subset of tiers to run (e.g. 'coordinated,other'); " + "default: all tiers") + ap.add_argument("--timeout-install", type=int, default=900) + ap.add_argument("--timeout-test", type=int, default=1800) + args = ap.parse_args() + + packages = _load_packages(args.config) + if args.tiers: + wanted_tiers = {t.strip() for t in args.tiers.split(",") if t.strip()} + packages = [p for p in packages if p.get("tier", "coordinated") in wanted_tiers] + if args.packages: + wanted = {n.strip() for n in args.packages.split(",") if n.strip()} + packages = [p for p in packages if p["pypi_name"] in wanted] + packages = _install_order(packages) + + repo_root = Path.cwd() + results_dir = Path(args.results_dir) + results_dir.mkdir(parents=True, exist_ok=True) + + timeouts = {"install": args.timeout_install, "test": args.timeout_test} + result, out_path = run_variant(args.variant, packages, repo_root, results_dir, timeouts) + print(f"\nDone: {_counts(result)}") + print(f"Wrote {out_path}") + + +if __name__ == "__main__": + main() diff --git a/status.py b/status.py new file mode 100644 index 0000000..41065e3 --- /dev/null +++ b/status.py @@ -0,0 +1,50 @@ +"""Shared status vocabulary for integration-test results. + +Both run_integration.py (which writes statuses into results JSON) and +build_dashboard.py (which renders them into the dashboard) use these +constants and helpers, so the strings are defined in exactly one place. +""" + +# The three astropy variants. Listed once here so both scripts agree. +VARIANTS = ("stable", "latest", "dev") + +# install_status values, used in results/.json +INSTALLED = "installed" +SKIPPED = "skipped" # resolver couldn't satisfy alongside the existing venv +INSTALL_FAIL = "install-fail" # install itself crashed (build error, network, etc.) +NO_SPEC = "no-spec" # couldn't build an install spec (no repo_url / no PyPI release) + +# test_status values +PASS = "pass" +FAIL = "fail" +NO_TESTS = "no-tests" +TIMEOUT = "timeout" + + +# Tabular maps used on the per-variant page. +INSTALL_BADGE = { + INSTALLED: {"status": "pass", "label": "installed"}, + SKIPPED: {"status": "skipped", "label": "skipped"}, + INSTALL_FAIL: {"status": "install-fail", "label": "install fail"}, + NO_SPEC: {"status": "missing", "label": "no spec"}, + "": {"status": "missing", "label": "-"}, +} + +TEST_BADGE = { + PASS: {"status": "pass", "label": "PASS"}, + FAIL: {"status": "fail", "label": "FAIL"}, + NO_TESTS: {"status": "no-tests", "label": "no tests"}, + TIMEOUT: {"status": "timeout", "label": "timeout"}, + "": {"status": "missing", "label": "-"}, +} + + +def cell_badge(entry): + """Map a result entry to the {status, label} for its matrix cell. + + Install failures (including no-spec) take precedence over test status. + """ + install = entry.get("install_status", "") + if install != INSTALLED: + return INSTALL_BADGE.get(install, INSTALL_BADGE[""]) + return TEST_BADGE.get(entry.get("test_status", ""), TEST_BADGE[""]) diff --git a/templates/_base.html b/templates/_base.html new file mode 100644 index 0000000..769f3ce --- /dev/null +++ b/templates/_base.html @@ -0,0 +1,12 @@ + + + + +{% block title %}{% endblock %} + + + +{% block nav %}{% endblock %} +{% block body %}{% endblock %} + + diff --git a/templates/_style.css b/templates/_style.css new file mode 100644 index 0000000..089e281 --- /dev/null +++ b/templates/_style.css @@ -0,0 +1,29 @@ +body { font-family: -apple-system, system-ui, sans-serif; margin: 2em; line-height: 1.5; color: #1a1a1a; max-width: 1200px; } +h1, h2 { margin-top: 0; } +a { color: #0366d6; text-decoration: none; } +a:hover { text-decoration: underline; } +table { border-collapse: collapse; } +th, td { border: 1px solid #ddd; padding: 6px 10px; text-align: left; vertical-align: top; } +th { background: #f6f8fa; } +.meta dl { margin: 0.3em 0; display: grid; grid-template-columns: max-content 1fr; gap: 0.2em 1em; color: #444; } +.meta dt { font-weight: 600; } +.pill { display: inline-block; padding: 2px 10px; border-radius: 12px; font-size: 0.85em; font-weight: 600; min-width: 4em; text-align: center; } +.pill.pass { background: #dcffe4; color: #22863a; } +.pill.fail { background: #ffdce0; color: #cb2431; } +.pill.no-tests { background: #eef2f6; color: #6a737d; } +.pill.skipped { background: #e6e9ee; color: #6a737d; font-style: italic; } +.pill.install-fail { background: #fff1d6; color: #b08000; } +.pill.timeout { background: #ece2ff; color: #5b3fb8; } +.pill.missing { background: #f1f1f1; color: #999; } +a.cell-link { text-decoration: none; } +a.cell-link:hover .pill { filter: brightness(0.94); } +.matrix tr:hover td { background: #fafafa; } +.pkg-name { font-weight: 600; } +pre { background: #1e1e1e; color: #d4d4d4; padding: 1em; border-radius: 6px; + overflow-x: auto; font-size: 0.85em; white-space: pre-wrap; word-wrap: break-word; max-height: 50em; } +details summary { cursor: pointer; padding: 4px 0; font-weight: 600; } +.meta-table td { padding: 4px 10px; } +.meta-table { width: auto; } +nav { margin-bottom: 1.5em; font-size: 0.9em; } +.legend { font-size: 0.85em; color: #6a737d; margin-top: 1em; } +.legend .pill { margin: 0 0.3em; } diff --git a/templates/cell.html b/templates/cell.html new file mode 100644 index 0000000..3e64972 --- /dev/null +++ b/templates/cell.html @@ -0,0 +1,25 @@ +{% extends "_base.html" %} +{% block title %}{{ entry.name }} - {{ variant }}{% endblock %} +{% block nav %}{% endblock %} +{% block body %} +

{{ entry.name }} in {{ variant }} venv

+ + + + + + + + +
Install status{{ status.label }}
Resolved version{{ entry.resolved_version or entry.target_version or '-' }}
Astropy version{{ astropy_version }} (variant: {{ variant }})
Install spec{{ entry.install_spec or '(none)' }}
Tier{{ entry.tier }}
Test status{{ entry.test_status or '-' }}
Duration{{ entry.duration }}s
+{% if entry.install_status in ("skipped", "install-fail", "no-spec") and entry.install_error %} +

{% if entry.install_status == "skipped" %}Resolver conflict + {%- elif entry.install_status == "install-fail" %}Install error + {%- else %}Spec error{% endif %}

+
{{ entry.install_error }}
+{% endif %} +{% if entry.test_output %} +

Test output

+
{{ entry.test_output }}
+{% endif %} +{% endblock %} diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..ef23e41 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,49 @@ +{% extends "_base.html" %} +{% block title %}Astropy integration matrix{% endblock %} +{% block body %} +

Astropy ecosystem integration matrix

+
+ {% for v in variants %} +
{{ v.name }}
+
{% if v.has_data %} + astropy {{ v.astropy_version }}{% if v.extra_index_urls %} (nightly){% endif %}, + Python {{ v.python_version }}, + started {{ v.started_at }}{% if v.finished_at %}, finished {{ v.finished_at }}{% endif %} · + variant detail + {% else %}no results{% endif %}
+ {% endfor %} +
+

Each row is a package. Each column is an astropy variant. Click any badge for the package's test log in that variant; click "variant detail" for the variant's full pip freeze and per-package install outcome.

+ + + + {% for v in variants %}{% endfor %} + + +{% for row in rows %} + + + {% for v in variants %} + {% set cell = row.cells[v.name] %} + + {% endfor %} + +{% endfor %} + +
Package{{ v.name }}
{{ row.name }} + {% if cell.detail_filename %} + {{ cell.label }} + {% else %}{{ cell.label }}{% endif %} + {% if cell.resolved_version %}
{{ cell.resolved_version }}{% endif %} +
+
Legend: + pass + fail + timeout + no tests + skipped + install fail + missing +
+

skipped: resolver couldn't satisfy this package alongside the rest of the venv. install fail: install step crashed for some other reason (build error, etc.).

+{% endblock %} diff --git a/templates/variant.html b/templates/variant.html new file mode 100644 index 0000000..7e1c234 --- /dev/null +++ b/templates/variant.html @@ -0,0 +1,42 @@ +{% extends "_base.html" %} +{% block title %}{{ variant }} variant{% endblock %} +{% block nav %}{% endblock %} +{% block body %} +

Variant: {{ variant }}

+ + + + + + + +
Astropy{{ data.astropy.version }} ({{ data.astropy.install_spec }})
Extra index URLs{% if data.astropy.extra_index_urls %}{% for u in data.astropy.extra_index_urls %}{{ u }}{% if not loop.last %}
{% endif %}{% endfor %}{% else %}(none){% endif %}
Pre-release strategy{{ data.astropy.prerelease_strategy }}
Python{{ data.python_version }}
Started{{ data.started_at }}
Finished{{ data.finished_at or "-" }}
+ +{% if data.fatal_error %} +

Fatal error (astropy install failed; no packages were tested)

+
{{ data.fatal_error }}
+{% endif %} + +

Per-package install & test outcome

+ + + +{% for entry in data.packages %} +{% set istat = install_badge[entry.install_status] %} +{% set tstat = test_badge[entry.test_status] %} + + + + + + + +{% endfor %} + +
PackageInstallResolvedTestDuration
{{ entry.name }}{{ istat.label }}{{ entry.resolved_version or "-" }}{% if entry.test_status %}{{ tstat.label }}{% else %}-{% endif %}{{ entry.duration }}s
+ +
Full venv freeze ({{ data.installed_deps|length }} packages) + +{% for name, ver in data.installed_deps.items()|sort %}{% endfor %} +
{{ name }}{{ ver }}
+{% endblock %} From d7e7c8151cf0da2f34d665b7706ceaee2452e011 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 May 2026 11:45:03 +0100 Subject: [PATCH 02/17] Add ability to preview page in PRs --- .github/workflows/preview.yml | 115 ++++++++++++++++++++++++++++++++++ README.md | 21 +++++++ build_dashboard.py | 61 +++++++++++++++--- run_integration.py | 8 +-- templates/single_page.html | 66 +++++++++++++++++++ 5 files changed, 259 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/preview.yml create mode 100644 templates/single_page.html diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml new file mode 100644 index 0000000..5ef1e4a --- /dev/null +++ b/.github/workflows/preview.yml @@ -0,0 +1,115 @@ +name: dashboard-preview + +on: + pull_request: + +permissions: + contents: read + actions: read # to find and download the latest main run's results + pull-requests: write # to (re)post the preview link comment + +concurrency: + group: dashboard-preview-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Install runtime deps + run: uv pip install --system jinja2 + + - name: Find latest successful main run + id: latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + run_id=$(gh run list \ + --workflow=integration.yml \ + --branch=main \ + --status=success \ + --limit=1 \ + --json databaseId \ + --jq '.[0].databaseId // empty') + if [ -z "$run_id" ]; then + echo "No successful main integration runs yet; preview will render with no results." + else + echo "Using main run $run_id" + fi + echo "run_id=$run_id" >> "$GITHUB_OUTPUT" + + - name: Download main run results + if: steps.latest.outputs.run_id != '' + uses: actions/download-artifact@v8 + with: + pattern: results-* + merge-multiple: true + path: results/ + run-id: ${{ steps.latest.outputs.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build single-page dashboard + run: python build_dashboard.py --single-page + + - name: Upload dashboard as non-zipped artifact + id: upload + uses: actions/upload-artifact@v7 + with: + name: dashboard-preview + path: site/dashboard.html + archive: false + # Keep one preview around per PR; the new upload supersedes the old one. + overwrite: true + + - name: Comment preview link on PR + if: github.event.pull_request.head.repo.full_name == github.repository + continue-on-error: true + uses: actions/github-script@v7 + env: + ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} + MAIN_RUN_ID: ${{ steps.latest.outputs.run_id }} + with: + script: | + const url = process.env.ARTIFACT_URL; + const mainRun = process.env.MAIN_RUN_ID; + const dataSource = mainRun + ? `Rendered against the latest successful main integration run (#${mainRun}).` + : `No main integration runs have completed yet, so the matrix shows no results.`; + const body = [ + "**Dashboard preview**: " + url, + "", + dataSource, + "", + "Rebuilt on every push to this PR. Open the link above, then click `dashboard.html` to view in the browser.", + ].join("\n"); + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const tag = "**Dashboard preview**"; + const existing = comments.find(c => + c.user.type === "Bot" && c.body && c.body.startsWith(tag)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } diff --git a/README.md b/README.md index 6d182a9..7622ed6 100644 --- a/README.md +++ b/README.md @@ -101,3 +101,24 @@ Triggering a run from GitHub The legacy `astropy_rc_basic` (tox-based) workflow is still present and triggerable separately. + +PR previews +----------- + +`dashboard-preview` runs on every pull request. It downloads the +results JSONs from the most recent successful main `integration-matrix` +run, builds a single-page `dashboard.html` with `build_dashboard.py +--single-page`, and uploads it as a non-zipped artifact +(`actions/upload-artifact@v7` with `archive: false`). A bot comment +on the PR links to the artifact; clicking through opens the page +directly in the browser. + +This is fast (under a minute) because it doesn't re-run the test +matrix; it only re-renders the dashboard with the PR's changes to +`packages.yaml`, `build_dashboard.py`, or `templates/` applied to +the latest real data. To actually validate a new package install, +trigger the full `integration-matrix` workflow manually. + +Preview is skipped for pull requests from forked repositories (the +build still runs but the PR comment is suppressed; reviewers can +still open the artifact link from the workflow run page). diff --git a/build_dashboard.py b/build_dashboard.py index e9c2621..c7cc557 100644 --- a/build_dashboard.py +++ b/build_dashboard.py @@ -75,32 +75,72 @@ def _make_rows(by_variant, names): return rows -def build(results_dir, output_dir, templates_dir): +def _issues(by_variant): + """List non-passing cells with a one-line error excerpt where available.""" + out = [] + for v in VARIANTS: + data = by_variant[v] + if not data: + continue + for entry in data["packages"]: + badge = status.cell_badge(entry) + if badge["status"] in ("pass", "missing"): + continue + excerpt = "" + for line in (entry.get("install_error") or "").splitlines(): + line = line.strip() + if line: + excerpt = line[:200] + break + out.append({ + "name": entry["name"], + "variant": v, + "status": badge["status"], + "label": badge["label"], + "error_excerpt": excerpt, + }) + return out + + +def build(results_dir, output_dir, templates_dir, single_page=False): results_dir = Path(results_dir) output_dir = Path(output_dir) if output_dir.exists(): shutil.rmtree(output_dir) - (output_dir / "cells").mkdir(parents=True) - (output_dir / "variants").mkdir(parents=True) + output_dir.mkdir(parents=True) env = Environment( loader=FileSystemLoader(str(templates_dir)), autoescape=select_autoescape(["html"]), ) - index_tpl = env.get_template("index.html") - cell_tpl = env.get_template("cell.html") - variant_tpl = env.get_template("variant.html") by_variant = { v: (json.loads((results_dir / f"{v}.json").read_text()) if (results_dir / f"{v}.json").exists() else None) for v in VARIANTS } - names = _ordered_packages(by_variant) rows = _make_rows(by_variant, names) variants_meta = [_variant_meta(v, by_variant[v]) for v in VARIANTS] + if single_page: + single_tpl = env.get_template("single_page.html") + (output_dir / "dashboard.html").write_text( + single_tpl.render( + variants=variants_meta, + rows=rows, + issues=_issues(by_variant), + ) + ) + print(f"Wrote dashboard.html to {output_dir}/") + return + + (output_dir / "cells").mkdir() + (output_dir / "variants").mkdir() + index_tpl = env.get_template("index.html") + cell_tpl = env.get_template("cell.html") + variant_tpl = env.get_template("variant.html") + (output_dir / "index.html").write_text( index_tpl.render(variants=variants_meta, rows=rows) ) @@ -138,8 +178,13 @@ def main(): ap.add_argument("--results-dir", default="results") ap.add_argument("--output", default="site") ap.add_argument("--templates-dir", default="templates") + ap.add_argument("--single-page", action="store_true", + help="Render a single self-contained dashboard.html (no cell or " + "variant pages). Suitable for serving as a non-zipped GH " + "Actions artifact in PR previews.") args = ap.parse_args() - build(args.results_dir, args.output, args.templates_dir) + build(args.results_dir, args.output, args.templates_dir, + single_page=args.single_page) if __name__ == "__main__": diff --git a/run_integration.py b/run_integration.py index af42ce6..854c56d 100644 --- a/run_integration.py +++ b/run_integration.py @@ -236,7 +236,7 @@ def _run_install(install_cmd, timeout): proc = subprocess.run(install_cmd, capture_output=True, text=True, timeout=timeout) except subprocess.TimeoutExpired: return None, "timeout during install" - return proc.returncode, (proc.stderr or proc.stdout)[-3000:] + return proc.returncode, (proc.stderr or proc.stdout) def run_variant(variant, packages, repo_root, results_dir, timeouts): @@ -273,7 +273,7 @@ def run_variant(variant, packages, repo_root, results_dir, timeouts): venv_proc = subprocess.run(["uv", "venv", venv, "-p", py_path, "-q"], capture_output=True, text=True, timeout=120) if venv_proc.returncode != 0: - result["fatal_error"] = f"uv venv: {(venv_proc.stderr or venv_proc.stdout)[-500:]}" + result["fatal_error"] = f"uv venv: {venv_proc.stderr or venv_proc.stdout}" return result, out_path python = os.path.join(venv, "bin", "python") result["python_version"] = _venv_python_version(python) @@ -353,9 +353,9 @@ def run_variant(variant, packages, repo_root, results_dir, timeouts): print(f" timeout after {entry['duration']}s") continue entry["duration"] = round(time.time() - start, 1) - out = proc.stdout[-50000:] + out = proc.stdout if proc.stderr: - out += "\n--- stderr ---\n" + proc.stderr[-5000:] + out += "\n--- stderr ---\n" + proc.stderr entry["test_output"] = out no_module = ( diff --git a/templates/single_page.html b/templates/single_page.html new file mode 100644 index 0000000..f8b3f57 --- /dev/null +++ b/templates/single_page.html @@ -0,0 +1,66 @@ +{% extends "_base.html" %} +{% block title %}Astropy integration matrix{% endblock %} +{% block body %} +

Astropy ecosystem integration matrix

+ + + + +{% for v in variants %} + + + + + + + +{% endfor %} + +
VariantAstropyPythonStartedFinished
{{ v.name }}{% if v.has_data %}{{ v.astropy_version }}{% if v.extra_index_urls %} (nightly){% endif %}{% else %}-{% endif %}{% if v.has_data %}{{ v.python_version }}{% else %}-{% endif %}{% if v.has_data %}{{ v.started_at }}{% else %}-{% endif %}{% if v.has_data %}{{ v.finished_at or "(in progress)" }}{% else %}-{% endif %}
+ +

Matrix

+ + + + {% for v in variants %}{% endfor %} + + +{% for row in rows %} + + + {% for v in variants %} + {% set cell = row.cells[v.name] %} + + {% endfor %} + +{% endfor %} + +
Package{{ v.name }}
{{ row.name }} + {{ cell.label }} + {% if cell.resolved_version %}
{{ cell.resolved_version }}{% endif %} +
+ +
Legend: + pass + fail + timeout + no tests + skipped + install fail + missing +
+ +{% if issues %} +

Issues

+
    +{% for i in issues %} +
  • {{ i.name }} ({{ i.variant }}): + {{ i.label }} + {%- if i.error_excerpt %} — {{ i.error_excerpt }}{% endif %}
    • +{% endfor %} +
+

Full install errors and test logs are in the per-variant JSON results uploaded alongside this preview.

+{% else %} +

No issues to report — every package passed in every variant that ran.

+{% endif %} +{% endblock %} From 1f73091f14203bd37c66667afac573ec62782611 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 May 2026 11:53:12 +0100 Subject: [PATCH 03/17] Fixes to CI --- .github/workflows/integration.yml | 16 +++++---- .github/workflows/preview-link.yml | 28 +++++++++++++++ .github/workflows/preview.yml | 56 +++--------------------------- 3 files changed, 42 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/preview-link.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a5858a3..de2e117 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -23,6 +23,10 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install uv uses: astral-sh/setup-uv@v3 @@ -35,11 +39,8 @@ jobs: uv-${{ runner.os }}-${{ matrix.variant }}- uv-${{ runner.os }}- - - name: Pre-install Python 3.12 - run: uv python install 3.12 - - name: Install runtime deps - run: uv pip install --system packaging pyyaml + run: pip install packaging pyyaml - name: Run ${{ matrix.variant }} variant run: python run_integration.py --variant ${{ matrix.variant }} @@ -59,11 +60,12 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v3 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' - name: Install runtime deps - run: uv pip install --system jinja2 + run: pip install jinja2 - name: Download all variant results uses: actions/download-artifact@v4 diff --git a/.github/workflows/preview-link.yml b/.github/workflows/preview-link.yml new file mode 100644 index 0000000..1d0e609 --- /dev/null +++ b/.github/workflows/preview-link.yml @@ -0,0 +1,28 @@ +name: preview-link + +# Companion to `dashboard-preview`. When that workflow finishes (or +# updates state), this one posts a status check on the originating +# commit whose "Details" link goes directly to the rendered +# dashboard.html artifact, skipping the artifact summary page. +# +# Must live on the default branch to take effect. + +on: + workflow_run: + workflows: ["dashboard-preview"] + types: [requested, in_progress, completed] + +permissions: {} + +jobs: + redirect: + runs-on: ubuntu-latest + permissions: + statuses: write # to attach a status check to the source commit + actions: read # to look up the source workflow's artifact id + steps: + - uses: agriyakhetarpal/github-actions-artifacts-redirector-action@683d25ace2cb0aefe8e6719c39c2ac7f3d22dd8c + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + artifact-name: dashboard-preview + job-title: View dashboard preview diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 5ef1e4a..87773d7 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -5,8 +5,7 @@ on: permissions: contents: read - actions: read # to find and download the latest main run's results - pull-requests: write # to (re)post the preview link comment + actions: read # to find and download the latest main run's results concurrency: group: dashboard-preview-${{ github.event.pull_request.number }} @@ -19,11 +18,12 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v3 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' - name: Install runtime deps - run: uv pip install --system jinja2 + run: pip install jinja2 - name: Find latest successful main run id: latest @@ -67,49 +67,3 @@ jobs: # Keep one preview around per PR; the new upload supersedes the old one. overwrite: true - - name: Comment preview link on PR - if: github.event.pull_request.head.repo.full_name == github.repository - continue-on-error: true - uses: actions/github-script@v7 - env: - ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} - MAIN_RUN_ID: ${{ steps.latest.outputs.run_id }} - with: - script: | - const url = process.env.ARTIFACT_URL; - const mainRun = process.env.MAIN_RUN_ID; - const dataSource = mainRun - ? `Rendered against the latest successful main integration run (#${mainRun}).` - : `No main integration runs have completed yet, so the matrix shows no results.`; - const body = [ - "**Dashboard preview**: " + url, - "", - dataSource, - "", - "Rebuilt on every push to this PR. Open the link above, then click `dashboard.html` to view in the browser.", - ].join("\n"); - - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - per_page: 100, - }); - const tag = "**Dashboard preview**"; - const existing = comments.find(c => - c.user.type === "Bot" && c.body && c.body.startsWith(tag)); - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - } From cb3eb58391ac77efc616a281e2112d0e7a91594c Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 May 2026 12:11:11 +0100 Subject: [PATCH 04/17] Simplify CI setup --- .github/workflows/integration.yml | 31 +++++++++++--- .github/workflows/preview-link.yml | 10 ++--- .github/workflows/preview.yml | 69 ------------------------------ README.md | 30 +++++++------ 4 files changed, 46 insertions(+), 94 deletions(-) delete mode 100644 .github/workflows/preview.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index de2e117..b4d7ce0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -4,13 +4,16 @@ on: schedule: - cron: '0 6 * * 0' # Sundays at 06:00 UTC workflow_dispatch: + pull_request: permissions: - contents: write # to push gh-pages + contents: write # to push gh-pages on main / schedule / dispatch concurrency: - group: integration-matrix - cancel-in-progress: false + # One in-flight run per PR, one per main/dispatch invocation. + # PR runs cancel previous in-progress ones; main runs queue. + group: integration-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: variant: @@ -74,13 +77,29 @@ jobs: merge-multiple: true path: results/ - - name: List results - run: ls -la results/ + # PR path: single-page summary, uploaded as a non-zipped artifact + # that GitHub's artifact viewer can render directly in the browser. + - name: Build single-page dashboard (PR preview) + if: github.event_name == 'pull_request' + run: python build_dashboard.py --single-page - - name: Build dashboard + - name: Upload single-page preview + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: dashboard-preview + path: site/dashboard.html + archive: false + overwrite: true + + # Main / schedule / dispatch path: full multi-page dashboard + # published to gh-pages. + - name: Build full dashboard + if: github.event_name != 'pull_request' run: python build_dashboard.py - name: Publish to gh-pages + if: github.event_name != 'pull_request' uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/preview-link.yml b/.github/workflows/preview-link.yml index 1d0e609..7a48054 100644 --- a/.github/workflows/preview-link.yml +++ b/.github/workflows/preview-link.yml @@ -1,15 +1,15 @@ name: preview-link -# Companion to `dashboard-preview`. When that workflow finishes (or -# updates state), this one posts a status check on the originating -# commit whose "Details" link goes directly to the rendered -# dashboard.html artifact, skipping the artifact summary page. +# Companion to `integration-matrix`. When that workflow finishes on +# a pull request, this one attaches a status check to the source +# commit whose "Details" link goes directly to the dashboard-preview +# artifact (rendered HTML), skipping the artifact summary page. # # Must live on the default branch to take effect. on: workflow_run: - workflows: ["dashboard-preview"] + workflows: ["integration-matrix"] types: [requested, in_progress, completed] permissions: {} diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml deleted file mode 100644 index 87773d7..0000000 --- a/.github/workflows/preview.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: dashboard-preview - -on: - pull_request: - -permissions: - contents: read - actions: read # to find and download the latest main run's results - -concurrency: - group: dashboard-preview-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install runtime deps - run: pip install jinja2 - - - name: Find latest successful main run - id: latest - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - run_id=$(gh run list \ - --workflow=integration.yml \ - --branch=main \ - --status=success \ - --limit=1 \ - --json databaseId \ - --jq '.[0].databaseId // empty') - if [ -z "$run_id" ]; then - echo "No successful main integration runs yet; preview will render with no results." - else - echo "Using main run $run_id" - fi - echo "run_id=$run_id" >> "$GITHUB_OUTPUT" - - - name: Download main run results - if: steps.latest.outputs.run_id != '' - uses: actions/download-artifact@v8 - with: - pattern: results-* - merge-multiple: true - path: results/ - run-id: ${{ steps.latest.outputs.run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Build single-page dashboard - run: python build_dashboard.py --single-page - - - name: Upload dashboard as non-zipped artifact - id: upload - uses: actions/upload-artifact@v7 - with: - name: dashboard-preview - path: site/dashboard.html - archive: false - # Keep one preview around per PR; the new upload supersedes the old one. - overwrite: true - diff --git a/README.md b/README.md index 7622ed6..6f440f8 100644 --- a/README.md +++ b/README.md @@ -105,20 +105,22 @@ and triggerable separately. PR previews ----------- -`dashboard-preview` runs on every pull request. It downloads the -results JSONs from the most recent successful main `integration-matrix` -run, builds a single-page `dashboard.html` with `build_dashboard.py ---single-page`, and uploads it as a non-zipped artifact -(`actions/upload-artifact@v7` with `archive: false`). A bot comment -on the PR links to the artifact; clicking through opens the page +`integration-matrix` also runs on pull requests. Same three-variant +matrix as the scheduled run, just with a different final step: the +`dashboard` job builds a single-page summary (`build_dashboard.py +--single-page`) and uploads it as a non-zipped artifact +(`actions/upload-artifact@v7` with `archive: false`). The companion +`preview-link` workflow attaches a "View dashboard preview" status +check to the commit whose "Details" link opens the rendered page directly in the browser. -This is fast (under a minute) because it doesn't re-run the test -matrix; it only re-renders the dashboard with the PR's changes to -`packages.yaml`, `build_dashboard.py`, or `templates/` applied to -the latest real data. To actually validate a new package install, -trigger the full `integration-matrix` workflow manually. +This means the PR preview reflects *this PR's actual matrix run*, +not last main's data. It's also slower than a render-only preview +would be — expect the same wall-clock as a normal main run, up to +a few hours per push. Concurrency cancels in-progress PR runs when +a new push lands, so only the latest push consumes CI time. -Preview is skipped for pull requests from forked repositories (the -build still runs but the PR comment is suppressed; reviewers can -still open the artifact link from the workflow run page). +`preview-link.yml` lives at `.github/workflows/preview-link.yml` +and must be on the default branch for its `workflow_run` trigger +to fire. The first PR after merging the workflow won't get the +status check. From 2e4f0fbe0fc4bfb82724638415e63f5d1ae7fa6a Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 May 2026 12:22:07 +0100 Subject: [PATCH 05/17] Fix CI issues --- .github/workflows/integration.yml | 2 +- README.md | 2 +- run_integration.py | 46 ++++++++++++++----------------- status.py | 2 +- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b4d7ce0..422efdd 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - variant: [stable, latest, dev] + variant: [stable, pre, dev] runs-on: ubuntu-latest timeout-minutes: 240 steps: diff --git a/README.md b/README.md index 6f440f8..81a6212 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ astropy variant: | Variant | Astropy | Each package | |----------|-----------------------------------------------------|----------------------------------------| | `stable` | Latest non-pre-release on PyPI | Latest non-pre-release on PyPI | -| `latest` | Latest including pre-releases (`--prerelease=allow`)| Latest including pre-releases | +| `pre` | Latest including pre-releases (`--prerelease=allow`)| Latest including pre-releases | | `dev` | Latest dev wheel from the astropy/simple channel | `git+` (HEAD of main branch) | Within each job, a single shared venv is built and packages are diff --git a/run_integration.py b/run_integration.py index 854c56d..4ff3581 100644 --- a/run_integration.py +++ b/run_integration.py @@ -9,8 +9,9 @@ data. Usage: + python run_integration.py # all three variants python run_integration.py --variant stable - python run_integration.py --variant latest + python run_integration.py --variant pre python run_integration.py --variant dev python run_integration.py --variant stable --packages reproject,sunpy python run_integration.py --variant stable --tiers coordinated @@ -19,7 +20,6 @@ import argparse import json import os -import re import shutil import subprocess import sys @@ -40,6 +40,10 @@ import status +# Make every print flush immediately so CI streams progress live +# instead of buffering until the script exits. +sys.stdout.reconfigure(line_buffering=True) + PYPI_JSON_URL = "https://pypi.org/pypi/{name}/json" ASTROPY_NIGHTLY_INDEX = "https://pypi.anaconda.org/astropy/simple" @@ -91,20 +95,6 @@ def latest_pypi(name, include_prereleases): return sorted(versions, key=_version_key)[-1] -def latest_astropy_nightly(): - """Return the latest astropy version from the astropy/simple channel.""" - html = _http_text(f"{ASTROPY_NIGHTLY_INDEX}/astropy/") - versions = set() - for href in re.findall(r'href="([^"]+\.whl)"', html): - fname = href.rsplit("/", 1)[-1] - parts = fname[: -len(".whl")].split("-") - if len(parts) >= 2 and parts[0].lower() == "astropy": - versions.add(parts[1]) - if not versions: - sys.exit("No astropy nightly wheels found on the astropy/simple channel.") - return sorted(versions, key=_version_key)[-1] - - def _extras_suffix(pkg): extras = pkg.get("install_extras") or [] return "[" + ",".join(extras) + "]" if extras else "" @@ -113,12 +103,15 @@ def _extras_suffix(pkg): def resolve_specs(packages, variant): """Resolve the astropy spec and per-package install specs for the variant.""" if variant == "dev": - astropy_ver = latest_astropy_nightly() + # Let uv resolve the latest dev version from the astropy/simple + # channel; we read the installed version back after install. No + # explicit pin avoids the PEP 440 local-version segment headaches + # that astropy's nightly wheels have (e.g. 8.1.0.dev53+gabcdef). astropy = { - "install": f"astropy=={astropy_ver}", - "version": astropy_ver, + "install": "astropy", + "version": "", "extra_index_urls": [ASTROPY_NIGHTLY_INDEX], - "prerelease_strategy": "if-necessary-or-explicit", + "prerelease_strategy": "allow", } def pkg_spec(pkg): @@ -128,7 +121,7 @@ def pkg_spec(pkg): return f"{pkg['pypi_name']}{_extras_suffix(pkg)} @ git+{repo}", None else: - include_pre = (variant == "latest") + include_pre = (variant == "pre") astropy_ver = latest_pypi("astropy", include_prereleases=include_pre) astropy = { "install": f"astropy=={astropy_ver}", @@ -391,7 +384,8 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--config", default="packages.yaml") ap.add_argument("--results-dir", default="results") - ap.add_argument("--variant", choices=status.VARIANTS, required=True) + ap.add_argument("--variant", choices=status.VARIANTS, + help="Variant to run; if omitted, runs all variants in sequence.") ap.add_argument("--packages", help="Comma-separated subset of package names to run") ap.add_argument("--tiers", help="Comma-separated subset of tiers to run (e.g. 'coordinated,other'); " @@ -414,9 +408,11 @@ def main(): results_dir.mkdir(parents=True, exist_ok=True) timeouts = {"install": args.timeout_install, "test": args.timeout_test} - result, out_path = run_variant(args.variant, packages, repo_root, results_dir, timeouts) - print(f"\nDone: {_counts(result)}") - print(f"Wrote {out_path}") + variants_to_run = [args.variant] if args.variant else list(status.VARIANTS) + for variant in variants_to_run: + result, out_path = run_variant(variant, packages, repo_root, results_dir, timeouts) + print(f"\nDone {variant}: {_counts(result)}") + print(f"Wrote {out_path}") if __name__ == "__main__": diff --git a/status.py b/status.py index 41065e3..c930bfc 100644 --- a/status.py +++ b/status.py @@ -6,7 +6,7 @@ """ # The three astropy variants. Listed once here so both scripts agree. -VARIANTS = ("stable", "latest", "dev") +VARIANTS = ("stable", "pre", "dev") # install_status values, used in results/.json INSTALLED = "installed" From 6de8c257c0f3fa28337368aaa34b59d939a48deb Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 May 2026 12:51:09 +0100 Subject: [PATCH 06/17] Simplify dashboard set-up --- .github/workflows/integration.yml | 20 ++---- .github/workflows/preview-link.yml | 4 +- README.md | 14 ++-- build_dashboard.py | 104 ++++++++--------------------- run_integration.py | 22 +++++- templates/cell.html | 25 ------- templates/index.html | 76 ++++++++++++++++----- templates/single_page.html | 66 ------------------ templates/variant.html | 42 ------------ 9 files changed, 128 insertions(+), 245 deletions(-) delete mode 100644 templates/cell.html delete mode 100644 templates/single_page.html delete mode 100644 templates/variant.html diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 422efdd..6e15508 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -77,27 +77,21 @@ jobs: merge-multiple: true path: results/ - # PR path: single-page summary, uploaded as a non-zipped artifact - # that GitHub's artifact viewer can render directly in the browser. - - name: Build single-page dashboard (PR preview) - if: github.event_name == 'pull_request' - run: python build_dashboard.py --single-page + - name: Build dashboard + run: python build_dashboard.py - - name: Upload single-page preview + # PR path: upload index.html as a non-zipped artifact for the + # in-browser preview (linked from a PR check by preview-link.yml). + - name: Upload preview artifact if: github.event_name == 'pull_request' uses: actions/upload-artifact@v7 with: name: dashboard-preview - path: site/dashboard.html + path: site/index.html archive: false overwrite: true - # Main / schedule / dispatch path: full multi-page dashboard - # published to gh-pages. - - name: Build full dashboard - if: github.event_name != 'pull_request' - run: python build_dashboard.py - + # Main / schedule / dispatch path: publish to gh-pages. - name: Publish to gh-pages if: github.event_name != 'pull_request' uses: peaceiris/actions-gh-pages@v4 diff --git a/.github/workflows/preview-link.yml b/.github/workflows/preview-link.yml index 7a48054..01cb6c1 100644 --- a/.github/workflows/preview-link.yml +++ b/.github/workflows/preview-link.yml @@ -24,5 +24,7 @@ jobs: - uses: agriyakhetarpal/github-actions-artifacts-redirector-action@683d25ace2cb0aefe8e6719c39c2ac7f3d22dd8c with: repo-token: ${{ secrets.GITHUB_TOKEN }} - artifact-name: dashboard-preview + # With `archive: false`, artifact-name is the uploaded file's + # basename without extension (we upload site/index.html). + artifact-name: index job-title: View dashboard preview diff --git a/README.md b/README.md index 81a6212..97abc48 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ What's in the repo |-------------------------------------|------------------------------------------------------| | `packages.yaml` | The list of packages tested (one block per package). | | `run_integration.py` | Runs one variant: resolve specs, install, test, write `results/.json`. | -| `build_dashboard.py` | Reads `results/*.json`, renders `site/`. | +| `build_dashboard.py` | Reads `results/*.json`, renders `site/index.html` (single self-contained page). | | `status.py` | Shared status vocabulary (used by both scripts). | | `templates/` | HTML/CSS for the dashboard. | | `.github/workflows/integration.yml` | The new matrix workflow (variant x3 + dashboard). | @@ -107,12 +107,12 @@ PR previews `integration-matrix` also runs on pull requests. Same three-variant matrix as the scheduled run, just with a different final step: the -`dashboard` job builds a single-page summary (`build_dashboard.py ---single-page`) and uploads it as a non-zipped artifact -(`actions/upload-artifact@v7` with `archive: false`). The companion -`preview-link` workflow attaches a "View dashboard preview" status -check to the commit whose "Details" link opens the rendered page -directly in the browser. +`dashboard` job uploads `site/index.html` as a non-zipped artifact +(`actions/upload-artifact@v7` with `archive: false`) instead of +publishing to gh-pages. The companion `preview-link` workflow +attaches a "View dashboard preview" status check to the commit +whose "Details" link opens the rendered page directly in the +browser. This means the PR preview reflects *this PR's actual matrix run*, not last main's data. It's also slower than a render-only preview diff --git a/build_dashboard.py b/build_dashboard.py index c7cc557..718f37e 100644 --- a/build_dashboard.py +++ b/build_dashboard.py @@ -1,15 +1,10 @@ #!/usr/bin/env python -"""Build the static integration-matrix dashboard. +"""Build the single-page integration-matrix dashboard. -Reads the per-variant JSONs from results/ (stable.json, latest.json, -dev.json - any missing variant just shows as missing cells) and emits: - - site/index.html Nx3 matrix - site/cells/__.html per-(package, variant) detail page - site/variants/.html per-variant detail with full freeze - -Publishing to gh-pages is done from CI via a published action; this -script only builds locally. +Reads the per-variant JSONs from results/ (stable.json, pre.json, +dev.json - any missing variant just shows as missing cells) and emits +a single self-contained `site/index.html` with the matrix, a legend, +and collapsible per-cell failure logs. """ import argparse @@ -25,6 +20,10 @@ VARIANTS = status.VARIANTS +def _anchor_id(name, variant): + return f"log-{name}__{variant}" + + def _variant_meta(name, data): if not data: return {"name": name, "has_data": False} @@ -64,19 +63,22 @@ def _make_rows(by_variant, names): entry = next((e for e in data["packages"] if e["name"] == name), None) if entry is None: cells[v] = {"status": "missing", "label": "-", - "detail_filename": "", "resolved_version": ""} + "anchor": "", "resolved_version": ""} else: + badge = status.cell_badge(entry) + anchor = (_anchor_id(name, v) + if badge["status"] not in ("pass", "missing") else "") cells[v] = { - **status.cell_badge(entry), - "detail_filename": f"{name}__{v}.html", + **badge, + "anchor": anchor, "resolved_version": entry.get("resolved_version", ""), } rows.append({"name": name, "cells": cells}) return rows -def _issues(by_variant): - """List non-passing cells with a one-line error excerpt where available.""" +def _failures(by_variant): + """List non-passing cells with their full logs.""" out = [] for v in VARIANTS: data = by_variant[v] @@ -86,23 +88,19 @@ def _issues(by_variant): badge = status.cell_badge(entry) if badge["status"] in ("pass", "missing"): continue - excerpt = "" - for line in (entry.get("install_error") or "").splitlines(): - line = line.strip() - if line: - excerpt = line[:200] - break out.append({ "name": entry["name"], "variant": v, "status": badge["status"], "label": badge["label"], - "error_excerpt": excerpt, + "install_error": entry.get("install_error", ""), + "test_output": entry.get("test_output", ""), + "anchor": _anchor_id(entry["name"], v), }) return out -def build(results_dir, output_dir, templates_dir, single_page=False): +def build(results_dir, output_dir, templates_dir): results_dir = Path(results_dir) output_dir = Path(output_dir) if output_dir.exists(): @@ -122,55 +120,16 @@ def build(results_dir, output_dir, templates_dir, single_page=False): names = _ordered_packages(by_variant) rows = _make_rows(by_variant, names) variants_meta = [_variant_meta(v, by_variant[v]) for v in VARIANTS] - - if single_page: - single_tpl = env.get_template("single_page.html") - (output_dir / "dashboard.html").write_text( - single_tpl.render( - variants=variants_meta, - rows=rows, - issues=_issues(by_variant), - ) - ) - print(f"Wrote dashboard.html to {output_dir}/") - return - - (output_dir / "cells").mkdir() - (output_dir / "variants").mkdir() - index_tpl = env.get_template("index.html") - cell_tpl = env.get_template("cell.html") - variant_tpl = env.get_template("variant.html") + failures = _failures(by_variant) (output_dir / "index.html").write_text( - index_tpl.render(variants=variants_meta, rows=rows) - ) - n_pages = 1 - - for v in VARIANTS: - data = by_variant[v] - if not data: - continue - for entry in data["packages"]: - (output_dir / "cells" / f"{entry['name']}__{v}.html").write_text( - cell_tpl.render( - entry=entry, - variant=v, - status=status.cell_badge(entry), - astropy_version=data["astropy"]["version"], - ) - ) - n_pages += 1 - (output_dir / "variants" / f"{v}.html").write_text( - variant_tpl.render( - variant=v, - data=data, - install_badge=status.INSTALL_BADGE, - test_badge=status.TEST_BADGE, - ) + env.get_template("index.html").render( + variants=variants_meta, + rows=rows, + failures=failures, ) - n_pages += 1 - - print(f"Wrote {n_pages} pages to {output_dir}/") + ) + print(f"Wrote {output_dir}/index.html") def main(): @@ -178,13 +137,8 @@ def main(): ap.add_argument("--results-dir", default="results") ap.add_argument("--output", default="site") ap.add_argument("--templates-dir", default="templates") - ap.add_argument("--single-page", action="store_true", - help="Render a single self-contained dashboard.html (no cell or " - "variant pages). Suitable for serving as a non-zipped GH " - "Actions artifact in PR previews.") args = ap.parse_args() - build(args.results_dir, args.output, args.templates_dir, - single_page=args.single_page) + build(args.results_dir, args.output, args.templates_dir) if __name__ == "__main__": diff --git a/run_integration.py b/run_integration.py index 4ff3581..76710e2 100644 --- a/run_integration.py +++ b/run_integration.py @@ -47,6 +47,7 @@ PYPI_JSON_URL = "https://pypi.org/pypi/{name}/json" ASTROPY_NIGHTLY_INDEX = "https://pypi.anaconda.org/astropy/simple" +LIBERFA_NIGHTLY_INDEX = "https://pypi.anaconda.org/liberfa/simple" # for pyerfa dev wheels PYTHON_VERSION = "3.12" @@ -107,11 +108,19 @@ def resolve_specs(packages, variant): # channel; we read the installed version back after install. No # explicit pin avoids the PEP 440 local-version segment headaches # that astropy's nightly wheels have (e.g. 8.1.0.dev53+gabcdef). + # + # `--index-strategy unsafe-best-match` is required because uv's + # default ("first-index") only considers a single index per + # package; astropy/simple hosts astropy AND pyerfa (the channel's + # latest wheels sometimes ship only musllinux). unsafe-best-match + # lets uv fall back to PyPI when the channel's only wheels don't + # match the runner platform. astropy = { "install": "astropy", "version": "", - "extra_index_urls": [ASTROPY_NIGHTLY_INDEX], + "extra_index_urls": [ASTROPY_NIGHTLY_INDEX, LIBERFA_NIGHTLY_INDEX], "prerelease_strategy": "allow", + "index_strategy": "unsafe-best-match", } def pkg_spec(pkg): @@ -128,6 +137,7 @@ def pkg_spec(pkg): "version": astropy_ver, "extra_index_urls": [], "prerelease_strategy": "allow" if include_pre else "if-necessary-or-explicit", + "index_strategy": None, } def pkg_spec(pkg): @@ -275,12 +285,15 @@ def run_variant(variant, packages, repo_root, results_dir, timeouts): for url in astropy["extra_index_urls"]: common += ["--extra-index-url", url] common += [f"--prerelease={astropy['prerelease_strategy']}"] + if astropy.get("index_strategy"): + common += [f"--index-strategy={astropy['index_strategy']}"] print("\nInstalling astropy + pytest...") rc, err = _run_install(common + [astropy["install"], "pytest", "pytest-timeout"], timeouts["install"]) if rc != 0: print(" FATAL: astropy install failed") + print(err) result["fatal_error"] = err return result, out_path result["astropy"]["version"] = _pkg_version(python, "astropy") or result["astropy"]["version"] @@ -323,6 +336,7 @@ def run_variant(variant, packages, repo_root, results_dir, timeouts): else: entry["install_status"] = status.INSTALL_FAIL print(" install failed") + print(err) result["packages"].append(entry) result["installed_deps"] = _freeze(python) @@ -409,10 +423,16 @@ def main(): timeouts = {"install": args.timeout_install, "test": args.timeout_test} variants_to_run = [args.variant] if args.variant else list(status.VARIANTS) + fatal_variants = [] for variant in variants_to_run: result, out_path = run_variant(variant, packages, repo_root, results_dir, timeouts) print(f"\nDone {variant}: {_counts(result)}") print(f"Wrote {out_path}") + if result.get("fatal_error"): + fatal_variants.append(variant) + + if fatal_variants: + sys.exit(f"\nAstropy install failed for variant(s): {', '.join(fatal_variants)}") if __name__ == "__main__": diff --git a/templates/cell.html b/templates/cell.html deleted file mode 100644 index 3e64972..0000000 --- a/templates/cell.html +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "_base.html" %} -{% block title %}{{ entry.name }} - {{ variant }}{% endblock %} -{% block nav %}{% endblock %} -{% block body %} -

{{ entry.name }} in {{ variant }} venv

- - - - - - - - -
Install status{{ status.label }}
Resolved version{{ entry.resolved_version or entry.target_version or '-' }}
Astropy version{{ astropy_version }} (variant: {{ variant }})
Install spec{{ entry.install_spec or '(none)' }}
Tier{{ entry.tier }}
Test status{{ entry.test_status or '-' }}
Duration{{ entry.duration }}s
-{% if entry.install_status in ("skipped", "install-fail", "no-spec") and entry.install_error %} -

{% if entry.install_status == "skipped" %}Resolver conflict - {%- elif entry.install_status == "install-fail" %}Install error - {%- else %}Spec error{% endif %}

-
{{ entry.install_error }}
-{% endif %} -{% if entry.test_output %} -

Test output

-
{{ entry.test_output }}
-{% endif %} -{% endblock %} diff --git a/templates/index.html b/templates/index.html index ef23e41..9910da1 100644 --- a/templates/index.html +++ b/templates/index.html @@ -2,18 +2,23 @@ {% block title %}Astropy integration matrix{% endblock %} {% block body %}

Astropy ecosystem integration matrix

-
- {% for v in variants %} -
{{ v.name }}
-
{% if v.has_data %} - astropy {{ v.astropy_version }}{% if v.extra_index_urls %} (nightly){% endif %}, - Python {{ v.python_version }}, - started {{ v.started_at }}{% if v.finished_at %}, finished {{ v.finished_at }}{% endif %} · - variant detail - {% else %}no results{% endif %}
- {% endfor %} -
-

Each row is a package. Each column is an astropy variant. Click any badge for the package's test log in that variant; click "variant detail" for the variant's full pip freeze and per-package install outcome.

+ + + + +{% for v in variants %} + + + + + + + +{% endfor %} + +
VariantAstropyPythonStartedFinished
{{ v.name }}{% if v.has_data %}{{ v.astropy_version }}{% if v.extra_index_urls %} (nightly){% endif %}{% else %}-{% endif %}{% if v.has_data %}{{ v.python_version }}{% else %}-{% endif %}{% if v.has_data %}{{ v.started_at }}{% else %}-{% endif %}{% if v.has_data %}{{ v.finished_at or "(in progress)" }}{% else %}-{% endif %}
+ +

Matrix

@@ -26,9 +31,11 @@

Astropy ecosystem integration matrix

{% for v in variants %} {% set cell = row.cells[v.name] %} {% endfor %} @@ -36,6 +43,7 @@

Astropy ecosystem integration matrix

{% endfor %}
Package - {% if cell.detail_filename %} - {{ cell.label }} - {% else %}{{ cell.label }}{% endif %} + {% if cell.anchor %} + {{ cell.label }} + {% else %} + {{ cell.label }} + {% endif %} {% if cell.resolved_version %}
{{ cell.resolved_version }}{% endif %}
+
Legend: pass fail @@ -46,4 +54,42 @@

Astropy ecosystem integration matrix

missing

skipped: resolver couldn't satisfy this package alongside the rest of the venv. install fail: install step crashed for some other reason (build error, etc.).

+ +{% if failures %} +

Failure logs

+

Click any non-passing badge above to jump to its log; the section will auto-expand.

+{% for f in failures %} +
+ + {{ f.name }} + ({{ f.variant }}) + {{ f.label }} + + {% if f.install_error %} +

Install error

+ 
{{ f.install_error }}
+ {% endif %} + {% if f.test_output %} +

Test output

+ 
{{ f.test_output }}
+ {% endif %} +
+{% endfor %} +{% else %} +

No failures — every package passed in every variant that ran.

+{% endif %} + + {% endblock %} diff --git a/templates/single_page.html b/templates/single_page.html deleted file mode 100644 index f8b3f57..0000000 --- a/templates/single_page.html +++ /dev/null @@ -1,66 +0,0 @@ -{% extends "_base.html" %} -{% block title %}Astropy integration matrix{% endblock %} -{% block body %} -

Astropy ecosystem integration matrix

- - - - -{% for v in variants %} - - - - - - - -{% endfor %} - -
VariantAstropyPythonStartedFinished
{{ v.name }}{% if v.has_data %}{{ v.astropy_version }}{% if v.extra_index_urls %} (nightly){% endif %}{% else %}-{% endif %}{% if v.has_data %}{{ v.python_version }}{% else %}-{% endif %}{% if v.has_data %}{{ v.started_at }}{% else %}-{% endif %}{% if v.has_data %}{{ v.finished_at or "(in progress)" }}{% else %}-{% endif %}
- -

Matrix

- - - - {% for v in variants %}{% endfor %} - - -{% for row in rows %} - - - {% for v in variants %} - {% set cell = row.cells[v.name] %} - - {% endfor %} - -{% endfor %} - -
Package{{ v.name }}
{{ row.name }} - {{ cell.label }} - {% if cell.resolved_version %}
{{ cell.resolved_version }}{% endif %} -
- -
Legend: - pass - fail - timeout - no tests - skipped - install fail - missing -
- -{% if issues %} -

Issues

-
    -{% for i in issues %} -
  • {{ i.name }} ({{ i.variant }}): - {{ i.label }} - {%- if i.error_excerpt %} — {{ i.error_excerpt }}{% endif %}
    • -{% endfor %} -
-

Full install errors and test logs are in the per-variant JSON results uploaded alongside this preview.

-{% else %} -

No issues to report — every package passed in every variant that ran.

-{% endif %} -{% endblock %} diff --git a/templates/variant.html b/templates/variant.html deleted file mode 100644 index 7e1c234..0000000 --- a/templates/variant.html +++ /dev/null @@ -1,42 +0,0 @@ -{% extends "_base.html" %} -{% block title %}{{ variant }} variant{% endblock %} -{% block nav %}{% endblock %} -{% block body %} -

Variant: {{ variant }}

- - - - - - - -
Astropy{{ data.astropy.version }} ({{ data.astropy.install_spec }})
Extra index URLs{% if data.astropy.extra_index_urls %}{% for u in data.astropy.extra_index_urls %}{{ u }}{% if not loop.last %}
{% endif %}{% endfor %}{% else %}(none){% endif %}
Pre-release strategy{{ data.astropy.prerelease_strategy }}
Python{{ data.python_version }}
Started{{ data.started_at }}
Finished{{ data.finished_at or "-" }}
- -{% if data.fatal_error %} -

Fatal error (astropy install failed; no packages were tested)

-
{{ data.fatal_error }}
-{% endif %} - -

Per-package install & test outcome

- - - -{% for entry in data.packages %} -{% set istat = install_badge[entry.install_status] %} -{% set tstat = test_badge[entry.test_status] %} - - - - - - - -{% endfor %} - -
PackageInstallResolvedTestDuration
{{ entry.name }}{{ istat.label }}{{ entry.resolved_version or "-" }}{% if entry.test_status %}{{ tstat.label }}{% else %}-{% endif %}{{ entry.duration }}s
- -
Full venv freeze ({{ data.installed_deps|length }} packages) - -{% for name, ver in data.installed_deps.items()|sort %}{% endfor %} -
{{ name }}{{ ver }}
-{% endblock %} From c1c85cc797021c603af34ff8ecc7ed19194b752f Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 May 2026 13:37:02 +0100 Subject: [PATCH 07/17] Improvements, including making it possible to run multiple Python versions --- .github/workflows/integration.yml | 14 ++-- README.md | 18 ++++ build_dashboard.py | 131 +++++++++++++++++++----------- packages.yaml | 16 +++- run_integration.py | 55 ++++++++----- templates/_style.css | 1 + templates/index.html | 26 +++--- 7 files changed, 174 insertions(+), 87 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6e15508..9d00d69 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,6 +20,9 @@ jobs: strategy: fail-fast: false matrix: + # Keep in sync with `python_versions` in packages.yaml. + # (free-threaded builds: use the "t" suffix, e.g. "3.14t".) + python: ["3.12", "3.14"] variant: [stable, pre, dev] runs-on: ubuntu-latest timeout-minutes: 240 @@ -37,23 +40,24 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/uv - key: uv-${{ runner.os }}-${{ matrix.variant }}-${{ hashFiles('packages.yaml') }} + key: uv-${{ runner.os }}-${{ matrix.variant }}-${{ matrix.python }}-${{ hashFiles('packages.yaml') }} restore-keys: | + uv-${{ runner.os }}-${{ matrix.variant }}-${{ matrix.python }}- uv-${{ runner.os }}-${{ matrix.variant }}- uv-${{ runner.os }}- - name: Install runtime deps run: pip install packaging pyyaml - - name: Run ${{ matrix.variant }} variant - run: python run_integration.py --variant ${{ matrix.variant }} + - name: Run ${{ matrix.variant }} variant on Python ${{ matrix.python }} + run: python run_integration.py --variant ${{ matrix.variant }} --python ${{ matrix.python }} - name: Upload results if: always() uses: actions/upload-artifact@v4 with: - name: results-${{ matrix.variant }} - path: results/${{ matrix.variant }}.json + name: results-${{ matrix.variant }}-${{ matrix.python }} + path: results/${{ matrix.variant }}__${{ matrix.python }}.json dashboard: needs: variant diff --git a/README.md b/README.md index 97abc48..ec74e7a 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,24 @@ python -m http.server -d site 8000 Results land in `results/.json`; the dashboard in `site/`. Both directories are gitignored. +Python versions +--------------- + +`packages.yaml` has a top-level `python_versions` list (uv notation, +so `"3.14t"` means the free-threaded 3.14 build): + +```yaml +python_versions: + - "3.12" + - "3.14t" +``` + +The runner tests every (variant x python_version) combination. The +dashboard renders Python versions as grouped header columns above the +three variants. **Keep the `matrix.python` list in +`.github/workflows/integration.yml` in sync** with this — the CI uses +its own matrix because GitHub Actions can't read it from YAML directly. + Adding or disabling a package ----------------------------- diff --git a/build_dashboard.py b/build_dashboard.py index 718f37e..7d9770f 100644 --- a/build_dashboard.py +++ b/build_dashboard.py @@ -1,14 +1,18 @@ #!/usr/bin/env python """Build the single-page integration-matrix dashboard. -Reads the per-variant JSONs from results/ (stable.json, pre.json, -dev.json - any missing variant just shows as missing cells) and emits -a single self-contained `site/index.html` with the matrix, a legend, -and collapsible per-cell failure logs. +Reads every results/__.json that run_integration.py +wrote and emits a single self-contained `site/index.html` with: + - one row per package + - one column group per Python version, subdivided into the three + astropy variants (stable / pre / dev) + - a "Failure logs" section at the bottom with collapsible
+ for every non-passing cell, anchored from the matching badge """ import argparse import json +import re import shutil from pathlib import Path @@ -20,68 +24,100 @@ VARIANTS = status.VARIANTS -def _anchor_id(name, variant): - return f"log-{name}__{variant}" +def _anchor_id(name, variant, python): + # Anchors are only used inside the page so we just need + # something HTML-id-safe and unique per (pkg, variant, python). + safe_python = re.sub(r"[^A-Za-z0-9]", "_", python) + return f"log-{name}__{variant}__{safe_python}" -def _variant_meta(name, data): +def _load_results(results_dir): + """Return {(variant, python): data, ...} for every results/*.json found.""" + by_combo = {} + for f in sorted(results_dir.glob("*.json")): + if f.name.startswith("_"): + continue + data = json.loads(f.read_text()) + v = data.get("variant") + p = data.get("python_requested") or data.get("python_version") or "" + if not v or not p: + continue + by_combo[(v, p)] = data + return by_combo + + +def _column_groups(by_combo): + """Discover Python versions and order columns. + + Returns: + pythons: sorted list of Python version strings (preserves config-style + ordering: shorter strings first, then alphabetical). + columns: flat list of (python, variant) tuples in display order. + """ + pythons = sorted({p for _, p in by_combo}, key=lambda s: (len(s), s)) + columns = [(p, v) for p in pythons for v in VARIANTS] + return pythons, columns + + +def _variant_meta(variant, python, data): if not data: - return {"name": name, "has_data": False} + return {"variant": variant, "python": python, "has_data": False} return { - "name": name, + "variant": variant, + "python": python, "has_data": True, "astropy_version": data["astropy"]["version"], "extra_index_urls": data["astropy"].get("extra_index_urls") or [], - "python_version": data["python_version"], + "python_version": data.get("python_version") or python, "started_at": data["started_at"], "finished_at": data["finished_at"], } -def _ordered_packages(by_variant): - """Return package names in the order they first appear across variants.""" +def _ordered_packages(by_combo): + """Package names in the order they first appear across all results.""" seen = [] seen_set = set() - for v in VARIANTS: - if not by_variant[v]: - continue - for entry in by_variant[v]["packages"]: + for _, data in by_combo.items(): + for entry in data["packages"]: if entry["name"] not in seen_set: seen.append(entry["name"]) seen_set.add(entry["name"]) return seen -def _make_rows(by_variant, names): +def _make_rows(by_combo, names, columns): rows = [] for name in names: - cells = {} - for v in VARIANTS: - data = by_variant[v] + cells = [] + for python, variant in columns: + data = by_combo.get((variant, python)) entry = None if data: entry = next((e for e in data["packages"] if e["name"] == name), None) if entry is None: - cells[v] = {"status": "missing", "label": "-", - "anchor": "", "resolved_version": ""} - else: - badge = status.cell_badge(entry) - anchor = (_anchor_id(name, v) - if badge["status"] not in ("pass", "missing") else "") - cells[v] = { - **badge, - "anchor": anchor, - "resolved_version": entry.get("resolved_version", ""), - } + cells.append({ + "status": "missing", "label": "-", + "anchor": "", "resolved_version": "", + }) + continue + badge = status.cell_badge(entry) + anchor = (_anchor_id(name, variant, python) + if badge["status"] not in ("pass", "missing") else "") + cells.append({ + **badge, + "anchor": anchor, + "resolved_version": entry.get("resolved_version", ""), + }) rows.append({"name": name, "cells": cells}) return rows -def _failures(by_variant): - """List non-passing cells with their full logs.""" +def _failures(by_combo, columns): + """Per-cell failure detail (full logs), ordered by column then row.""" out = [] - for v in VARIANTS: - data = by_variant[v] + for python, variant in columns: + data = by_combo.get((variant, python)) if not data: continue for entry in data["packages"]: @@ -90,12 +126,13 @@ def _failures(by_variant): continue out.append({ "name": entry["name"], - "variant": v, + "variant": variant, + "python": python, "status": badge["status"], "label": badge["label"], "install_error": entry.get("install_error", ""), "test_output": entry.get("test_output", ""), - "anchor": _anchor_id(entry["name"], v), + "anchor": _anchor_id(entry["name"], variant, python), }) return out @@ -112,19 +149,19 @@ def build(results_dir, output_dir, templates_dir): autoescape=select_autoescape(["html"]), ) - by_variant = { - v: (json.loads((results_dir / f"{v}.json").read_text()) - if (results_dir / f"{v}.json").exists() else None) - for v in VARIANTS - } - names = _ordered_packages(by_variant) - rows = _make_rows(by_variant, names) - variants_meta = [_variant_meta(v, by_variant[v]) for v in VARIANTS] - failures = _failures(by_variant) + by_combo = _load_results(results_dir) + pythons, columns = _column_groups(by_combo) + names = _ordered_packages(by_combo) + rows = _make_rows(by_combo, names, columns) + failures = _failures(by_combo, columns) + variants_meta = [_variant_meta(v, p, by_combo.get((v, p))) + for p in pythons for v in VARIANTS] (output_dir / "index.html").write_text( env.get_template("index.html").render( - variants=variants_meta, + pythons=pythons, + variants=list(VARIANTS), + variants_meta=variants_meta, rows=rows, failures=failures, ) diff --git a/packages.yaml b/packages.yaml index 44c8d32..6b6f32f 100644 --- a/packages.yaml +++ b/packages.yaml @@ -1,15 +1,23 @@ # Packages tested by run_integration.py. # # One block per package = one row in the dashboard. For each astropy -# variant (stable / latest / dev), the runner builds a shared venv, -# installs astropy first then each package in turn (skipping any that -# the resolver can't satisfy alongside the existing set), then runs -# `pytest --pyargs ` for each one that installed. +# variant (stable / pre / dev) and each Python version, the runner +# builds a shared venv, installs astropy first then each package in +# turn (skipping any that the resolver can't satisfy alongside the +# existing set), then runs `pytest --pyargs ` for each one +# that installed. # # `tier` is purely a category label used for ordering and the optional # `--tiers` CLI filter. Conventions: `coordinated` for the official # coordinated set, `affiliated` for affiliated packages, `other` for # anything else worth testing. +# +# `python_versions` is the list of Python versions to test against; +# use uv's notation, so e.g. "3.14t" for the free-threaded build. + +python_versions: + - "3.12" + - "3.14" packages: - pypi_name: asdf_astropy diff --git a/run_integration.py b/run_integration.py index 76710e2..c63ce1a 100644 --- a/run_integration.py +++ b/run_integration.py @@ -9,10 +9,10 @@ data. Usage: - python run_integration.py # all three variants - python run_integration.py --variant stable - python run_integration.py --variant pre - python run_integration.py --variant dev + python run_integration.py # full matrix (all variants x all Python versions from config) + python run_integration.py --variant stable # one variant, all configured Pythons + python run_integration.py --variant stable --python 3.12 # one combo + python run_integration.py --python 3.14t # all variants on free-threaded 3.14 python run_integration.py --variant stable --packages reproject,sunpy python run_integration.py --variant stable --tiers coordinated """ @@ -48,7 +48,6 @@ PYPI_JSON_URL = "https://pypi.org/pypi/{name}/json" ASTROPY_NIGHTLY_INDEX = "https://pypi.anaconda.org/astropy/simple" LIBERFA_NIGHTLY_INDEX = "https://pypi.anaconda.org/liberfa/simple" # for pyerfa dev wheels -PYTHON_VERSION = "3.12" def _http_json(url, timeout=20): @@ -221,6 +220,14 @@ def _load_packages(path): return list(raw.get("packages", [])) +def _load_python_versions(path): + raw = yaml.safe_load(Path(path).read_text()) or {} + versions = raw.get("python_versions") or [] + if not versions: + versions = ["3.12"] + return [str(v) for v in versions] + + # Ordering of tiers when installing/displaying. Unknown tiers sort last. TIER_RANK = {"coordinated": 0, "affiliated": 1, "other": 2} @@ -242,9 +249,9 @@ def _run_install(install_cmd, timeout): return proc.returncode, (proc.stderr or proc.stdout) -def run_variant(variant, packages, repo_root, results_dir, timeouts): +def run_variant(variant, python_version, packages, repo_root, results_dir, timeouts): astropy, pkg_specs = resolve_specs(packages, variant) - print(f"\n=== Variant: {variant} ===") + print(f"\n=== Variant: {variant} (Python {python_version}) ===") print(f" astropy: {astropy['install']}") for pkg, spec, target in pkg_specs: print(f" {pkg['pypi_name']:<20} {spec or '(no install spec)'}") @@ -259,19 +266,20 @@ def run_variant(variant, packages, repo_root, results_dir, timeouts): "extra_index_urls": astropy["extra_index_urls"], "prerelease_strategy": astropy["prerelease_strategy"], }, + "python_requested": python_version, "python_version": "", "fatal_error": "", "installed_deps": {}, "packages": [], } - out_path = results_dir / f"{variant}.json" + out_path = results_dir / f"{variant}__{python_version}.json" Path(repo_root, ".tmp").mkdir(exist_ok=True) - tmpdir = tempfile.mkdtemp(prefix=f"int-{variant}-", + tmpdir = tempfile.mkdtemp(prefix=f"int-{variant}-{python_version}-", dir=str(Path(repo_root, ".tmp").resolve())) try: - py_path = ensure_python(PYTHON_VERSION) + py_path = ensure_python(python_version) venv = os.path.join(tmpdir, "venv") venv_proc = subprocess.run(["uv", "venv", venv, "-p", py_path, "-q"], capture_output=True, text=True, timeout=120) @@ -400,6 +408,9 @@ def main(): ap.add_argument("--results-dir", default="results") ap.add_argument("--variant", choices=status.VARIANTS, help="Variant to run; if omitted, runs all variants in sequence.") + ap.add_argument("--python", + help="Python version to run against (e.g. '3.12', '3.14t'); " + "if omitted, runs every version listed in the config.") ap.add_argument("--packages", help="Comma-separated subset of package names to run") ap.add_argument("--tiers", help="Comma-separated subset of tiers to run (e.g. 'coordinated,other'); " @@ -423,16 +434,20 @@ def main(): timeouts = {"install": args.timeout_install, "test": args.timeout_test} variants_to_run = [args.variant] if args.variant else list(status.VARIANTS) - fatal_variants = [] - for variant in variants_to_run: - result, out_path = run_variant(variant, packages, repo_root, results_dir, timeouts) - print(f"\nDone {variant}: {_counts(result)}") - print(f"Wrote {out_path}") - if result.get("fatal_error"): - fatal_variants.append(variant) - - if fatal_variants: - sys.exit(f"\nAstropy install failed for variant(s): {', '.join(fatal_variants)}") + pythons_to_run = [args.python] if args.python else _load_python_versions(args.config) + + fatal_combos = [] + for python_version in pythons_to_run: + for variant in variants_to_run: + result, out_path = run_variant(variant, python_version, packages, + repo_root, results_dir, timeouts) + print(f"\nDone {variant}/{python_version}: {_counts(result)}") + print(f"Wrote {out_path}") + if result.get("fatal_error"): + fatal_combos.append(f"{variant}/{python_version}") + + if fatal_combos: + sys.exit(f"\nAstropy install failed for: {', '.join(fatal_combos)}") if __name__ == "__main__": diff --git a/templates/_style.css b/templates/_style.css index 089e281..ae5b8c0 100644 --- a/templates/_style.css +++ b/templates/_style.css @@ -5,6 +5,7 @@ a:hover { text-decoration: underline; } table { border-collapse: collapse; } th, td { border: 1px solid #ddd; padding: 6px 10px; text-align: left; vertical-align: top; } th { background: #f6f8fa; } +th.python-group { text-align: center; border-bottom: 2px solid #ccc; } .meta dl { margin: 0.3em 0; display: grid; grid-template-columns: max-content 1fr; gap: 0.2em 1em; color: #444; } .meta dt { font-weight: 600; } .pill { display: inline-block; padding: 2px 10px; border-radius: 12px; font-size: 0.85em; font-weight: 600; min-width: 4em; text-align: center; } diff --git a/templates/index.html b/templates/index.html index 9910da1..4fb1684 100644 --- a/templates/index.html +++ b/templates/index.html @@ -4,13 +4,13 @@

Astropy ecosystem integration matrix

- + -{% for v in variants %} +{% for v in variants_meta %} - + + - @@ -20,16 +20,20 @@

Astropy ecosystem integration matrix

Matrix

VariantAstropyPythonStartedFinished
PythonVariantAstropyStartedFinished
{{ v.name }}{{ v.python }}{{ v.variant }} {% if v.has_data %}{{ v.astropy_version }}{% if v.extra_index_urls %} (nightly){% endif %}{% else %}-{% endif %}{% if v.has_data %}{{ v.python_version }}{% else %}-{% endif %} {% if v.has_data %}{{ v.started_at }}{% else %}-{% endif %} {% if v.has_data %}{{ v.finished_at or "(in progress)" }}{% else %}-{% endif %}
- - - {% for v in variants %}{% endfor %} - + + + + {% for p in pythons %}{% endfor %} + + + {% for p in pythons %}{% for v in variants %}{% endfor %}{% endfor %} + + {% for row in rows %} - {% for v in variants %} - {% set cell = row.cells[v.name] %} + {% for cell in row.cells %}
Package{{ v.name }}
PackagePython {{ p }}
{{ v }}
{{ row.name }} {% if cell.anchor %} {{ cell.label }} @@ -62,7 +66,7 @@

Failure logs

{{ f.name }} - ({{ f.variant }}) + ({{ f.variant }} / Python {{ f.python }}) {{ f.label }} {% if f.install_error %} From f10df45bb9697ec442b9ff3fada774ad34112155 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Mon, 11 May 2026 13:51:08 +0100 Subject: [PATCH 08/17] Remove old infrastructure --- .github/workflows/integration_testing.yml | 39 ------------- .gitignore | 3 - README.md | 17 +++--- pyproject.toml | 8 --- tox.ini | 69 ----------------------- 5 files changed, 8 insertions(+), 128 deletions(-) delete mode 100644 .github/workflows/integration_testing.yml delete mode 100644 pyproject.toml delete mode 100644 tox.ini diff --git a/.github/workflows/integration_testing.yml b/.github/workflows/integration_testing.yml deleted file mode 100644 index 8a1b737..0000000 --- a/.github/workflows/integration_testing.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: astropy_rc_basic - -on: - workflow_dispatch: - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -# These only prove that basic test suite in Linux works. -# It is up to the individual packages to do detailed testing with astropy RC. -jobs: - test: - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d9b81a07e789d1b1921ab5fe33de2ddcbbd02c3f # v2.3.0 - with: - submodules: false - envs: | - - linux: py311-asdf_astropy - - linux: py311-asdf_astropy-dev - - linux: py311-astropy_healpix - - linux: py311-astropy_healpix-dev - - linux: py311-ccdproc - - linux: py311-ccdproc-dev - - linux: py311-photutils - - linux: py311-photutils-dev - - linux: py311-regions - - linux: py311-regions-dev - - linux: py311-reproject - - linux: py311-reproject-dev - - linux: py311-specreduce - - linux: py311-specreduce-dev - - linux: py311-specutils - - linux: py311-specutils-dev - - linux: py312-sunpy - - linux: py312-sunpy-dev diff --git a/.gitignore b/.gitignore index 40a230f..1b10245 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -.tox -.hypothesis -result_images .tmp/ results/ site/ diff --git a/README.md b/README.md index ec74e7a..4a10ecd 100644 --- a/README.md +++ b/README.md @@ -40,13 +40,14 @@ What's in the repo | File | Purpose | |-------------------------------------|------------------------------------------------------| -| `packages.yaml` | The list of packages tested (one block per package). | -| `run_integration.py` | Runs one variant: resolve specs, install, test, write `results/.json`. | +| `packages.yaml` | The list of packages tested + `python_versions` to test against. | +| `run_integration.py` | Runs one or more (variant, python) combos: resolve specs, install, test, write `results/__.json`. | | `build_dashboard.py` | Reads `results/*.json`, renders `site/index.html` (single self-contained page). | | `status.py` | Shared status vocabulary (used by both scripts). | | `templates/` | HTML/CSS for the dashboard. | -| `.github/workflows/integration.yml` | The new matrix workflow (variant x3 + dashboard). | -| `tox.ini`, `sunpy_pytest.ini`, `.github/workflows/integration_testing.yml` | Legacy tox setup, kept for now. | +| `.github/workflows/integration.yml` | The matrix workflow (variant x python + dashboard). | +| `.github/workflows/preview-link.yml`| Companion that posts the "View dashboard preview" status check on PRs. | +| `sunpy_pytest.ini` | Custom pytest config referenced by sunpy's `pytest_args` (sunpy's own config requires plugins we don't install). | Running locally --------------- @@ -114,11 +115,9 @@ Triggering a run from GitHub 1. Actions tab -> `integration-matrix` workflow. 2. "Run workflow" dropdown -> green button. -3. Three variant jobs run in parallel; the `dashboard` job waits for - them and publishes to `gh-pages`. - -The legacy `astropy_rc_basic` (tox-based) workflow is still present -and triggerable separately. +3. The matrix expands to `len(variants) x len(python_versions)` + parallel jobs; the `dashboard` job waits for them and publishes + to `gh-pages`. PR previews ----------- diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 6ff410d..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,8 +0,0 @@ -[tool.pytest.ini_options] -astropy_header = true -doctest_plus = "enabled" -text_file_format = "rst" -filterwarnings = [ - "ignore:numpy\\.ufunc size changed:RuntimeWarning", - "ignore:numpy\\.ndarray size changed:RuntimeWarning", -] diff --git a/tox.ini b/tox.ini deleted file mode 100644 index a1b0f65..0000000 --- a/tox.ini +++ /dev/null @@ -1,69 +0,0 @@ -[tox] -# We define an environment per package so that we can then run 'tox' on its own -# and have all packages be tested even if some have failures. -envlist = - # Check compatibility of Coordinated Packages. Additionally: - # sunpy - # These are widely used packages that have historically - # had issues with some astropy releases, so we include them here - # as regression testing. - py{311,312,313,314}-{all,asdf_astropy,astropy_healpix,astroquery,ccdproc,photutils,regions,reproject,specreduce,specutils,sunpy}{,-dev} - -[testenv] -# Pass through the following environment variables which are needed for the CI -passenv = HOME,CI - -pip_pre = true - -# Note that we install all dependencies in all environments to catch any -# side effects and make sure all test suites pass with all packages -deps = - astropy[all,test] - - asdf_astropy-!dev,all-!dev: asdf_astropy[test] - asdf_astropy-dev,all-dev: asdf_astropy[test] @ git+https://github.com/astropy/asdf-astropy.git - - astropy_healpix-!dev,all-!dev: astropy_healpix[test] - astropy_healpix-dev,all-dev: astropy_healpix[test] @ git+https://github.com/astropy/astropy-healpix.git - - # https://github.com/astropy/astropy-integration-testing/issues/30 - #astroquery-!dev,all-!dev: astroquery[test,all] - #astroquery-dev,all-dev: astroquery[test,all] @ git+https://github.com/astropy/astroquery.git - - ccdproc,all: psutil - ccdproc-!dev,all-!dev: ccdproc[test,all] - ccdproc-dev,all-dev: ccdproc[test,all] @ git+https://github.com/astropy/ccdproc.git - - photutils-!dev,all-!dev: photutils[test,all] - photutils-dev,all-dev: photutils[test,all] @ git+https://github.com/astropy/photutils.git - - regions-!dev,all-!dev: regions[test,all] - regions-dev,all-dev: regions[test,all] @ git+https://github.com/astropy/regions.git - - reproject,all: gwcs - reproject-!dev,all-!dev: reproject[test,all] - reproject-dev,all-dev: reproject[test,all] @ git+https://github.com/astropy/reproject.git - - specreduce-!dev,all-!dev: specreduce[test] - specreduce-dev,all-dev: specreduce[test] @ git+https://github.com/astropy/specreduce.git - - specutils-!dev,all-!dev: specutils[all,test] - specutils-dev,all-dev: specutils[all,test] @ git+https://github.com/astropy/specutils.git - - sunpy-!dev,all-!dev: sunpy[tests,all] - sunpy-dev,all-dev: sunpy[tests,all] @ git+https://github.com/sunpy/sunpy.git - -skip_install = true - -commands = - {list_dependencies_command} - asdf_astropy,all: pytest --pyargs asdf_astropy - astropy_healpix,all: pytest --pyargs astropy_healpix - astroquery,all: pytest --pyargs astroquery -k "not test_deprecated_namespace_import_warning and not test_raises_deprecation_warning" - ccdproc,all: pytest --pyargs ccdproc - photutils,all: pytest --pyargs photutils - regions,all: pytest --pyargs regions - reproject,all: pytest --pyargs reproject --ignore reproject/spherical_intersect/setup_package.py - specreduce,all: pytest --pyargs specreduce - specutils,all: pytest --pyargs specutils - sunpy,all: pytest --pyargs sunpy -c sunpy_pytest.ini From 10dfa607d27fe4217700e5dc91d9c7d239bd949d Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 May 2026 00:10:00 +0100 Subject: [PATCH 09/17] Add affiliated packages and make improvements to infrastructure --- .github/workflows/integration.yml | 18 ++- build_dashboard.py | 22 ++- packages.yaml | 225 ++++++++++++++++++++++++++++++ run_integration.py | 11 +- templates/_style.css | 4 + templates/index.html | 15 +- 6 files changed, 288 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9d00d69..d0769f8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -36,11 +36,21 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v3 + - name: Compute cache key suffix (ISO week) + id: cache_key + run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT" + - name: Cache ~/.cache/uv uses: actions/cache@v4 with: path: ~/.cache/uv - key: uv-${{ runner.os }}-${{ matrix.variant }}-${{ matrix.python }}-${{ hashFiles('packages.yaml') }} + # Wheels for a given version are immutable on PyPI, so we + # rotate the primary key weekly: same-week runs hit the + # primary key (no re-save), the first run each week restores + # from last week's cache via restore-keys and saves a fresh + # one. Keeps cache storage bounded while still picking up + # newly-released wheels eventually. + key: uv-${{ runner.os }}-${{ matrix.variant }}-${{ matrix.python }}-${{ steps.cache_key.outputs.week }} restore-keys: | uv-${{ runner.os }}-${{ matrix.variant }}-${{ matrix.python }}- uv-${{ runner.os }}-${{ matrix.variant }}- @@ -50,7 +60,11 @@ jobs: run: pip install packaging pyyaml - name: Run ${{ matrix.variant }} variant on Python ${{ matrix.python }} - run: python run_integration.py --variant ${{ matrix.variant }} --python ${{ matrix.python }} + env: + # Short timeout for PR previews so the matrix finishes in + # minutes; full timeout for scheduled/dispatch runs. + TIMEOUT_FLAG: ${{ github.event_name == 'pull_request' && '--timeout-test 20' || '' }} + run: python run_integration.py --variant ${{ matrix.variant }} --python ${{ matrix.python }} $TIMEOUT_FLAG - name: Upload results if: always() diff --git a/build_dashboard.py b/build_dashboard.py index 7d9770f..a56486b 100644 --- a/build_dashboard.py +++ b/build_dashboard.py @@ -90,6 +90,7 @@ def _make_rows(by_combo, names, columns): rows = [] for name in names: cells = [] + row_tier = None for python, variant in columns: data = by_combo.get((variant, python)) entry = None @@ -101,6 +102,8 @@ def _make_rows(by_combo, names, columns): "anchor": "", "resolved_version": "", }) continue + if row_tier is None: + row_tier = entry.get("tier", "coordinated") badge = status.cell_badge(entry) anchor = (_anchor_id(name, variant, python) if badge["status"] not in ("pass", "missing") else "") @@ -109,10 +112,17 @@ def _make_rows(by_combo, names, columns): "anchor": anchor, "resolved_version": entry.get("resolved_version", ""), }) - rows.append({"name": name, "cells": cells}) + rows.append({"name": name, "tier": row_tier or "coordinated", "cells": cells}) return rows +def _group_rows_by_tier(rows): + """Group rows by tier, preserving order. Returns [(tier, [rows]), ...].""" + from itertools import groupby + return [(tier, list(group)) + for tier, group in groupby(rows, key=lambda r: r["tier"])] + + def _failures(by_combo, columns): """Per-cell failure detail (full logs), ordered by column then row.""" out = [] @@ -153,17 +163,25 @@ def build(results_dir, output_dir, templates_dir): pythons, columns = _column_groups(by_combo) names = _ordered_packages(by_combo) rows = _make_rows(by_combo, names, columns) + tier_groups = _group_rows_by_tier(rows) failures = _failures(by_combo, columns) variants_meta = [_variant_meta(v, p, by_combo.get((v, p))) for p in pythons for v in VARIANTS] + # If any variant ran with an unusually short test timeout, surface + # it in a banner; that's typical for PR preview runs. + timeouts = {d.get("timeout_test_seconds") for d in by_combo.values() + if d and d.get("timeout_test_seconds")} + short_timeout = min((t for t in timeouts if t and t < 600), default=None) + (output_dir / "index.html").write_text( env.get_template("index.html").render( pythons=pythons, variants=list(VARIANTS), variants_meta=variants_meta, - rows=rows, + tier_groups=tier_groups, failures=failures, + short_timeout=short_timeout, ) ) print(f"Wrote {output_dir}/index.html") diff --git a/packages.yaml b/packages.yaml index 6b6f32f..f769742 100644 --- a/packages.yaml +++ b/packages.yaml @@ -82,3 +82,228 @@ packages: repo_url: https://github.com/sunpy/sunpy.git install_extras: [tests, all] pytest_args: ["-c", "sunpy_pytest.ini"] + + # --- Affiliated packages auto-imported from + # https://github.com/astropy/astropy.github.com/blob/main/affiliated/registry.json + # Module names defaulted to pypi_name with hyphens/dots replaced by underscores; + # individual packages may need overrides if pytest --pyargs fails to find them. + - pypi_name: agnpy + tier: affiliated + module: agnpy + repo_url: https://github.com/cosimoNigro/agnpy.git + + - pypi_name: APLpy + tier: affiliated + module: APLpy + repo_url: https://github.com/aplpy/aplpy.git + + - pypi_name: astroalign + tier: affiliated + module: astroalign + repo_url: https://github.com/quatrope/astroalign.git + + - pypi_name: astroML + tier: affiliated + module: astroML + repo_url: https://github.com/astroML/astroML.git + + - pypi_name: astroplan + tier: affiliated + module: astroplan + repo_url: https://github.com/astropy/astroplan.git + + - pypi_name: astroscrappy + tier: affiliated + module: astroscrappy + repo_url: https://github.com/astropy/astroscrappy.git + + - pypi_name: baseband + tier: affiliated + module: baseband + repo_url: https://github.com/mhvk/baseband.git + + - pypi_name: BayesicFitting + tier: affiliated + module: BayesicFitting + repo_url: https://github.com/dokester/BayesicFitting.git + + - pypi_name: cluster-lensing + tier: affiliated + module: cluster_lensing + repo_url: https://github.com/jesford/cluster-lensing.git + + - pypi_name: corral-pipeline + tier: affiliated + module: corral_pipeline + repo_url: https://github.com/toros-astro/corral.git + + - pypi_name: dust_extinction + tier: affiliated + module: dust_extinction + repo_url: https://github.com/karllark/dust_extinction.git + + - pypi_name: einsteinpy + tier: affiliated + module: einsteinpy + repo_url: https://github.com/einsteinpy/einsteinpy.git + + - pypi_name: feets + tier: affiliated + module: feets + repo_url: https://github.com/carpyncho/feets.git + + - pypi_name: gala + tier: affiliated + module: gala + repo_url: https://github.com/adrn/gala.git + + - pypi_name: galpy + tier: affiliated + module: galpy + repo_url: https://github.com/jobovy/galpy.git + + - pypi_name: gammapy + tier: affiliated + module: gammapy + repo_url: https://github.com/gammapy/gammapy.git + + - pypi_name: ginga + tier: affiliated + module: ginga + repo_url: https://github.com/ejeschke/ginga.git + + - pypi_name: glueviz + tier: affiliated + module: glueviz + repo_url: https://github.com/glue-viz/glue.git + + - pypi_name: gwcs + tier: affiliated + module: gwcs + repo_url: https://github.com/spacetelescope/gwcs.git + + - pypi_name: halotools + tier: affiliated + module: halotools + repo_url: https://github.com/astropy/halotools.git + + - pypi_name: hendrics + tier: affiliated + module: hendrics + repo_url: https://github.com/StingraySoftware/HENDRICS.git + + - pypi_name: hips + tier: affiliated + module: hips + repo_url: https://github.com/hipspy/hips.git + + - pypi_name: imexam + tier: affiliated + module: imexam + repo_url: https://github.com/spacetelescope/imexam.git + + - pypi_name: kanon + tier: affiliated + module: kanon + repo_url: https://github.com/legau/kanon.git + + - pypi_name: lenstronomy + tier: affiliated + module: lenstronomy + repo_url: https://github.com/lenstronomy/lenstronomy.git + + - pypi_name: ligo.skymap + tier: affiliated + module: ligo_skymap + repo_url: https://git.ligo.org/lscsoft/ligo.skymap.git + + - pypi_name: linetools + tier: affiliated + module: linetools + repo_url: https://github.com/linetools/linetools.git + + - pypi_name: marxs + tier: affiliated + module: marxs + repo_url: https://github.com/Chandra-MARX/marxs.git + + - pypi_name: mocpy + tier: affiliated + module: mocpy + repo_url: https://github.com/cds-astro/mocpy.git + + - pypi_name: naima + tier: affiliated + module: naima + repo_url: https://github.com/zblz/naima.git + + - pypi_name: omnifit + tier: affiliated + module: omnifit + repo_url: https://github.com/RiceMunk/omnifit.git + + - pypi_name: poliastro + tier: affiliated + module: poliastro + repo_url: https://github.com/poliastro/poliastro.git + + - pypi_name: pycbc + tier: affiliated + module: pycbc + repo_url: https://github.com/gwastro/pycbc.git + + - pypi_name: pydl + tier: affiliated + module: pydl + repo_url: https://github.com/weaverba137/pydl.git + + - pypi_name: pyregion + tier: affiliated + module: pyregion + repo_url: https://github.com/astropy/pyregion.git + + - pypi_name: pyspeckit + tier: affiliated + module: pyspeckit + repo_url: https://github.com/pyspeckit/pyspeckit.git + + - pypi_name: pyvo + tier: affiliated + module: pyvo + repo_url: https://github.com/astropy/pyvo.git + + - pypi_name: regularizepsf + tier: affiliated + module: regularizepsf + repo_url: https://github.com/punch-mission/regularizepsf.git + + - pypi_name: sbpy + tier: affiliated + module: sbpy + repo_url: https://github.com/NASA-Planetary-Science/sbpy.git + + - pypi_name: sncosmo + tier: affiliated + module: sncosmo + repo_url: https://github.com/sncosmo/sncosmo.git + + - pypi_name: spectral-cube + tier: affiliated + module: spectral_cube + repo_url: https://github.com/radio-astro-tools/spectral-cube.git + + - pypi_name: spherical-geometry + tier: affiliated + module: spherical_geometry + repo_url: https://github.com/spacetelescope/spherical_geometry.git + + - pypi_name: statmorph + tier: affiliated + module: statmorph + repo_url: https://github.com/vrodgom/statmorph.git + + - pypi_name: synphot + tier: affiliated + module: synphot + repo_url: https://github.com/spacetelescope/synphot_refactor.git + diff --git a/run_integration.py b/run_integration.py index c63ce1a..c1f84a8 100644 --- a/run_integration.py +++ b/run_integration.py @@ -268,6 +268,7 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo }, "python_requested": python_version, "python_version": "", + "timeout_test_seconds": timeouts["test"], "fatal_error": "", "installed_deps": {}, "packages": [], @@ -297,8 +298,14 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo common += [f"--index-strategy={astropy['index_strategy']}"] print("\nInstalling astropy + pytest...") - rc, err = _run_install(common + [astropy["install"], "pytest", "pytest-timeout"], - timeouts["install"]) + # pytest-remotedata registers the `remote_data` marker many + # astropy ecosystem packages use; with the plugin installed but + # `--remote-data` not passed, those tests are skipped automatically + # instead of running and timing out on network calls. + rc, err = _run_install( + common + [astropy["install"], "pytest", "pytest-timeout", "pytest-remotedata"], + timeouts["install"], + ) if rc != 0: print(" FATAL: astropy install failed") print(err) diff --git a/templates/_style.css b/templates/_style.css index ae5b8c0..8fb1755 100644 --- a/templates/_style.css +++ b/templates/_style.css @@ -6,6 +6,10 @@ table { border-collapse: collapse; } th, td { border: 1px solid #ddd; padding: 6px 10px; text-align: left; vertical-align: top; } th { background: #f6f8fa; } th.python-group { text-align: center; border-bottom: 2px solid #ccc; } +.banner { background: #fff8c5; border: 1px solid #d4a72c; padding: 0.75em 1em; + border-radius: 6px; margin: 1em 0; } +tr.tier-header th { background: #eef2f6; text-align: left; font-weight: 600; + text-transform: lowercase; letter-spacing: 0.5px; color: #444; } .meta dl { margin: 0.3em 0; display: grid; grid-template-columns: max-content 1fr; gap: 0.2em 1em; color: #444; } .meta dt { font-weight: 600; } .pill { display: inline-block; padding: 2px 10px; border-radius: 12px; font-size: 0.85em; font-weight: 600; min-width: 4em; text-align: center; } diff --git a/templates/index.html b/templates/index.html index 4fb1684..661eb55 100644 --- a/templates/index.html +++ b/templates/index.html @@ -3,6 +3,14 @@ {% block body %}

Astropy ecosystem integration matrix

+{% if short_timeout %} + +{% endif %} + @@ -30,7 +38,11 @@

Matrix

-{% for row in rows %} +{% for tier, group in tier_groups %} + + + +{% for row in group %} {% for cell in row.cells %} @@ -45,6 +57,7 @@

Matrix

{% endfor %} {% endfor %} +{% endfor %}
PythonVariantAstropyStartedFinished
{{ tier|capitalize }}
{{ row.name }}
From 7a5a9c99a0b698e9b8fb0cf3b015e9c95ae55ec6 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Tue, 12 May 2026 00:45:52 +0100 Subject: [PATCH 10/17] Fix more issues and only run first 10 tests in PRs --- .github/workflows/integration.yml | 9 ++++---- README.md | 10 +++++---- build_dashboard.py | 12 +++++------ conftest.py | 24 +++++++++++++++++++++ packages.yaml | 35 +++++++++++++++++++++++++++---- run_integration.py | 3 +++ templates/index.html | 8 +++---- 7 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 conftest.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d0769f8..94ffc8e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -61,10 +61,11 @@ jobs: - name: Run ${{ matrix.variant }} variant on Python ${{ matrix.python }} env: - # Short timeout for PR previews so the matrix finishes in - # minutes; full timeout for scheduled/dispatch runs. - TIMEOUT_FLAG: ${{ github.event_name == 'pull_request' && '--timeout-test 20' || '' }} - run: python run_integration.py --variant ${{ matrix.variant }} --python ${{ matrix.python }} $TIMEOUT_FLAG + # PR previews limit each package to the first 10 tests so the + # matrix finishes in minutes; conftest.py at the repo root reads + # this env var and truncates the collected items. + PYTEST_LIMIT_N: ${{ github.event_name == 'pull_request' && '10' || '' }} + run: python run_integration.py --variant ${{ matrix.variant }} --python ${{ matrix.python }} - name: Upload results if: always() diff --git a/README.md b/README.md index 4a10ecd..dc70a88 100644 --- a/README.md +++ b/README.md @@ -132,10 +132,12 @@ whose "Details" link opens the rendered page directly in the browser. This means the PR preview reflects *this PR's actual matrix run*, -not last main's data. It's also slower than a render-only preview -would be — expect the same wall-clock as a normal main run, up to -a few hours per push. Concurrency cancels in-progress PR runs when -a new push lands, so only the latest push consumes CI time. +not last main's data. To keep PR feedback fast, each package is +capped at the first 10 collected tests (via `PYTEST_LIMIT_N=10`, +applied by the repo-level `conftest.py`); the preview is a smoke +check of layout, install resolution, and the workflow itself, not +a full regression signal. Concurrency cancels in-progress PR runs +when a new push lands, so only the latest push consumes CI time. `preview-link.yml` lives at `.github/workflows/preview-link.yml` and must be on the default branch for its `workflow_run` trigger diff --git a/build_dashboard.py b/build_dashboard.py index a56486b..de3a96c 100644 --- a/build_dashboard.py +++ b/build_dashboard.py @@ -168,11 +168,11 @@ def build(results_dir, output_dir, templates_dir): variants_meta = [_variant_meta(v, p, by_combo.get((v, p))) for p in pythons for v in VARIANTS] - # If any variant ran with an unusually short test timeout, surface - # it in a banner; that's typical for PR preview runs. - timeouts = {d.get("timeout_test_seconds") for d in by_combo.values() - if d and d.get("timeout_test_seconds")} - short_timeout = min((t for t in timeouts if t and t < 600), default=None) + # If any variant ran with a per-package test limit, surface it in + # a banner; PR previews use this to keep wall time bounded. + limits = {d.get("pytest_limit_n") for d in by_combo.values() + if d and d.get("pytest_limit_n")} + pytest_limit = min(limits) if limits else None (output_dir / "index.html").write_text( env.get_template("index.html").render( @@ -181,7 +181,7 @@ def build(results_dir, output_dir, templates_dir): variants_meta=variants_meta, tier_groups=tier_groups, failures=failures, - short_timeout=short_timeout, + pytest_limit=pytest_limit, ) ) print(f"Wrote {output_dir}/index.html") diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..5180a3f --- /dev/null +++ b/conftest.py @@ -0,0 +1,24 @@ +"""Repo-level pytest plugin used during PR-preview runs. + +When the env var `PYTEST_LIMIT_N` is set to a positive integer, pytest +collects normally and then truncates the test list to the first N +items. This lets the PR matrix surface a fast smoke signal per +package (typically 10 tests each) without per-package configuration. + +Discovered automatically by pytest because cwd is the repo root for +every `pytest --pyargs ` invocation in the runner. +""" + +import os + + +def pytest_collection_modifyitems(config, items): + raw = os.environ.get("PYTEST_LIMIT_N") + if not raw: + return + try: + n = int(raw) + except ValueError: + return + if n > 0 and len(items) > n: + del items[n:] diff --git a/packages.yaml b/packages.yaml index f769742..b57cb84 100644 --- a/packages.yaml +++ b/packages.yaml @@ -94,8 +94,9 @@ packages: - pypi_name: APLpy tier: affiliated - module: APLpy + module: aplpy # installed package name is lowercase repo_url: https://github.com/aplpy/aplpy.git + install_extras: [test] - pypi_name: astroalign tier: affiliated @@ -126,10 +127,13 @@ packages: tier: affiliated module: BayesicFitting repo_url: https://github.com/dokester/BayesicFitting.git + # BayesicFitting ships unittest-style tests as Test*.py (capital T); + # override pytest's default `python_files` glob so it picks them up. + pytest_args: ["-o", "python_files=Test*.py"] - pypi_name: cluster-lensing tier: affiliated - module: cluster_lensing + module: clusterlensing # installed package has no underscore repo_url: https://github.com/jesford/cluster-lensing.git - pypi_name: corral-pipeline @@ -166,15 +170,21 @@ packages: tier: affiliated module: gammapy repo_url: https://github.com/gammapy/gammapy.git + install_extras: [test] - pypi_name: ginga tier: affiliated module: ginga repo_url: https://github.com/ejeschke/ginga.git + # `ginga/tests/test_zarr.py` uses removed zarr 3.x API; ignore it + # so the rest of the suite (256 passing tests) can run. + pytest_args: ["--ignore-glob=*test_zarr.py"] - pypi_name: glueviz tier: affiliated - module: glueviz + # glueviz is a meta-package; the actual code (and tests) lives in + # glue-core which it pulls in. Its installed module is `glue`. + module: glue repo_url: https://github.com/glue-viz/glue.git - pypi_name: gwcs @@ -191,6 +201,7 @@ packages: tier: affiliated module: hendrics repo_url: https://github.com/StingraySoftware/HENDRICS.git + install_extras: [test] - pypi_name: hips tier: affiliated @@ -214,23 +225,36 @@ packages: - pypi_name: ligo.skymap tier: affiliated - module: ligo_skymap + # PEP 420 namespace package: installed as ligo/skymap/, not ligo_skymap. + module: ligo.skymap repo_url: https://git.ligo.org/lscsoft/ligo.skymap.git + install_extras: [test] - pypi_name: linetools tier: affiliated module: linetools repo_url: https://github.com/linetools/linetools.git + install_extras: [test] + # linetools imports pkg_resources (deprecated stdlib alias); pull + # in setuptools so it's available at runtime. + extra_deps: [setuptools] - pypi_name: marxs tier: affiliated module: marxs repo_url: https://github.com/Chandra-MARX/marxs.git + install_extras: [test] + # Two test files require an external MARX C library we don't have; + # ignoring them lets the rest of the suite collect. + pytest_args: + - "--ignore-glob=*test_hrma.py" + - "--ignore-glob=*test_all_optics.py" - pypi_name: mocpy tier: affiliated module: mocpy repo_url: https://github.com/cds-astro/mocpy.git + install_extras: [dev] - pypi_name: naima tier: affiliated @@ -256,6 +280,7 @@ packages: tier: affiliated module: pydl repo_url: https://github.com/weaverba137/pydl.git + install_extras: [test] - pypi_name: pyregion tier: affiliated @@ -271,6 +296,7 @@ packages: tier: affiliated module: pyvo repo_url: https://github.com/astropy/pyvo.git + install_extras: [test] - pypi_name: regularizepsf tier: affiliated @@ -291,6 +317,7 @@ packages: tier: affiliated module: spectral_cube repo_url: https://github.com/radio-astro-tools/spectral-cube.git + install_extras: [test] - pypi_name: spherical-geometry tier: affiliated diff --git a/run_integration.py b/run_integration.py index c1f84a8..8a7cf5e 100644 --- a/run_integration.py +++ b/run_integration.py @@ -269,6 +269,9 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo "python_requested": python_version, "python_version": "", "timeout_test_seconds": timeouts["test"], + "pytest_limit_n": (int(os.environ["PYTEST_LIMIT_N"]) + if (os.environ.get("PYTEST_LIMIT_N") or "").isdigit() + else None), "fatal_error": "", "installed_deps": {}, "packages": [], diff --git a/templates/index.html b/templates/index.html index 661eb55..3602b77 100644 --- a/templates/index.html +++ b/templates/index.html @@ -3,11 +3,11 @@ {% block body %}

Astropy ecosystem integration matrix

-{% if short_timeout %} +{% if pytest_limit %} {% endif %} From 1e11a2395bf02078919734c611ca09cf28e07ada Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 May 2026 11:41:42 +0100 Subject: [PATCH 11/17] Tidy up into package --- .github/workflows/integration.yml | 12 +++---- .gitignore | 1 + astropy_integration/__init__.py | 0 astropy_integration/cli.py | 32 +++++++++++++++++ .../dashboard.py | 21 ++++++------ .../run.py | 34 +++++++------------ status.py => astropy_integration/status.py | 0 .../templates}/_base.html | 0 .../templates}/_style.css | 0 .../templates}/index.html | 0 pyproject.toml | 23 +++++++++++++ 11 files changed, 84 insertions(+), 39 deletions(-) create mode 100644 astropy_integration/__init__.py create mode 100644 astropy_integration/cli.py rename build_dashboard.py => astropy_integration/dashboard.py (95%) rename run_integration.py => astropy_integration/run.py (94%) rename status.py => astropy_integration/status.py (100%) rename {templates => astropy_integration/templates}/_base.html (100%) rename {templates => astropy_integration/templates}/_style.css (100%) rename {templates => astropy_integration/templates}/index.html (100%) create mode 100644 pyproject.toml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 94ffc8e..1d5c92b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -56,8 +56,8 @@ jobs: uv-${{ runner.os }}-${{ matrix.variant }}- uv-${{ runner.os }}- - - name: Install runtime deps - run: pip install packaging pyyaml + - name: Install astropy-integration + run: pip install -e . - name: Run ${{ matrix.variant }} variant on Python ${{ matrix.python }} env: @@ -65,7 +65,7 @@ jobs: # matrix finishes in minutes; conftest.py at the repo root reads # this env var and truncates the collected items. PYTEST_LIMIT_N: ${{ github.event_name == 'pull_request' && '10' || '' }} - run: python run_integration.py --variant ${{ matrix.variant }} --python ${{ matrix.python }} + run: astropy-integration run --variant ${{ matrix.variant }} --python ${{ matrix.python }} - name: Upload results if: always() @@ -86,8 +86,8 @@ jobs: with: python-version: '3.12' - - name: Install runtime deps - run: pip install jinja2 + - name: Install astropy-integration + run: pip install -e . - name: Download all variant results uses: actions/download-artifact@v4 @@ -97,7 +97,7 @@ jobs: path: results/ - name: Build dashboard - run: python build_dashboard.py + run: astropy-integration dashboard # PR path: upload index.html as a non-zipped artifact for the # in-browser preview (linked from a PR check by preview-link.yml). diff --git a/.gitignore b/.gitignore index 1b10245..99fac6d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ results/ site/ __pycache__/ *.pyc +*.egg-info/ diff --git a/astropy_integration/__init__.py b/astropy_integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/astropy_integration/cli.py b/astropy_integration/cli.py new file mode 100644 index 0000000..39e1e0f --- /dev/null +++ b/astropy_integration/cli.py @@ -0,0 +1,32 @@ +"""Console entry point: dispatches `run` and `dashboard` subcommands.""" + +import argparse + +from . import dashboard, run + + +def main(): + parser = argparse.ArgumentParser(prog="astropy-integration") + sub = parser.add_subparsers(dest="cmd", required=True) + + run_p = sub.add_parser( + "run", + help="Run one or more (variant, python) cells of the matrix.", + description=run.__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + run.add_arguments(run_p) + + dash_p = sub.add_parser( + "dashboard", + help="Build the dashboard HTML from results/*.json.", + description=dashboard.__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + dashboard.add_arguments(dash_p) + + args = parser.parse_args() + if args.cmd == "run": + run.run(args) + elif args.cmd == "dashboard": + dashboard.run(args) diff --git a/build_dashboard.py b/astropy_integration/dashboard.py similarity index 95% rename from build_dashboard.py rename to astropy_integration/dashboard.py index de3a96c..9a8ecb7 100644 --- a/build_dashboard.py +++ b/astropy_integration/dashboard.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python """Build the single-page integration-matrix dashboard. -Reads every results/__.json that run_integration.py +Reads every results/__.json that `astropy-integration run` wrote and emits a single self-contained `site/index.html` with: - one row per package - one column group per Python version, subdivided into the three @@ -10,15 +9,18 @@ for every non-passing cell, anchored from the matching badge """ -import argparse import json import re import shutil +from importlib import resources from pathlib import Path from jinja2 import Environment, FileSystemLoader, select_autoescape -import status +from . import status + + +DEFAULT_TEMPLATES_DIR = str(resources.files(__package__) / "templates") VARIANTS = status.VARIANTS @@ -187,14 +189,11 @@ def build(results_dir, output_dir, templates_dir): print(f"Wrote {output_dir}/index.html") -def main(): - ap = argparse.ArgumentParser() +def add_arguments(ap): ap.add_argument("--results-dir", default="results") ap.add_argument("--output", default="site") - ap.add_argument("--templates-dir", default="templates") - args = ap.parse_args() - build(args.results_dir, args.output, args.templates_dir) + ap.add_argument("--templates-dir", default=DEFAULT_TEMPLATES_DIR) -if __name__ == "__main__": - main() +def run(args): + build(args.results_dir, args.output, args.templates_dir) diff --git a/run_integration.py b/astropy_integration/run.py similarity index 94% rename from run_integration.py rename to astropy_integration/run.py index 8a7cf5e..96804b2 100644 --- a/run_integration.py +++ b/astropy_integration/run.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Run one variant of the astropy ecosystem integration matrix. Creates a single shared venv, installs astropy first then each package @@ -9,12 +8,12 @@ data. Usage: - python run_integration.py # full matrix (all variants x all Python versions from config) - python run_integration.py --variant stable # one variant, all configured Pythons - python run_integration.py --variant stable --python 3.12 # one combo - python run_integration.py --python 3.14t # all variants on free-threaded 3.14 - python run_integration.py --variant stable --packages reproject,sunpy - python run_integration.py --variant stable --tiers coordinated + astropy-integration run # full matrix (all variants x all Python versions from config) + astropy-integration run --variant stable # one variant, all configured Pythons + astropy-integration run --variant stable --python 3.12 # one combo + astropy-integration run --python 3.14t # all variants on free-threaded 3.14 + astropy-integration run --variant stable --packages reproject,sunpy + astropy-integration run --variant stable --tiers coordinated """ import argparse @@ -31,14 +30,10 @@ from pathlib import Path -try: - import yaml - from packaging.version import Version, InvalidVersion -except ImportError: - sys.exit("This script requires 'pyyaml' and 'packaging'. " - "Install with: pip install pyyaml packaging") +import yaml +from packaging.version import Version, InvalidVersion -import status +from . import status # Make every print flush immediately so CI streams progress live # instead of buffering until the script exits. @@ -411,9 +406,7 @@ def _counts(result): return {"install": dict(install), "test": dict(test)} -def main(): - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) +def add_arguments(ap): ap.add_argument("--config", default="packages.yaml") ap.add_argument("--results-dir", default="results") ap.add_argument("--variant", choices=status.VARIANTS, @@ -427,8 +420,9 @@ def main(): "default: all tiers") ap.add_argument("--timeout-install", type=int, default=900) ap.add_argument("--timeout-test", type=int, default=1800) - args = ap.parse_args() + +def run(args): packages = _load_packages(args.config) if args.tiers: wanted_tiers = {t.strip() for t in args.tiers.split(",") if t.strip()} @@ -458,7 +452,3 @@ def main(): if fatal_combos: sys.exit(f"\nAstropy install failed for: {', '.join(fatal_combos)}") - - -if __name__ == "__main__": - main() diff --git a/status.py b/astropy_integration/status.py similarity index 100% rename from status.py rename to astropy_integration/status.py diff --git a/templates/_base.html b/astropy_integration/templates/_base.html similarity index 100% rename from templates/_base.html rename to astropy_integration/templates/_base.html diff --git a/templates/_style.css b/astropy_integration/templates/_style.css similarity index 100% rename from templates/_style.css rename to astropy_integration/templates/_style.css diff --git a/templates/index.html b/astropy_integration/templates/index.html similarity index 100% rename from templates/index.html rename to astropy_integration/templates/index.html diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7dcfe9b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "astropy-integration" +version = "0.1.0" +description = "Run and dashboard the astropy ecosystem integration matrix." +requires-python = ">=3.11" +dependencies = [ + "jinja2", + "packaging", + "pyyaml", +] + +[project.scripts] +astropy-integration = "astropy_integration.cli:main" + +[tool.setuptools.packages.find] +include = ["astropy_integration*"] + +[tool.setuptools.package-data] +astropy_integration = ["templates/*.html", "templates/*.css"] From 0ba2e0b7625f99e886742a29d889f5b79d22a1e1 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 May 2026 11:51:30 +0100 Subject: [PATCH 12/17] Improve uv cache handling --- .github/workflows/integration.yml | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1d5c92b..a09b534 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -35,22 +35,23 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v3 - - - name: Compute cache key suffix (ISO week) - id: cache_key - run: echo "week=$(date -u +%G-%V)" >> "$GITHUB_OUTPUT" + with: + # We manage the cache ourselves via actions/cache below; turning + # off setup-uv's own caching avoids it saving a duplicate of the + # same directory under its own key. + enable-cache: false - name: Cache ~/.cache/uv uses: actions/cache@v4 with: path: ~/.cache/uv - # Wheels for a given version are immutable on PyPI, so we - # rotate the primary key weekly: same-week runs hit the - # primary key (no re-save), the first run each week restores - # from last week's cache via restore-keys and saves a fresh - # one. Keeps cache storage bounded while still picking up - # newly-released wheels eventually. - key: uv-${{ runner.os }}-${{ matrix.variant }}-${{ matrix.python }}-${{ steps.cache_key.outputs.week }} + # Primary key uses the run id, so every run writes a fresh + # snapshot (after `uv cache prune --ci` slims it to wheels + # actually used in this run). The restore-keys are prefix + # matches against saved keys, so the first one always finds + # the most recent run's cache for the same (os, variant, + # python) and the next run starts warm. + key: uv-${{ runner.os }}-${{ matrix.variant }}-${{ matrix.python }}-${{ github.run_id }} restore-keys: | uv-${{ runner.os }}-${{ matrix.variant }}-${{ matrix.python }}- uv-${{ runner.os }}-${{ matrix.variant }}- @@ -67,6 +68,12 @@ jobs: PYTEST_LIMIT_N: ${{ github.event_name == 'pull_request' && '10' || '' }} run: astropy-integration run --variant ${{ matrix.variant }} --python ${{ matrix.python }} + - name: Prune uv cache before save + if: always() + # Strips pre-built sdists and other entries uv won't reuse, so + # the snapshot the cache step uploads stays lean. + run: uv cache prune --ci + - name: Upload results if: always() uses: actions/upload-artifact@v4 From e0a799201620597239469614a9585dd90d2c6673 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 May 2026 12:10:06 +0100 Subject: [PATCH 13/17] Cleanup and added pre-commit --- .github/workflows/integration.yml | 4 +- .gitignore | 1 + .pre-commit-config.yaml | 14 +++ README.md | 39 ++++--- astropy_integration/dashboard.py | 103 ++++++++++------- astropy_integration/run.py | 180 +++++++++++++++++++----------- astropy_integration/status.py | 11 +- packages.yaml | 3 +- 8 files changed, 220 insertions(+), 135 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a09b534..65ca739 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -58,7 +58,7 @@ jobs: uv-${{ runner.os }}- - name: Install astropy-integration - run: pip install -e . + run: pip install . - name: Run ${{ matrix.variant }} variant on Python ${{ matrix.python }} env: @@ -94,7 +94,7 @@ jobs: python-version: '3.12' - name: Install astropy-integration - run: pip install -e . + run: pip install . - name: Download all variant results uses: actions/download-artifact@v4 diff --git a/.gitignore b/.gitignore index 99fac6d..d581eb9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ site/ __pycache__/ *.pyc *.egg-info/ +build/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7417a4f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.12 + hooks: + - id: ruff-check + args: [--fix] + - id: ruff-format diff --git a/README.md b/README.md index dc70a88..8a7f313 100644 --- a/README.md +++ b/README.md @@ -38,42 +38,45 @@ dashboard to `gh-pages`. What's in the repo ------------------ -| File | Purpose | -|-------------------------------------|------------------------------------------------------| -| `packages.yaml` | The list of packages tested + `python_versions` to test against. | -| `run_integration.py` | Runs one or more (variant, python) combos: resolve specs, install, test, write `results/__.json`. | -| `build_dashboard.py` | Reads `results/*.json`, renders `site/index.html` (single self-contained page). | -| `status.py` | Shared status vocabulary (used by both scripts). | -| `templates/` | HTML/CSS for the dashboard. | -| `.github/workflows/integration.yml` | The matrix workflow (variant x python + dashboard). | -| `.github/workflows/preview-link.yml`| Companion that posts the "View dashboard preview" status check on PRs. | -| `sunpy_pytest.ini` | Custom pytest config referenced by sunpy's `pytest_args` (sunpy's own config requires plugins we don't install). | +| File | Purpose | +|---------------------------------------|------------------------------------------------------| +| `packages.yaml` | The list of packages tested + `python_versions` to test against. | +| `astropy_integration/run.py` | Runs one or more (variant, python) combos: resolve specs, install, test, write `results/__.json`. | +| `astropy_integration/dashboard.py` | Reads `results/*.json`, renders `site/index.html` (single self-contained page). | +| `astropy_integration/cli.py` | Console entry point that dispatches the `run` and `dashboard` subcommands. | +| `astropy_integration/status.py` | Shared status vocabulary (used by both `run` and `dashboard`). | +| `astropy_integration/templates/` | HTML/CSS for the dashboard (loaded as package data). | +| `pyproject.toml` | Package metadata; declares the `astropy-integration` console script. | +| `conftest.py` | Repo-root pytest plugin that caps each package to the first `PYTEST_LIMIT_N` tests for PR previews. | +| `.github/workflows/integration.yml` | The matrix workflow (variant x python + dashboard). | +| `.github/workflows/preview-link.yml` | Companion that posts the "View dashboard preview" status check on PRs. | +| `sunpy_pytest.ini` | Custom pytest config referenced by sunpy's `pytest_args` (sunpy's own config requires plugins we don't install). | Running locally --------------- ```bash -pip install jinja2 packaging pyyaml +pip install . # uv is required; see https://docs.astral.sh/uv/ # Run one variant. Each variant takes 30-90 min depending on package count. -python run_integration.py --variant stable +astropy-integration run --variant stable # Or a single package, to iterate faster: -python run_integration.py --variant stable --packages reproject +astropy-integration run --variant stable --packages reproject # Or a tier subset (default: all tiers run): -python run_integration.py --variant stable --tiers coordinated,other +astropy-integration run --variant stable --tiers coordinated,other -# Build the dashboard from whatever results/.json files exist: -python build_dashboard.py +# Build the dashboard from whatever results/*.json files exist: +astropy-integration dashboard # Preview locally: python -m http.server -d site 8000 ``` -Results land in `results/.json`; the dashboard in `site/`. -Both directories are gitignored. +Results land in `results/__.json`; the dashboard in +`site/`. Both directories are gitignored. Python versions --------------- diff --git a/astropy_integration/dashboard.py b/astropy_integration/dashboard.py index 9a8ecb7..6b745d6 100644 --- a/astropy_integration/dashboard.py +++ b/astropy_integration/dashboard.py @@ -12,20 +12,14 @@ import json import re import shutil -from importlib import resources +from itertools import groupby from pathlib import Path -from jinja2 import Environment, FileSystemLoader, select_autoescape +from jinja2 import Environment, FileSystemLoader, PackageLoader, select_autoescape from . import status -DEFAULT_TEMPLATES_DIR = str(resources.files(__package__) / "templates") - - -VARIANTS = status.VARIANTS - - def _anchor_id(name, variant, python): # Anchors are only used inside the page so we just need # something HTML-id-safe and unique per (pkg, variant, python). @@ -57,7 +51,7 @@ def _column_groups(by_combo): columns: flat list of (python, variant) tuples in display order. """ pythons = sorted({p for _, p in by_combo}, key=lambda s: (len(s), s)) - columns = [(p, v) for p in pythons for v in VARIANTS] + columns = [(p, v) for p in pythons for v in status.VARIANTS] return pythons, columns @@ -99,30 +93,39 @@ def _make_rows(by_combo, names, columns): if data: entry = next((e for e in data["packages"] if e["name"] == name), None) if entry is None: - cells.append({ - "status": "missing", "label": "-", - "anchor": "", "resolved_version": "", - }) + cells.append( + { + "status": "missing", + "label": "-", + "anchor": "", + "resolved_version": "", + } + ) continue if row_tier is None: row_tier = entry.get("tier", "coordinated") badge = status.cell_badge(entry) - anchor = (_anchor_id(name, variant, python) - if badge["status"] not in ("pass", "missing") else "") - cells.append({ - **badge, - "anchor": anchor, - "resolved_version": entry.get("resolved_version", ""), - }) + anchor = ( + _anchor_id(name, variant, python) + if badge["status"] not in ("pass", "missing") + else "" + ) + cells.append( + { + **badge, + "anchor": anchor, + "resolved_version": entry.get("resolved_version", ""), + } + ) rows.append({"name": name, "tier": row_tier or "coordinated", "cells": cells}) return rows def _group_rows_by_tier(rows): """Group rows by tier, preserving order. Returns [(tier, [rows]), ...].""" - from itertools import groupby - return [(tier, list(group)) - for tier, group in groupby(rows, key=lambda r: r["tier"])] + return [ + (tier, list(group)) for tier, group in groupby(rows, key=lambda r: r["tier"]) + ] def _failures(by_combo, columns): @@ -136,30 +139,34 @@ def _failures(by_combo, columns): badge = status.cell_badge(entry) if badge["status"] in ("pass", "missing"): continue - out.append({ - "name": entry["name"], - "variant": variant, - "python": python, - "status": badge["status"], - "label": badge["label"], - "install_error": entry.get("install_error", ""), - "test_output": entry.get("test_output", ""), - "anchor": _anchor_id(entry["name"], variant, python), - }) + out.append( + { + "name": entry["name"], + "variant": variant, + "python": python, + "status": badge["status"], + "label": badge["label"], + "install_error": entry.get("install_error", ""), + "test_output": entry.get("test_output", ""), + "anchor": _anchor_id(entry["name"], variant, python), + } + ) return out -def build(results_dir, output_dir, templates_dir): +def build(results_dir, output_dir, templates_dir=None): results_dir = Path(results_dir) output_dir = Path(output_dir) if output_dir.exists(): shutil.rmtree(output_dir) output_dir.mkdir(parents=True) - env = Environment( - loader=FileSystemLoader(str(templates_dir)), - autoescape=select_autoescape(["html"]), + loader = ( + FileSystemLoader(str(templates_dir)) + if templates_dir + else PackageLoader("astropy_integration", "templates") ) + env = Environment(loader=loader, autoescape=select_autoescape(["html"])) by_combo = _load_results(results_dir) pythons, columns = _column_groups(by_combo) @@ -167,19 +174,25 @@ def build(results_dir, output_dir, templates_dir): rows = _make_rows(by_combo, names, columns) tier_groups = _group_rows_by_tier(rows) failures = _failures(by_combo, columns) - variants_meta = [_variant_meta(v, p, by_combo.get((v, p))) - for p in pythons for v in VARIANTS] + variants_meta = [ + _variant_meta(v, p, by_combo.get((v, p))) + for p in pythons + for v in status.VARIANTS + ] # If any variant ran with a per-package test limit, surface it in # a banner; PR previews use this to keep wall time bounded. - limits = {d.get("pytest_limit_n") for d in by_combo.values() - if d and d.get("pytest_limit_n")} + limits = { + d.get("pytest_limit_n") + for d in by_combo.values() + if d and d.get("pytest_limit_n") + } pytest_limit = min(limits) if limits else None (output_dir / "index.html").write_text( env.get_template("index.html").render( pythons=pythons, - variants=list(VARIANTS), + variants=list(status.VARIANTS), variants_meta=variants_meta, tier_groups=tier_groups, failures=failures, @@ -192,7 +205,11 @@ def build(results_dir, output_dir, templates_dir): def add_arguments(ap): ap.add_argument("--results-dir", default="results") ap.add_argument("--output", default="site") - ap.add_argument("--templates-dir", default=DEFAULT_TEMPLATES_DIR) + ap.add_argument( + "--templates-dir", + default=None, + help="Override templates directory (default: package's bundled templates).", + ) def run(args): diff --git a/astropy_integration/run.py b/astropy_integration/run.py index 96804b2..164cb2a 100644 --- a/astropy_integration/run.py +++ b/astropy_integration/run.py @@ -4,8 +4,8 @@ from packages.yaml in order, recording per-package install outcome (installed / skipped / install-fail / no-spec). Then runs `pytest --pyargs ` for each successfully-installed package. -Writes results/.json with the full venv freeze and per-package -data. +Writes results/__.json with the full venv freeze and +per-package data. Usage: astropy-integration run # full matrix (all variants x all Python versions from config) @@ -16,7 +16,6 @@ astropy-integration run --variant stable --tiers coordinated """ -import argparse import json import os import shutil @@ -29,20 +28,16 @@ from datetime import datetime, timezone from pathlib import Path - import yaml -from packaging.version import Version, InvalidVersion +from packaging.version import InvalidVersion, Version from . import status -# Make every print flush immediately so CI streams progress live -# instead of buffering until the script exits. -sys.stdout.reconfigure(line_buffering=True) - - PYPI_JSON_URL = "https://pypi.org/pypi/{name}/json" ASTROPY_NIGHTLY_INDEX = "https://pypi.anaconda.org/astropy/simple" -LIBERFA_NIGHTLY_INDEX = "https://pypi.anaconda.org/liberfa/simple" # for pyerfa dev wheels +LIBERFA_NIGHTLY_INDEX = ( + "https://pypi.anaconda.org/liberfa/simple" # for pyerfa dev wheels +) def _http_json(url, timeout=20): @@ -50,11 +45,6 @@ def _http_json(url, timeout=20): return json.load(r) -def _http_text(url, timeout=20): - with urllib.request.urlopen(url, timeout=timeout) as r: - return r.read().decode("utf-8", "replace") - - def _version_key(v): try: return Version(v) @@ -124,13 +114,15 @@ def pkg_spec(pkg): return f"{pkg['pypi_name']}{_extras_suffix(pkg)} @ git+{repo}", None else: - include_pre = (variant == "pre") + include_pre = variant == "pre" astropy_ver = latest_pypi("astropy", include_prereleases=include_pre) astropy = { "install": f"astropy=={astropy_ver}", "version": astropy_ver, "extra_index_urls": [], - "prerelease_strategy": "allow" if include_pre else "if-necessary-or-explicit", + "prerelease_strategy": "allow" + if include_pre + else "if-necessary-or-explicit", "index_strategy": None, } @@ -162,16 +154,22 @@ def _resolver_conflict(stderr): def ensure_python(version): - proc = subprocess.run(["uv", "python", "find", version], - capture_output=True, text=True, timeout=60) + proc = subprocess.run( + ["uv", "python", "find", version], capture_output=True, text=True, timeout=60 + ) if proc.returncode == 0: return proc.stdout.strip() - inst = subprocess.run(["uv", "python", "install", version], - capture_output=True, text=True, timeout=600) + inst = subprocess.run( + ["uv", "python", "install", version], + capture_output=True, + text=True, + timeout=600, + ) if inst.returncode != 0: sys.exit(f"uv python install {version}: {inst.stderr.strip()}") - proc = subprocess.run(["uv", "python", "find", version], - capture_output=True, text=True, timeout=60) + proc = subprocess.run( + ["uv", "python", "find", version], capture_output=True, text=True, timeout=60 + ) if proc.returncode != 0: sys.exit(f"uv python find {version}: {proc.stderr.strip()}") return proc.stdout.strip() @@ -179,25 +177,40 @@ def ensure_python(version): def _venv_python_version(python): proc = subprocess.run( - [python, "-c", "import sys; print('.'.join(str(x) for x in sys.version_info[:3]))"], - capture_output=True, text=True, timeout=30, + [ + python, + "-c", + "import sys; print('.'.join(str(x) for x in sys.version_info[:3]))", + ], + capture_output=True, + text=True, + timeout=30, ) return proc.stdout.strip() if proc.returncode == 0 else "" def _pkg_version(python, name): proc = subprocess.run( - [python, "-c", - "import importlib.metadata as md, sys; print(md.version(sys.argv[1]))", - name], - capture_output=True, text=True, timeout=30, + [ + python, + "-c", + "import importlib.metadata as md, sys; print(md.version(sys.argv[1]))", + name, + ], + capture_output=True, + text=True, + timeout=30, ) return proc.stdout.strip() if proc.returncode == 0 else "" def _freeze(python): - proc = subprocess.run(["uv", "pip", "freeze", "--python", python], - capture_output=True, text=True, timeout=60) + proc = subprocess.run( + ["uv", "pip", "freeze", "--python", python], + capture_output=True, + text=True, + timeout=60, + ) out = {} if proc.returncode == 0: for line in proc.stdout.splitlines(): @@ -229,16 +242,21 @@ def _load_python_versions(path): def _install_order(packages): """Coordinated before affiliated before other, alphabetical within each tier.""" - return sorted(packages, key=lambda p: ( - TIER_RANK.get(p.get("tier", "coordinated"), 9), - p["pypi_name"].lower(), - )) + return sorted( + packages, + key=lambda p: ( + TIER_RANK.get(p.get("tier", "coordinated"), 9), + p["pypi_name"].lower(), + ), + ) def _run_install(install_cmd, timeout): """Wrap subprocess.run with TimeoutExpired catch. Returns (rc, stderr_or_msg).""" try: - proc = subprocess.run(install_cmd, capture_output=True, text=True, timeout=timeout) + proc = subprocess.run( + install_cmd, capture_output=True, text=True, timeout=timeout + ) except subprocess.TimeoutExpired: return None, "timeout during install" return proc.returncode, (proc.stderr or proc.stdout) @@ -264,9 +282,11 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo "python_requested": python_version, "python_version": "", "timeout_test_seconds": timeouts["test"], - "pytest_limit_n": (int(os.environ["PYTEST_LIMIT_N"]) - if (os.environ.get("PYTEST_LIMIT_N") or "").isdigit() - else None), + "pytest_limit_n": ( + int(os.environ["PYTEST_LIMIT_N"]) + if (os.environ.get("PYTEST_LIMIT_N") or "").isdigit() + else None + ), "fatal_error": "", "installed_deps": {}, "packages": [], @@ -274,14 +294,20 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo out_path = results_dir / f"{variant}__{python_version}.json" Path(repo_root, ".tmp").mkdir(exist_ok=True) - tmpdir = tempfile.mkdtemp(prefix=f"int-{variant}-{python_version}-", - dir=str(Path(repo_root, ".tmp").resolve())) + tmpdir = tempfile.mkdtemp( + prefix=f"int-{variant}-{python_version}-", + dir=str(Path(repo_root, ".tmp").resolve()), + ) try: py_path = ensure_python(python_version) venv = os.path.join(tmpdir, "venv") - venv_proc = subprocess.run(["uv", "venv", venv, "-p", py_path, "-q"], - capture_output=True, text=True, timeout=120) + venv_proc = subprocess.run( + ["uv", "venv", venv, "-p", py_path, "-q"], + capture_output=True, + text=True, + timeout=120, + ) if venv_proc.returncode != 0: result["fatal_error"] = f"uv venv: {venv_proc.stderr or venv_proc.stdout}" return result, out_path @@ -301,7 +327,8 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo # `--remote-data` not passed, those tests are skipped automatically # instead of running and timing out on network calls. rc, err = _run_install( - common + [astropy["install"], "pytest", "pytest-timeout", "pytest-remotedata"], + common + + [astropy["install"], "pytest", "pytest-timeout", "pytest-remotedata"], timeouts["install"], ) if rc != 0: @@ -309,7 +336,9 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo print(err) result["fatal_error"] = err return result, out_path - result["astropy"]["version"] = _pkg_version(python, "astropy") or result["astropy"]["version"] + result["astropy"]["version"] = ( + _pkg_version(python, "astropy") or result["astropy"]["version"] + ) installed_pkgs = [] for pkg, install_spec, target_version in pkg_specs: @@ -320,16 +349,18 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo "install_spec": install_spec, "target_version": target_version, "resolved_version": "", - "install_status": "", # installed | skipped | install-fail | no-spec + "install_status": "", # installed | skipped | install-fail | no-spec "install_error": "", - "test_status": "", # pass | fail | no-tests | timeout | not-run + "test_status": "", # pass | fail | no-tests | timeout | not-run "tests_passed": None, "test_output": "", "duration": 0, } if install_spec is None: entry["install_status"] = status.NO_SPEC - entry["install_error"] = "no install spec (missing repo_url, or no PyPI release)" + entry["install_error"] = ( + "no install spec (missing repo_url, or no PyPI release)" + ) result["packages"].append(entry) continue @@ -364,8 +395,14 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo env = {**os.environ, "MPLBACKEND": "Agg", "DISPLAY": ""} start = time.time() try: - proc = subprocess.run(cmd, capture_output=True, text=True, - timeout=timeouts["test"], cwd=repo_root, env=env) + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeouts["test"], + cwd=repo_root, + env=env, + ) except subprocess.TimeoutExpired: entry["test_status"] = status.TIMEOUT entry["test_output"] = "timeout" @@ -378,9 +415,8 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo out += "\n--- stderr ---\n" + proc.stderr entry["test_output"] = out - no_module = ( - proc.returncode == 4 - and "module or package not found" in (proc.stdout + proc.stderr) + no_module = proc.returncode == 4 and "module or package not found" in ( + proc.stdout + proc.stderr ) if proc.returncode == 5 or no_module: entry["test_status"] = status.NO_TESTS @@ -409,20 +445,31 @@ def _counts(result): def add_arguments(ap): ap.add_argument("--config", default="packages.yaml") ap.add_argument("--results-dir", default="results") - ap.add_argument("--variant", choices=status.VARIANTS, - help="Variant to run; if omitted, runs all variants in sequence.") - ap.add_argument("--python", - help="Python version to run against (e.g. '3.12', '3.14t'); " - "if omitted, runs every version listed in the config.") + ap.add_argument( + "--variant", + choices=status.VARIANTS, + help="Variant to run; if omitted, runs all variants in sequence.", + ) + ap.add_argument( + "--python", + help="Python version to run against (e.g. '3.12', '3.14t'); " + "if omitted, runs every version listed in the config.", + ) ap.add_argument("--packages", help="Comma-separated subset of package names to run") - ap.add_argument("--tiers", - help="Comma-separated subset of tiers to run (e.g. 'coordinated,other'); " - "default: all tiers") + ap.add_argument( + "--tiers", + help="Comma-separated subset of tiers to run (e.g. 'coordinated,other'); " + "default: all tiers", + ) ap.add_argument("--timeout-install", type=int, default=900) ap.add_argument("--timeout-test", type=int, default=1800) def run(args): + # Make every print flush immediately so CI streams progress live + # instead of buffering until the script exits. + sys.stdout.reconfigure(line_buffering=True) + packages = _load_packages(args.config) if args.tiers: wanted_tiers = {t.strip() for t in args.tiers.split(",") if t.strip()} @@ -438,13 +485,16 @@ def run(args): timeouts = {"install": args.timeout_install, "test": args.timeout_test} variants_to_run = [args.variant] if args.variant else list(status.VARIANTS) - pythons_to_run = [args.python] if args.python else _load_python_versions(args.config) + pythons_to_run = ( + [args.python] if args.python else _load_python_versions(args.config) + ) fatal_combos = [] for python_version in pythons_to_run: for variant in variants_to_run: - result, out_path = run_variant(variant, python_version, packages, - repo_root, results_dir, timeouts) + result, out_path = run_variant( + variant, python_version, packages, repo_root, results_dir, timeouts + ) print(f"\nDone {variant}/{python_version}: {_counts(result)}") print(f"Wrote {out_path}") if result.get("fatal_error"): diff --git a/astropy_integration/status.py b/astropy_integration/status.py index c930bfc..47d8ac5 100644 --- a/astropy_integration/status.py +++ b/astropy_integration/status.py @@ -1,8 +1,9 @@ """Shared status vocabulary for integration-test results. -Both run_integration.py (which writes statuses into results JSON) and -build_dashboard.py (which renders them into the dashboard) use these -constants and helpers, so the strings are defined in exactly one place. +Both `astropy_integration.run` (which writes statuses into results JSON) +and `astropy_integration.dashboard` (which renders them into the +dashboard) use these constants and helpers, so the strings are defined +in exactly one place. """ # The three astropy variants. Listed once here so both scripts agree. @@ -10,9 +11,9 @@ # install_status values, used in results/.json INSTALLED = "installed" -SKIPPED = "skipped" # resolver couldn't satisfy alongside the existing venv +SKIPPED = "skipped" # resolver couldn't satisfy alongside the existing venv INSTALL_FAIL = "install-fail" # install itself crashed (build error, network, etc.) -NO_SPEC = "no-spec" # couldn't build an install spec (no repo_url / no PyPI release) +NO_SPEC = "no-spec" # couldn't build an install spec (no repo_url / no PyPI release) # test_status values PASS = "pass" diff --git a/packages.yaml b/packages.yaml index b57cb84..40c3a1a 100644 --- a/packages.yaml +++ b/packages.yaml @@ -1,4 +1,4 @@ -# Packages tested by run_integration.py. +# Packages tested by `astropy-integration run`. # # One block per package = one row in the dashboard. For each astropy # variant (stable / pre / dev) and each Python version, the runner @@ -333,4 +333,3 @@ packages: tier: affiliated module: synphot repo_url: https://github.com/spacetelescope/synphot_refactor.git - From 22594f39fa1fdd203609d1999aa446e38c49a672 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Wed, 13 May 2026 12:14:50 +0100 Subject: [PATCH 14/17] More cleanup --- astropy_integration/dashboard.py | 19 ++++++------------- astropy_integration/run.py | 3 --- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/astropy_integration/dashboard.py b/astropy_integration/dashboard.py index 6b745d6..1d463f5 100644 --- a/astropy_integration/dashboard.py +++ b/astropy_integration/dashboard.py @@ -15,7 +15,7 @@ from itertools import groupby from pathlib import Path -from jinja2 import Environment, FileSystemLoader, PackageLoader, select_autoescape +from jinja2 import Environment, PackageLoader, select_autoescape from . import status @@ -154,19 +154,17 @@ def _failures(by_combo, columns): return out -def build(results_dir, output_dir, templates_dir=None): +def build(results_dir, output_dir): results_dir = Path(results_dir) output_dir = Path(output_dir) if output_dir.exists(): shutil.rmtree(output_dir) output_dir.mkdir(parents=True) - loader = ( - FileSystemLoader(str(templates_dir)) - if templates_dir - else PackageLoader("astropy_integration", "templates") + env = Environment( + loader=PackageLoader("astropy_integration", "templates"), + autoescape=select_autoescape(["html"]), ) - env = Environment(loader=loader, autoescape=select_autoescape(["html"])) by_combo = _load_results(results_dir) pythons, columns = _column_groups(by_combo) @@ -205,12 +203,7 @@ def build(results_dir, output_dir, templates_dir=None): def add_arguments(ap): ap.add_argument("--results-dir", default="results") ap.add_argument("--output", default="site") - ap.add_argument( - "--templates-dir", - default=None, - help="Override templates directory (default: package's bundled templates).", - ) def run(args): - build(args.results_dir, args.output, args.templates_dir) + build(args.results_dir, args.output) diff --git a/astropy_integration/run.py b/astropy_integration/run.py index 164cb2a..ea4efbd 100644 --- a/astropy_integration/run.py +++ b/astropy_integration/run.py @@ -352,7 +352,6 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo "install_status": "", # installed | skipped | install-fail | no-spec "install_error": "", "test_status": "", # pass | fail | no-tests | timeout | not-run - "tests_passed": None, "test_output": "", "duration": 0, } @@ -422,10 +421,8 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo entry["test_status"] = status.NO_TESTS elif proc.returncode == 0: entry["test_status"] = status.PASS - entry["tests_passed"] = True else: entry["test_status"] = status.FAIL - entry["tests_passed"] = False print(f" {entry['test_status']} in {entry['duration']}s") finally: From 3ed80ec9e53bc9f3c9f2293ebc76285f22c5ad98 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Thu, 14 May 2026 09:40:09 +0100 Subject: [PATCH 15/17] Apply suggestions from code review Co-authored-by: P. L. Lim <2090236+pllim@users.noreply.github.com> --- README.md | 2 +- astropy_integration/dashboard.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a7f313..458a6b6 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ astropy variant: Within each job, a single shared venv is built and packages are installed one at a time in a deterministic order (coordinated first, alphabetical within each tier). If a package can't be installed -alongside the existing venv (e.g. it pins `astropy<7` but we already +alongside the existing venv (e.g., it pins `astropy<7` but we already installed astropy 8), it's skipped and recorded; the rest of the venv is untouched. After installs, `pytest --pyargs ` runs for each package that installed successfully. diff --git a/astropy_integration/dashboard.py b/astropy_integration/dashboard.py index 1d463f5..790d299 100644 --- a/astropy_integration/dashboard.py +++ b/astropy_integration/dashboard.py @@ -2,6 +2,7 @@ Reads every results/__.json that `astropy-integration run` wrote and emits a single self-contained `site/index.html` with: + - one row per package - one column group per Python version, subdivided into the three astropy variants (stable / pre / dev) From 222fa000eee6502f2e5218e0f137296bd1df3740 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Thu, 14 May 2026 08:58:48 +0000 Subject: [PATCH 16/17] Add pyopensci package tier and tidy workflow and docs Add seven pyOpenSci-reviewed affiliated packages (astrodata, astromartini, autogalaxy, GALFITools, petrofit, stingray, zodipy) under a new pyopensci tier, ranked after affiliated in install and dashboard order. Set an explicit pull-requests: none workflow permission, document why setup-python uses a fixed version, and name the CI jobs explicitly in the README. --- .github/workflows/integration.yml | 6 +++- README.md | 12 ++++---- astropy_integration/run.py | 4 +-- packages.yaml | 49 +++++++++++++++++++++++++++++-- 4 files changed, 60 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 65ca739..292b741 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -7,7 +7,8 @@ on: pull_request: permissions: - contents: write # to push gh-pages on main / schedule / dispatch + contents: write # to push gh-pages on main / schedule / dispatch + pull-requests: none # explicit: this workflow never touches PRs concurrency: # One in-flight run per PR, one per main/dispatch invocation. @@ -29,6 +30,9 @@ jobs: steps: - uses: actions/checkout@v4 + # This Python only runs the `astropy-integration` orchestrator CLI; + # the test venv for `matrix.python` is built separately by uv (see + # the `run` step below), so a fixed version is fine here. - uses: actions/setup-python@v5 with: python-version: '3.12' diff --git a/README.md b/README.md index 458a6b6..b1a9335 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ after each scheduled run. How it works ------------ -Three CI jobs run on a schedule (and on `workflow_dispatch`), one per -astropy variant: +The `variant` job runs on a schedule (and on `workflow_dispatch`) as a +matrix over astropy variant and Python version. The three variants are: | Variant | Astropy | Each package | |----------|-----------------------------------------------------|----------------------------------------| @@ -24,7 +24,7 @@ astropy variant: | `pre` | Latest including pre-releases (`--prerelease=allow`)| Latest including pre-releases | | `dev` | Latest dev wheel from the astropy/simple channel | `git+` (HEAD of main branch) | -Within each job, a single shared venv is built and packages are +Within each matrix job, a single shared venv is built and packages are installed one at a time in a deterministic order (coordinated first, alphabetical within each tier). If a package can't be installed alongside the existing venv (e.g., it pins `astropy<7` but we already @@ -32,8 +32,8 @@ installed astropy 8), it's skipped and recorded; the rest of the venv is untouched. After installs, `pytest --pyargs ` runs for each package that installed successfully. -A fourth job downloads the three result JSONs and publishes the -dashboard to `gh-pages`. +The `dashboard` job then downloads the per-matrix-job result JSONs and +publishes the dashboard to `gh-pages`. What's in the repo ------------------ @@ -103,7 +103,7 @@ Edit `packages.yaml`. Each entry takes: - `pypi_name` (the package's name on PyPI; also used as the row label) - `tier` (label used for ordering and the `--tiers` filter; conventional - values are `coordinated`, `affiliated`, `other`) + values are `coordinated`, `affiliated`, `pyopensci`, `other`) - `module` (the top-level Python module name, for `pytest --pyargs`) - `repo_url` (for the `dev` variant install) - `install_extras` (list, e.g. `[test, all]`) diff --git a/astropy_integration/run.py b/astropy_integration/run.py index ea4efbd..b0dda3c 100644 --- a/astropy_integration/run.py +++ b/astropy_integration/run.py @@ -237,11 +237,11 @@ def _load_python_versions(path): # Ordering of tiers when installing/displaying. Unknown tiers sort last. -TIER_RANK = {"coordinated": 0, "affiliated": 1, "other": 2} +TIER_RANK = {"coordinated": 0, "affiliated": 1, "pyopensci": 2, "other": 3} def _install_order(packages): - """Coordinated before affiliated before other, alphabetical within each tier.""" + """Coordinated, then affiliated, then pyopensci, then other; alphabetical within each tier.""" return sorted( packages, key=lambda p: ( diff --git a/packages.yaml b/packages.yaml index 40c3a1a..cc49902 100644 --- a/packages.yaml +++ b/packages.yaml @@ -9,8 +9,9 @@ # # `tier` is purely a category label used for ordering and the optional # `--tiers` CLI filter. Conventions: `coordinated` for the official -# coordinated set, `affiliated` for affiliated packages, `other` for -# anything else worth testing. +# coordinated set, `affiliated` for affiliated packages from the frozen +# pre-APE 22 registry, `pyopensci` for affiliated packages reviewed +# through the pyOpenSci process, `other` for anything else worth testing. # # `python_versions` is the list of Python versions to test against; # use uv's notation, so e.g. "3.14t" for the free-threaded build. @@ -333,3 +334,47 @@ packages: tier: affiliated module: synphot repo_url: https://github.com/spacetelescope/synphot_refactor.git + + # --- Affiliated packages reviewed through the pyOpenSci process and + # cross-listed at https://www.pyopensci.org/python-packages.html + # (post-APE 22; not in the frozen registry.json above). + - pypi_name: astrodata + tier: pyopensci + module: astrodata + repo_url: https://github.com/GeminiDRSoftware/astrodata.git + + - pypi_name: astromartini + tier: pyopensci + # installed and imported as `martini`; `astromartini` is just the PyPI name. + module: martini + repo_url: https://github.com/kyleaoman/martini.git + install_extras: [testing] + + - pypi_name: autogalaxy + tier: pyopensci + module: autogalaxy + repo_url: https://github.com/PyAutoLabs/PyAutoGalaxy.git + install_extras: [test] + + - pypi_name: GALFITools + tier: pyopensci + module: galfitools # installed package name is lowercase + repo_url: https://github.com/canorve/GALFITools.git + install_extras: [testing] + + - pypi_name: petrofit + tier: pyopensci + module: petrofit + repo_url: https://github.com/PetroFit/petrofit.git + install_extras: [dev] + + - pypi_name: stingray + tier: pyopensci + module: stingray + repo_url: https://github.com/StingraySoftware/stingray.git + install_extras: [test] + + - pypi_name: zodipy + tier: pyopensci + module: zodipy + repo_url: https://github.com/Cosmoglobe/zodipy.git From 56e38e959b0cf463d68aeabfab3e1332d857b158 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Thu, 14 May 2026 09:02:36 +0000 Subject: [PATCH 17/17] Pin workflow actions to commit SHAs and add Dependabot cooldown Pin every action in the integration and preview-link workflows to a full release commit SHA with a version comment. Add a 7-day cooldown to the github-actions Dependabot entry so a release has time to settle before a bump is proposed. --- .github/dependabot.yml | 3 +++ .github/workflows/integration.yml | 20 ++++++++++---------- .github/workflows/preview-link.yml | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6b0c235..c765c24 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,9 @@ updates: directory: ".github/workflows" # Location of package manifests schedule: interval: "weekly" + cooldown: + # Let a release settle before Dependabot proposes bumping to it. + default-days: 7 groups: actions: patterns: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 292b741..477eecc 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -28,17 +28,17 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 240 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # This Python only runs the `astropy-integration` orchestrator CLI; # the test venv for `matrix.python` is built separately by uv (see # the `run` step below), so a fixed version is fine here. - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' - name: Install uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3.2.4 with: # We manage the cache ourselves via actions/cache below; turning # off setup-uv's own caching avoids it saving a duplicate of the @@ -46,7 +46,7 @@ jobs: enable-cache: false - name: Cache ~/.cache/uv - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ~/.cache/uv # Primary key uses the run id, so every run writes a fresh @@ -80,7 +80,7 @@ jobs: - name: Upload results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: results-${{ matrix.variant }}-${{ matrix.python }} path: results/${{ matrix.variant }}__${{ matrix.python }}.json @@ -91,9 +91,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.12' @@ -101,7 +101,7 @@ jobs: run: pip install . - name: Download all variant results - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: results-* merge-multiple: true @@ -114,7 +114,7 @@ jobs: # in-browser preview (linked from a PR check by preview-link.yml). - name: Upload preview artifact if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dashboard-preview path: site/index.html @@ -124,7 +124,7 @@ jobs: # Main / schedule / dispatch path: publish to gh-pages. - name: Publish to gh-pages if: github.event_name != 'pull_request' - uses: peaceiris/actions-gh-pages@v4 + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./site diff --git a/.github/workflows/preview-link.yml b/.github/workflows/preview-link.yml index 01cb6c1..5a9b47f 100644 --- a/.github/workflows/preview-link.yml +++ b/.github/workflows/preview-link.yml @@ -21,7 +21,7 @@ jobs: statuses: write # to attach a status check to the source commit actions: read # to look up the source workflow's artifact id steps: - - uses: agriyakhetarpal/github-actions-artifacts-redirector-action@683d25ace2cb0aefe8e6719c39c2ac7f3d22dd8c + - uses: agriyakhetarpal/github-actions-artifacts-redirector-action@683d25ace2cb0aefe8e6719c39c2ac7f3d22dd8c # v1.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # With `archive: false`, artifact-name is the uploaded file's