Fix more issues and only run first 10 tests in PRs

Author: astrofrog

.github/workflows/integration.yml

@@ -61,10 +61,11 @@ jobs:
 
       - name: Run ${{ matrix.variant }} variant on Python ${{ matrix.python }}
         env:
-          # Short timeout for PR previews so the matrix finishes in
-          # minutes; full timeout for scheduled/dispatch runs.
-          TIMEOUT_FLAG: ${{ github.event_name == 'pull_request' && '--timeout-test 20' || '' }}
-        run: python run_integration.py --variant ${{ matrix.variant }} --python ${{ matrix.python }} $TIMEOUT_FLAG
+          # PR previews limit each package to the first 10 tests so the
+          # matrix finishes in minutes; conftest.py at the repo root reads
+          # this env var and truncates the collected items.
+          PYTEST_LIMIT_N: ${{ github.event_name == 'pull_request' && '10' || '' }}
+        run: python run_integration.py --variant ${{ matrix.variant }} --python ${{ matrix.python }}
 
       - name: Upload results
         if: always()

README.md

@@ -132,10 +132,12 @@ whose "Details" link opens the rendered page directly in the
 browser.
 
 This means the PR preview reflects *this PR's actual matrix run*,
-not last main's data. It's also slower than a render-only preview
-would be — expect the same wall-clock as a normal main run, up to
-a few hours per push. Concurrency cancels in-progress PR runs when
-a new push lands, so only the latest push consumes CI time.
+not last main's data. To keep PR feedback fast, each package is
+capped at the first 10 collected tests (via `PYTEST_LIMIT_N=10`,
+applied by the repo-level `conftest.py`); the preview is a smoke
+check of layout, install resolution, and the workflow itself, not
+a full regression signal. Concurrency cancels in-progress PR runs
+when a new push lands, so only the latest push consumes CI time.
 
 `preview-link.yml` lives at `.github/workflows/preview-link.yml`
 and must be on the default branch for its `workflow_run` trigger

build_dashboard.py

@@ -168,11 +168,11 @@ def build(results_dir, output_dir, templates_dir):
     variants_meta = [_variant_meta(v, p, by_combo.get((v, p)))
                      for p in pythons for v in VARIANTS]
 
-    # If any variant ran with an unusually short test timeout, surface
-    # it in a banner; that's typical for PR preview runs.
-    timeouts = {d.get("timeout_test_seconds") for d in by_combo.values()
-                if d and d.get("timeout_test_seconds")}
-    short_timeout = min((t for t in timeouts if t and t < 600), default=None)
+    # If any variant ran with a per-package test limit, surface it in
+    # a banner; PR previews use this to keep wall time bounded.
+    limits = {d.get("pytest_limit_n") for d in by_combo.values()
+              if d and d.get("pytest_limit_n")}
+    pytest_limit = min(limits) if limits else None
 
     (output_dir / "index.html").write_text(
         env.get_template("index.html").render(
@@ -181,7 +181,7 @@ def build(results_dir, output_dir, templates_dir):
             variants_meta=variants_meta,
             tier_groups=tier_groups,
             failures=failures,
-            short_timeout=short_timeout,
+            pytest_limit=pytest_limit,
         )
     )
     print(f"Wrote {output_dir}/index.html")

conftest.py

@@ -0,0 +1,24 @@
+"""Repo-level pytest plugin used during PR-preview runs.
+
+When the env var `PYTEST_LIMIT_N` is set to a positive integer, pytest
+collects normally and then truncates the test list to the first N
+items. This lets the PR matrix surface a fast smoke signal per
+package (typically 10 tests each) without per-package configuration.
+
+Discovered automatically by pytest because cwd is the repo root for
+every `pytest --pyargs <module>` invocation in the runner.
+"""
+
+import os
+
+
+def pytest_collection_modifyitems(config, items):
+    raw = os.environ.get("PYTEST_LIMIT_N")
+    if not raw:
+        return
+    try:
+        n = int(raw)
+    except ValueError:
+        return
+    if n > 0 and len(items) > n:
+        del items[n:]

packages.yaml

@@ -94,8 +94,9 @@ packages:
 
   - pypi_name: APLpy
     tier: affiliated
-    module: APLpy
+    module: aplpy   # installed package name is lowercase
     repo_url: https://github.com/aplpy/aplpy.git
+    install_extras: [test]
 
   - pypi_name: astroalign
     tier: affiliated
@@ -126,10 +127,13 @@ packages:
     tier: affiliated
     module: BayesicFitting
     repo_url: https://github.com/dokester/BayesicFitting.git
+    # BayesicFitting ships unittest-style tests as Test*.py (capital T);
+    # override pytest's default `python_files` glob so it picks them up.
+    pytest_args: ["-o", "python_files=Test*.py"]
 
   - pypi_name: cluster-lensing
     tier: affiliated
-    module: cluster_lensing
+    module: clusterlensing   # installed package has no underscore
     repo_url: https://github.com/jesford/cluster-lensing.git
 
   - pypi_name: corral-pipeline
@@ -166,15 +170,21 @@ packages:
     tier: affiliated
     module: gammapy
     repo_url: https://github.com/gammapy/gammapy.git
+    install_extras: [test]
 
   - pypi_name: ginga
     tier: affiliated
     module: ginga
     repo_url: https://github.com/ejeschke/ginga.git
+    # `ginga/tests/test_zarr.py` uses removed zarr 3.x API; ignore it
+    # so the rest of the suite (256 passing tests) can run.
+    pytest_args: ["--ignore-glob=*test_zarr.py"]
 
   - pypi_name: glueviz
     tier: affiliated
-    module: glueviz
+    # glueviz is a meta-package; the actual code (and tests) lives in
+    # glue-core which it pulls in. Its installed module is `glue`.
+    module: glue
     repo_url: https://github.com/glue-viz/glue.git
 
   - pypi_name: gwcs
@@ -191,6 +201,7 @@ packages:
     tier: affiliated
     module: hendrics
     repo_url: https://github.com/StingraySoftware/HENDRICS.git
+    install_extras: [test]
 
   - pypi_name: hips
     tier: affiliated
@@ -214,23 +225,36 @@ packages:
 
   - pypi_name: ligo.skymap
     tier: affiliated
-    module: ligo_skymap
+    # PEP 420 namespace package: installed as ligo/skymap/, not ligo_skymap.
+    module: ligo.skymap
     repo_url: https://git.ligo.org/lscsoft/ligo.skymap.git
+    install_extras: [test]
 
   - pypi_name: linetools
     tier: affiliated
     module: linetools
     repo_url: https://github.com/linetools/linetools.git
+    install_extras: [test]
+    # linetools imports pkg_resources (deprecated stdlib alias); pull
+    # in setuptools so it's available at runtime.
+    extra_deps: [setuptools]
 
   - pypi_name: marxs
     tier: affiliated
     module: marxs
     repo_url: https://github.com/Chandra-MARX/marxs.git
+    install_extras: [test]
+    # Two test files require an external MARX C library we don't have;
+    # ignoring them lets the rest of the suite collect.
+    pytest_args:
+      - "--ignore-glob=*test_hrma.py"
+      - "--ignore-glob=*test_all_optics.py"
 
   - pypi_name: mocpy
     tier: affiliated
     module: mocpy
     repo_url: https://github.com/cds-astro/mocpy.git
+    install_extras: [dev]
 
   - pypi_name: naima
     tier: affiliated
@@ -256,6 +280,7 @@ packages:
     tier: affiliated
     module: pydl
     repo_url: https://github.com/weaverba137/pydl.git
+    install_extras: [test]
 
   - pypi_name: pyregion
     tier: affiliated
@@ -271,6 +296,7 @@ packages:
     tier: affiliated
     module: pyvo
     repo_url: https://github.com/astropy/pyvo.git
+    install_extras: [test]
 
   - pypi_name: regularizepsf
     tier: affiliated
@@ -291,6 +317,7 @@ packages:
     tier: affiliated
     module: spectral_cube
     repo_url: https://github.com/radio-astro-tools/spectral-cube.git
+    install_extras: [test]
 
   - pypi_name: spherical-geometry
     tier: affiliated

run_integration.py

@@ -269,6 +269,9 @@ def run_variant(variant, python_version, packages, repo_root, results_dir, timeo
         "python_requested": python_version,
         "python_version": "",
         "timeout_test_seconds": timeouts["test"],
+        "pytest_limit_n": (int(os.environ["PYTEST_LIMIT_N"])
+                           if (os.environ.get("PYTEST_LIMIT_N") or "").isdigit()
+                           else None),
         "fatal_error": "",
         "installed_deps": {},
         "packages": [],

templates/index.html

@@ -3,11 +3,11 @@
 {% block body %}
 <h1>Astropy ecosystem integration matrix</h1>
 
-{% if short_timeout %}
+{% if pytest_limit %}
 <div class="banner">
-  <strong>Preview run:</strong> per-package test timeout set to <code>{{ short_timeout }}s</code>
-  for fast PR feedback. Many tests will time out; treat the matrix as a layout/workflow
-  check, not a real regression signal.
+  <strong>Preview run:</strong> each package limited to the first
+  <code>{{ pytest_limit }}</code> tests for fast PR feedback. Treat the matrix as
+  a smoke check, not a full regression signal.
 </div>
 {% endif %}