ci: decouple upstream dev-version testing from the pixi workspace (#997)

* ci: decouple upstream dev-version testing from the pixi workspace

Move the 11 `git+https://…` dev-version sources out of pyproject.toml's
`[dependency-groups].upstream` and into a standalone
`ci/upstream-overrides.txt`. The Upstream CI job pip-overlays them on top
of the regular `upstream` pixi env via `--no-deps --upgrade`.

Why: pixi solves a single workspace-wide lockfile covering every defined
environment. With git+https sources present, every solve (docs,
minimum-versions, etc.) had to build a wheel for each git source for
metadata extraction, and on memory-constrained CI runners these parallel
builds intermittently SIGSEGV'd or hit ETXTBSY races — blocking docs and
other unrelated environments. Keeping git sources out of pyproject keeps
every other env's solve trivial. See #995.

* ci: wrap the upstream overlay command in a pixi task

Author: TomNicholas

.github/workflows/upstream.yml

@@ -38,9 +38,17 @@ jobs:
           pixi-version: v0.59.0
           environments: upstream
 
-      - name: List installed libraries
+      - name: Install pixi upstream environment
         run: |
           pixi install --environment upstream
+
+      - name: Overlay dev versions of upstream libraries
+        # See ci/upstream-overrides.txt for why these aren't in pyproject.toml.
+        run: |
+          pixi run -e upstream install-upstream-overrides
+
+      - name: List installed libraries
+        run: |
           pixi list --environment upstream
 
       - name: Cache tutorial data

ci/upstream-overrides.txt

@@ -0,0 +1,21 @@
+# Dev versions of upstream libraries used by the `Upstream` CI job to test
+# VirtualiZarr against bleeding-edge releases of its dependencies.
+#
+# Installed via `pip install --upgrade --no-deps -r ci/upstream-overrides.txt`
+# on top of an already-resolved pixi `upstream` env, so pip only swaps the
+# named packages and does not re-resolve the dependency graph.
+#
+# These deliberately do NOT live in pyproject.toml: keeping git+https sources
+# out of the pixi workspace prevents every other env's lockfile solve (docs,
+# minimum-versions, etc.) from having to build them. See issue #995.
+xarray @ git+https://github.com/pydata/xarray
+universal_pathlib @ git+https://github.com/fsspec/universal_pathlib
+numcodecs @ git+https://github.com/zarr-developers/numcodecs
+ujson @ git+https://github.com/ultrajson/ultrajson
+zarr @ git+https://github.com/zarr-developers/zarr-python
+obspec_utils @ git+https://github.com/virtual-zarr/obspec-utils
+astropy @ git+https://github.com/astropy/astropy
+s3fs @ git+https://github.com/fsspec/s3fs
+kerchunk @ git+https://github.com/fsspec/kerchunk
+icechunk @ git+https://github.com/earth-mover/icechunk#subdirectory=icechunk-python
+virtual_tiff @ git+https://github.com/virtual-zarr/virtual-tiff

docs/contributing.md

@@ -23,11 +23,19 @@ You can also run tests in other environments:
 
 ```bash
 pixi run --environment min-deps run-tests # Test with the minimal set of dependencies installed
-pixi run --environment upstream run-tests # Test with unreleased versions of upstream libraries
 # List which versions are installed in the `min-deps` environment
 pixi list --environment min-deps
 ```
 
+To test against unreleased dev versions of upstream libraries (xarray, zarr, numcodecs, etc.),
+install the regular `upstream` env and then pip-overlay the dev versions on top:
+
+```bash
+pixi install --environment upstream
+pixi run -e upstream install-upstream-overrides
+pixi run --environment upstream run-tests
+```
+
 Further, the `pytest-cov` plugin is a test dependency, so you can generate a test
 coverage report locally, if you wish (CI will automatically do so).  Here are some
 examples:

pyproject.toml

@@ -96,20 +96,13 @@ all_writers = [
 
 # Dependency sets under dependencies-groups are NOT available via PyPI
 [dependency-groups]
-upstream = [
-    'xarray @ git+https://github.com/pydata/xarray',
-    'universal_pathlib @ git+https://github.com/fsspec/universal_pathlib',
-    'numcodecs @ git+https://github.com/zarr-developers/numcodecs',
-    'ujson @ git+https://github.com/ultrajson/ultrajson',
-    'zarr @ git+https://github.com/zarr-developers/zarr-python',
-    'obspec_utils @ git+https://github.com/virtual-zarr/obspec-utils',
-    # optional dependencies
-    'astropy @ git+https://github.com/astropy/astropy',
-    's3fs @ git+https://github.com/fsspec/s3fs',
-    'kerchunk @ git+https://github.com/fsspec/kerchunk',
-    'icechunk @ git+https://github.com/earth-mover/icechunk#subdirectory=icechunk-python',
-    'virtual_tiff @ git+https://github.com/virtual-zarr/virtual-tiff',
-]
+# Dev versions of upstream libraries are NOT defined here as a dependency-group.
+# Including git+https sources in a pixi-managed pyproject.toml forces every
+# `pixi install` (including the docs and minimum-versions envs) to perform a
+# fresh wheel build of each git source as part of the workspace-wide lockfile
+# solve, which is unreliable on memory-constrained CI runners. The upstream
+# canary job instead pip-overlays dev versions from ci/upstream-overrides.txt
+# on top of the regular `upstream` pixi env. See issue #995.
 docs = [
     "mkdocs-material[imaging]>=9.6.14",
     "mkdocs>=1.6.1",
@@ -182,6 +175,14 @@ h5netcdf = ">=1.5.0,<2"
 [tool.pixi.feature.icechunk-dev.dependencies]
 rust = "*"
 
+# `pip` is needed by the Upstream CI job to overlay dev versions from
+# ci/upstream-overrides.txt on top of the pixi-resolved env. See issue #995.
+[tool.pixi.feature.upstream-overlay.dependencies]
+pip = "*"
+
+[tool.pixi.feature.upstream-overlay.tasks]
+install-upstream-overrides = { cmd = "pip install --upgrade --no-deps -r ci/upstream-overrides.txt" }
+
 [tool.pixi.feature.minimum-versions.dependencies]
 xarray = "==2025.6.0"
 numpy = "==2.1.0"
@@ -213,7 +214,7 @@ test-py313 = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "ke
 test-py314 = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "zarr", "py314"] # test against python 3.14
 minio = ["dev", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "hdf5-lib", "py314", "zarr", "minio"]
 minimum-versions = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "tiff", "zarr","minimum-versions", "py312"]
-upstream = ["dev", "test", "hdf", "hdf5-lib", "netcdf3", "upstream", "icechunk-dev", "zarr", "py313"]
+upstream = ["dev", "test", "hdf", "hdf5-lib", "netcdf3", "icechunk-dev", "upstream-overlay", "zarr", "py313"]
 all = ["dev", "test", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "tiff", "zarr", "all_parsers", "all_writers", "py313"]
 docs = ["docs", "dev", "remote", "hdf", "netcdf3", "fits", "icechunk", "kerchunk", "kerchunk_parquet", "hdf5-lib", "tiff","zarr", "py313"]