|
| 1 | +"""Backwards compatibility tests for glum. |
| 2 | +
|
| 3 | +Usage: python tests/backwards_compatibility/run_all.py |
| 4 | + (or via: pixi run test-backwards-compatibility) |
| 5 | +
|
| 6 | +1. Fits the current (HEAD) glum to produce reference predictions. |
| 7 | +2. Queries conda-forge via `pixi search` to discover the latest patch release |
| 8 | + for each minor version of glum. |
| 9 | +3. For each version, uses `pixi exec` to fit a model and save artifacts |
| 10 | + (model.pkl + predictions.npy) under artifacts/<version>/. |
| 11 | +4. Unpickles each saved model using the current glum and verifies that |
| 12 | + predictions match the HEAD reference. |
| 13 | +""" |
| 14 | + |
| 15 | +import json |
| 16 | +import pickle |
| 17 | +import subprocess |
| 18 | +import sys |
| 19 | +from pathlib import Path |
| 20 | + |
| 21 | +import numpy as np |
| 22 | +from packaging.version import Version |
| 23 | +from sklearn.datasets import make_regression |
| 24 | + |
| 25 | +SCRIPT_DIR = Path(__file__).resolve().parent |
| 26 | +ARTIFACTS_DIR = SCRIPT_DIR / "artifacts" |
| 27 | + |
| 28 | +SKIP_VERSIONS: set[str] = set() |
| 29 | + |
| 30 | + |
| 31 | +def write_dataset() -> None: |
| 32 | + """Write the fixed dataset to disk so all fit.py invocations use identical data.""" |
| 33 | + X, y = make_regression(n_samples=500, n_features=5, noise=1.0, random_state=42) |
| 34 | + ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) |
| 35 | + np.save(str(ARTIFACTS_DIR / "X.npy"), X) |
| 36 | + np.save(str(ARTIFACTS_DIR / "y.npy"), y) |
| 37 | + |
| 38 | + |
| 39 | +def discover_versions() -> list[str]: |
| 40 | + """Return the latest patch release for each minor version of glum on conda-forge.""" |
| 41 | + result = subprocess.run( |
| 42 | + ["pixi", "search", "glum", "--json"], |
| 43 | + check=True, |
| 44 | + capture_output=True, |
| 45 | + text=True, |
| 46 | + ) |
| 47 | + data = json.loads(result.stdout) |
| 48 | + platform = next(iter(data)) |
| 49 | + best: dict[tuple[int, int], str] = {} |
| 50 | + for entry in data[platform]: |
| 51 | + v = Version(entry["version"]) |
| 52 | + key = (v.major, v.minor) |
| 53 | + if key not in best or v > Version(best[key]): |
| 54 | + best[key] = entry["version"] |
| 55 | + return sorted(best.values(), key=Version) |
| 56 | + |
| 57 | + |
| 58 | +def fit_version(version: str) -> bool: |
| 59 | + """Run fit.py for the given version and return True on success. |
| 60 | +
|
| 61 | + Uses ``pixi run`` for HEAD and ``pixi exec`` for released versions. |
| 62 | + """ |
| 63 | + if version == "HEAD": |
| 64 | + cmd = ["pixi", "run", "python", str(SCRIPT_DIR / "fit.py"), "HEAD"] |
| 65 | + else: |
| 66 | + v = Version(version) |
| 67 | + cmd = ["pixi", "exec", f"--spec=glum=={version}"] |
| 68 | + # glum <=2.6.0 imports pkg_resources from setuptools, which was removed |
| 69 | + # in setuptools 82. Pin setuptools<82 for those old versions. |
| 70 | + if v <= Version("2.6.0"): |
| 71 | + cmd += ["--spec=setuptools<82"] |
| 72 | + # glum <=2.3.0: sklearn 1.3 added const qualifiers to _cython_blas |
| 73 | + # function pointers, breaking the Cython ABI of older glum builds. |
| 74 | + if v <= Version("2.3.0"): |
| 75 | + cmd += ["--spec=scikit-learn<1.3"] |
| 76 | + # glum 3.0.x: sklearn 1.6 removed BaseEstimator._validate_data. |
| 77 | + elif v < Version("3.1.0"): |
| 78 | + cmd += ["--spec=scikit-learn<1.6"] |
| 79 | + cmd += ["python", str(SCRIPT_DIR / "fit.py"), version] |
| 80 | + result = subprocess.run(cmd, capture_output=True, text=True) |
| 81 | + if result.returncode != 0: |
| 82 | + print(result.stdout, end="") |
| 83 | + print(result.stderr, end="", file=sys.stderr) |
| 84 | + return result.returncode == 0 |
| 85 | + |
| 86 | + |
| 87 | +def compare_versions(versions: list[str]) -> bool: |
| 88 | + """Unpickle each version's model and verify its predictions match HEAD. |
| 89 | +
|
| 90 | + Also checks that predictions match the array stored by fit.py to confirm |
| 91 | + the pickle round-trip is stable. Returns True if all versions pass. |
| 92 | + """ |
| 93 | + version_dirs = [ARTIFACTS_DIR / v for v in versions if (ARTIFACTS_DIR / v).is_dir()] |
| 94 | + |
| 95 | + if not version_dirs: |
| 96 | + print("ERROR: No artifact directories found. Did fit step produce any output?") |
| 97 | + return False |
| 98 | + |
| 99 | + X = np.load(str(ARTIFACTS_DIR / "X.npy")) |
| 100 | + head_predictions = np.load(str(ARTIFACTS_DIR / "HEAD" / "predictions.npy")) |
| 101 | + |
| 102 | + import glum |
| 103 | + |
| 104 | + current_version = glum.__version__ |
| 105 | + print(f"Current glum version: {current_version}") |
| 106 | + print(f"Testing {len(version_dirs)} version(s): {[d.name for d in version_dirs]}\n") |
| 107 | + |
| 108 | + failures = [] |
| 109 | + |
| 110 | + for version_dir in version_dirs: |
| 111 | + version = version_dir.name |
| 112 | + pickle_path = version_dir / "model.pkl" |
| 113 | + predictions_path = version_dir / "predictions.npy" |
| 114 | + |
| 115 | + try: |
| 116 | + with open(pickle_path, "rb") as f: |
| 117 | + old_model = pickle.load(f) |
| 118 | + except Exception as e: |
| 119 | + failures.append(f"{version}: unpickling failed: {e}") |
| 120 | + continue |
| 121 | + |
| 122 | + try: |
| 123 | + old_predictions = old_model.predict(X) |
| 124 | + except Exception as e: |
| 125 | + failures.append(f"{version}: predict() failed after unpickling: {e}") |
| 126 | + continue |
| 127 | + |
| 128 | + stored_predictions = np.load(str(predictions_path)) |
| 129 | + |
| 130 | + try: |
| 131 | + np.testing.assert_allclose( |
| 132 | + old_predictions, |
| 133 | + stored_predictions, |
| 134 | + rtol=1e-5, |
| 135 | + err_msg=f"[{version}] Unpickled predictions do not match stored array", |
| 136 | + ) |
| 137 | + print(f"[{version}] PASS: unpickled predictions match stored predictions") |
| 138 | + except AssertionError as e: |
| 139 | + failures.append(str(e)) |
| 140 | + |
| 141 | + try: |
| 142 | + np.testing.assert_allclose( |
| 143 | + old_predictions, |
| 144 | + head_predictions, |
| 145 | + rtol=1e-5, |
| 146 | + err_msg=f"[{version}] Predictions from old model do not match HEAD", |
| 147 | + ) |
| 148 | + print(f"[{version}] PASS: old model predictions match HEAD") |
| 149 | + except AssertionError as e: |
| 150 | + failures.append(str(e)) |
| 151 | + |
| 152 | + print() |
| 153 | + if failures: |
| 154 | + print("FAILURES:") |
| 155 | + for msg in failures: |
| 156 | + print(f" - {msg}") |
| 157 | + return False |
| 158 | + |
| 159 | + print(f"All {len(version_dirs)} version(s) passed.") |
| 160 | + return True |
| 161 | + |
| 162 | + |
| 163 | +def main() -> None: |
| 164 | + """Fit HEAD and all released minor versions, then compare predictions.""" |
| 165 | + write_dataset() |
| 166 | + |
| 167 | + print("=== Fitting HEAD ===") |
| 168 | + if not fit_version("HEAD"): |
| 169 | + print("ERROR: Failed to fit HEAD model.") |
| 170 | + sys.exit(1) |
| 171 | + |
| 172 | + print("\n=== Discovering glum versions from conda-forge ===") |
| 173 | + versions = discover_versions() |
| 174 | + print(f"Found {len(versions)} minor release(s): {' '.join(versions)}") |
| 175 | + |
| 176 | + print("\n=== Generating compatibility artifacts ===") |
| 177 | + fitted_versions = [] |
| 178 | + for version in versions: |
| 179 | + if version in SKIP_VERSIONS: |
| 180 | + print(f"--- Skipping glum=={version} (known incompatibility) ---") |
| 181 | + continue |
| 182 | + print(f"--- Fitting glum=={version} ---") |
| 183 | + if fit_version(version): |
| 184 | + fitted_versions.append(version) |
| 185 | + else: |
| 186 | + print(f"WARNING: glum=={version} failed, skipping.") |
| 187 | + |
| 188 | + print("\n=== Comparing against HEAD ===") |
| 189 | + if not compare_versions(fitted_versions): |
| 190 | + sys.exit(1) |
| 191 | + |
| 192 | + |
| 193 | +if __name__ == "__main__": |
| 194 | + main() |
0 commit comments