From 79cc464b027f66eb98f9907d35ae4bc8857749f5 Mon Sep 17 00:00:00 2001 From: Adrian Hayler Date: Thu, 25 Jun 2026 18:41:03 +0200 Subject: [PATCH 1/4] test: run example files as CI tests; fix bugs they surface (PRI-328) Rewrite the example-test harness to run each example as a subprocess, with fast per-PR smoke semantics (a timeout passes; only an error fails) and a strict mode for the scheduled GPU run (PRI-330). Add a path-filtered CPU smoke workflow that runs it on PRs touching examples/. Running the examples for real surfaced several breakages, fixed here: - shapiq examples: bump the interpretability floor to shapiq>=1.2.0. The old >=1.1.0 floor let lowest-direct install 1.1.0, which predates class_index (1.1.1) and TabPFNExplainer (1.2.0); the example crashed on both. - shap_example: add shap to a new "examples" dependency group, floored at 0.46.0 (0.41.0 pulled an ancient, unbuildable numba). - generate_data_following_dag: import the unsupervised.experiments submodule explicitly instead of relying on attribute access. - tabebm example: xfail with a tracking note; broken with current TabPFN's inference_config API (#225). Closes #51. Supersedes #321. Co-Authored-By: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/example_tests.yml | 79 ++++ .../tabebm/tabebm_augment_real_world_data.py | 13 + .../generate_data_following_dag.py | 3 +- pyproject.toml | 27 +- tests/conftest.py | 20 + tests/test_examples.py | 355 +++++++++--------- 6 files changed, 312 insertions(+), 185 deletions(-) create mode 100644 .github/workflows/example_tests.yml diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml new file mode 100644 index 00000000..586810ab --- /dev/null +++ b/.github/workflows/example_tests.yml @@ -0,0 +1,79 @@ +name: Example smoke tests +run-name: "Example smoke on ${{ github.head_ref || github.ref_name }}" + +# Fast per-PR smoke gate for the scripts in examples/. Each example is run as a +# subprocess; the test fails only if it *errors*. An example that does not finish +# within the timeout still passes -- we only assert it starts and runs without +# crashing. Running every example to completion (where a timeout is a failure) is +# the job of the scheduled GPU run; see PRI-330. +# +# Path-filtered so this only runs when an example or the harness changes (~most +# PRs don't touch examples/). +on: + pull_request: + branches: + - main + paths: + - "examples/**" + - "tests/test_examples.py" + - "tests/conftest.py" + - "pyproject.toml" + - ".github/workflows/example_tests.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: example-smoke-${{ github.ref }} + cancel-in-progress: true + +jobs: + example_smoke: + name: Example smoke (CPU) + runs-on: ubuntu-latest + env: + TABPFN_MODEL_CACHE_DIR: ${{ github.workspace }}/model_cache + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + + # Highest resolution on purpose: examples are user-facing tutorials, so we + # test them against the deps a fresh install actually gets. The "examples" + # group adds shap (used only by shap_example.py for plotting). + - name: Install dependencies + run: uv sync --all-extras --group examples --group ci + + - name: Restore model cache + id: restore-model-cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/model_cache + key: model-cache-${{ hashFiles('model_cache/**') }} + restore-keys: | + model-cache- + enableCrossOsArchive: true + + - name: Download models from Hugging Face + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + TABPFN_TOKEN: ${{ secrets.TABPFN_TOKEN }} + run: uv run --no-sync python scripts/download_all_models.py + + # Smoke mode (no --example-strict): a timeout is a pass. -n auto runs the + # examples in parallel to keep wall-clock down. + - name: Run example smoke tests + env: + TABPFN_EXCLUDE_DEVICES: mps + run: | + FAST_TEST_MODE=1 uv run --no-sync pytest tests/test_examples.py \ + --run-examples -n auto --example-timeout 120 diff --git a/examples/tabebm/tabebm_augment_real_world_data.py b/examples/tabebm/tabebm_augment_real_world_data.py index 1840b852..83ee2d34 100644 --- a/examples/tabebm/tabebm_augment_real_world_data.py +++ b/examples/tabebm/tabebm_augment_real_world_data.py @@ -3,6 +3,19 @@ This script replicates the functionality of the notebook but uses sklearn instead of TabCamel for data loading and preprocessing. + +.. warning:: + TabEBM does not currently work with recent TabPFN versions: it relies on + ``tabpfn.config.ModelInterfaceConfig`` and ``PREPROCESS_TRANSFORMS`` to + disable preprocessing for SGLD sampling, both of which were removed/renamed + in TabPFN's ``inference_config`` API. Tracked in + https://github.com/PriorLabs/tabpfn-extensions/issues/225 -- this example + will fail to import until that is resolved. + + As a workaround you can run it with the TabPFN version TabEBM was built + against, where ``tabpfn.config`` still exists:: + + pip install "tabpfn>=2.1.1,<3" """ import warnings diff --git a/examples/unsupervised/generate_data_following_dag.py b/examples/unsupervised/generate_data_following_dag.py index 8c9daac5..98cfa241 100644 --- a/examples/unsupervised/generate_data_following_dag.py +++ b/examples/unsupervised/generate_data_following_dag.py @@ -16,6 +16,7 @@ from sklearn.model_selection import train_test_split from tabpfn_extensions import TabPFNClassifier, TabPFNRegressor, unsupervised +from tabpfn_extensions.unsupervised import experiments df = load_wine(return_X_y=False) X, y = df["data"], df["target"] @@ -54,7 +55,7 @@ tabpfn_reg=reg, ) -exp_synthetic = unsupervised.experiments.GenerateSyntheticDataExperiment( +exp_synthetic = experiments.GenerateSyntheticDataExperiment( task_type="unsupervised", ) diff --git a/pyproject.toml b/pyproject.toml index b927e423..09bc977b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,12 @@ source = "https://github.com/PriorLabs/tabpfn-extensions" [project.optional-dependencies] interpretability = [ - # 1.1.0 is the lowest release where `imputer="baseline"` + `index="SV"` + - # `max_order=1` (the default config used by get_tabpfn_imputation_explainer) - # works without raising. - "shapiq>=1.1.0", + # 1.2.0 is the lowest release providing every shapiq feature this package + # uses: `class_index` on TabularExplainer/Explainer (added in 1.1.1) and + # `shapiq.TabPFNExplainer` (added in 1.2.0, used by get_tabpfn_explainer). + # With the old `>=1.1.0` floor, CI's lowest-direct resolution installed + # 1.1.0 and the examples crashed (missing class_index / TabPFNExplainer). + "shapiq>=1.2.0", "seaborn>=0.12.2", ] post_hoc_ensembles = [ @@ -101,6 +103,20 @@ dev = [ "build>=1.3.0", "twine>=6.2.0", ] +# Extra deps needed only to *run* the example scripts (see examples/), used by the +# example-file CI job. Deliberately not a published extra: the library itself does +# not need these, only the example demos do, and the examples ship in the repo (not +# the wheel), so anyone running them works from a clone where this group is available. +examples = [ + # interpretability/shap_example.py bridges shapiq values into a shap.Explanation + # purely for plotting (shap.summary_plot / shap.plots.*). The library dropped shap + # as a runtime dep in #283 (RES-1467) since shapiq supersedes the adapter; it + # survives only as an example plotting convenience. + # Floor is 0.46.0, not the older 0.41.0: 0.41.0 pins an ancient numba (0.47.0) + # with no cp310+ wheel, which fails to build from source. 0.46.0 leaves numba + # unconstrained so a modern, wheel-backed numba is used. + "shap>=0.46.0", +] [tool.pytest.ini_options] testpaths = ["tests"] # Where the tests are located @@ -110,6 +126,9 @@ log_cli = false log_level = "DEBUG" xfail_strict = true addopts = "--durations=10 -vv" +markers = [ + "example: runs an example script from examples/ (deselect with -m 'not example')", +] # https://github.com/astral-sh/ruff [tool.ruff] diff --git a/tests/conftest.py b/tests/conftest.py index da9d535e..01412605 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,6 +116,26 @@ def pytest_addoption(parser): default="all", help="Specify which TabPFN backend to use: 'tabpfn', 'tabpfn_client', or 'all' (use both if available)", ) + parser.addoption( + "--example-timeout", + action="store", + type=int, + default=120, + help="Per-example timeout in seconds for normal examples (see tests/test_examples.py)", + ) + parser.addoption( + "--example-long-timeout", + action="store", + type=int, + default=120, + help="Per-example timeout in seconds for long-running examples (a timeout never fails these)", + ) + parser.addoption( + "--example-strict", + action="store_true", + default=False, + help="Treat a normal example that doesn't finish within the timeout as a failure (weekly full run)", + ) # Define markers for tests diff --git a/tests/test_examples.py b/tests/test_examples.py index 42d3e6fa..714717a0 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,227 +1,222 @@ -"""Test examples to ensure they work as expected. - -This module provides a testing framework for TabPFN example files. It automatically: -1. Detects all example files in the examples/ directory -2. Categorizes them as fast, slow, or large dataset examples -3. Runs fast examples normally -4. Tests slow examples with a short timeout (expecting timeout as success) -5. Skips large dataset examples entirely unless explicitly requested -6. Handles backend compatibility for TabPFN package vs. TabPFN client +"""Run the example files in ``examples/`` and check they work. + +Each example is executed as a subprocess. The outcome is interpreted with one +of two policies, selected per CI layer: + +* **Per-PR smoke (default):** each example gets a short timeout. An example that + *errors* fails the test; one that simply doesn't finish in time *passes* — we + only assert it starts and runs without crashing. This is a fast, cheap guard + against broken example scripts and import errors. +* **Weekly full run (``--example-strict``):** a longer timeout, and a normal + example that doesn't finish is a *failure* — i.e. we assert it actually runs + to completion. Examples explicitly marked long-running (``LONG_RUNNING``) are + still allowed to time out even here, since they cannot complete in CI by + design; we still run them to catch import/startup errors. + +Regardless of policy: +* An example that needs an optional dependency which isn't installed is + **skipped** (not failed). +* A GPU-only example is **skipped** when no CUDA device is available (some + examples exceed TabPFN's CPU sample guard and only run on a GPU). Usage: - # Run only fast examples: - FAST_TEST_MODE=1 python -m pytest tests/test_examples.py - - # Run all examples except large dataset ones (slow ones will timeout and be marked as xfail): - FAST_TEST_MODE=1 python -m pytest tests/test_examples.py --run-examples + # Per-PR smoke (short timeout; not finishing is OK): + uv run --no-sync pytest tests/test_examples.py --run-examples \ + --example-timeout 120 - # Run specific example, even if it's a large dataset example: - python -m pytest tests/test_examples.py::test_example[large_datasets_example.py] --run-examples + # Weekly full run (long timeout; a normal example not finishing fails): + uv run --no-sync pytest tests/test_examples.py --run-examples \ + --example-strict --example-timeout 600 --example-long-timeout 120 """ from __future__ import annotations import importlib.util import os +import subprocess import sys from pathlib import Path import pytest -# Enable test mode to make examples run faster +# Enable test mode so examples shrink their workload where they support it. os.environ["TEST_MODE"] = "1" +# Directories whose examples need the full TabPFN package (won't work with the +# TabPFN client). +REQUIRES_TABPFN_DIRS = ["embedding/"] + +# Examples needing a module that is intentionally never installed, so they skip +# (rather than fail) when it is absent. Reserved for the GPL-excluded case: +# scikit-survival is in no extra or group, so survival_example always skips unless +# the user installs it manually. (Other example deps -- shapiq, shap, hyperopt, ... -- +# are installed via --all-extras / the "examples" group and are expected to be present.) +REQUIRES_MODULE = { + "survival_example.py": "sksurv", +} + +# Examples that exceed TabPFN's CPU sample guard and only run on a GPU. +# Skipped when no CUDA device is available. +GPU_ONLY = { + "get_embeddings.py", +} + +# Examples that cannot complete within any CI time budget by design (e.g. PHE's +# hard-coded ``max_time``). A timeout is NOT a failure for these, even in strict +# mode; we still run them to catch import/startup errors. +LONG_RUNNING = { + "phe_example.py", +} + +# Examples known to be broken against current dependencies, with a tracking issue. +# xfailed so the suite stays green while the breakage is documented; when the +# upstream fix lands the example will start passing (reported as XPASS) and the +# entry should be removed. +KNOWN_BROKEN = { + # TabEBM relies on tabpfn.config.ModelInterfaceConfig / PREPROCESS_TRANSFORMS, + # removed/renamed in TabPFN's inference_config API -> import error. + "tabebm_augment_real_world_data.py": ( + "TabEBM is broken with current TabPFN; see " + "https://github.com/PriorLabs/tabpfn-extensions/issues/225" + ), +} + + +def _example_params() -> list: + """Build the parametrize values, xfailing known-broken examples.""" + params = [] + for example_file in get_example_files(): + marks = [] + reason = KNOWN_BROKEN.get(example_file["name"]) + if reason is not None: + # Non-strict: in smoke mode a timeout counts as a pass, so a partially + # fixed example could XPASS without truly being fixed -- don't fail the + # suite on that, just surface it as XPASS for follow-up. + marks.append(pytest.mark.xfail(reason=reason, strict=False)) + params.append( + pytest.param(example_file, marks=marks, id=example_file["name"]), + ) + return params -def get_example_files() -> list[dict]: - """Get all Python files from the examples directory with metadata. - - Each example is categorized as: - - fast: Can run quickly (runs in both normal and fast test mode) - - slow: Takes longer to run (runs with a 1-second timeout, expected to timeout) - - always_timeout: Examples with large datasets that are always skipped unless explicitly requested - - requires_tabpfn: If True, requires the full TabPFN package and won't work with client; - if False, works with either TabPFN package or TabPFN client - Returns: - List of dictionaries containing example file info - """ +def get_example_files() -> list[dict]: + """Discover example files and attach the metadata the runner needs.""" package_root = Path(__file__).parent.parent examples_dir = package_root / "examples" - # The only example that runs fast enough for CI - FAST_EXAMPLES = [] - - # These directories/files need the full TabPFN package and won't work with client - REQUIRES_TABPFN_DIRS = ["embedding/"] - - # Large dataset examples are always expected to timeout, - # even if --run-examples is provided - ALWAYS_TIMEOUT_PATTERNS = ["large_datasets_example.py"] - - # Find all Python files in the examples directory - all_file_paths = list(examples_dir.glob("**/*.py")) - all_files = [] - - # Process each file with appropriate metadata - for file_path in all_file_paths: + files = [] + for file_path in sorted(examples_dir.glob("**/*.py")): rel_path = str(file_path.relative_to(package_root)) - file_name = file_path.name - - file_info = { - "path": file_path, - "name": file_name, - # Default classification - most examples work with both implementations - "requires_tabpfn": False, # By default, examples work with either implementation - "fast": file_name in FAST_EXAMPLES, # Only listed examples are fast - "slow": file_name not in FAST_EXAMPLES, # All others are slow - "always_timeout": any( - pattern in file_name for pattern in ALWAYS_TIMEOUT_PATTERNS - ), - "timeout": 1 - if file_name not in FAST_EXAMPLES - else 30, # Short timeout for slow examples - } - - # Check if example requires full TabPFN package - if any(pattern in rel_path for pattern in REQUIRES_TABPFN_DIRS): - # Example explicitly requires TabPFN package - file_info["requires_tabpfn"] = True - - all_files.append(file_info) - - return all_files - - -def import_module_from_path(path: Path, timeout: int = None) -> object: - """Dynamically import a Python module from a file path. + name = file_path.name + files.append( + { + "path": file_path, + "name": name, + "requires_tabpfn": any( + pattern in rel_path for pattern in REQUIRES_TABPFN_DIRS + ), + "requires_module": REQUIRES_MODULE.get(name), + "gpu_only": name in GPU_ONLY, + "long": name in LONG_RUNNING, + }, + ) + return files - Args: - path: Path to the Python file to import - timeout: Optional timeout parameter (no longer used internally, kept for backward compatibility) - Returns: - The imported module object - """ - # Add the parent directory to sys.path to allow imports within example files - parent_dir = str(path.parent) - if parent_dir not in sys.path: - sys.path.insert(0, parent_dir) +def _cuda_available() -> bool: + """Whether a CUDA device is visible to torch (best-effort).""" + try: + import torch - # Import the module - spec = importlib.util.spec_from_file_location(path.stem, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return torch.cuda.is_available() + except Exception: # noqa: BLE001 + return False @pytest.mark.example -@pytest.mark.parametrize("example_file", get_example_files(), ids=lambda x: x["name"]) +@pytest.mark.parametrize("example_file", _example_params()) def test_example(request, example_file): - """Run example files to ensure they work as expected. - - Test strategy: - 1. Fast examples are run with normal timeout - 2. Slow examples are run with 1-second timeout, expected to timeout - 3. Examples are skipped if they require missing backends - 4. In FAST_TEST_MODE, only fast examples and examples with --run-examples flag run + """Run a single example file as a subprocess and check the outcome. Args: - request: PyTest request fixture - example_file: Dictionary with example file metadata + request: PyTest request fixture. + example_file: Dictionary with example file metadata. """ from conftest import HAS_TABPFN, TABPFN_SOURCE - file_name = example_file["name"] - file_path = example_file["path"] + name = example_file["name"] + path = example_file["path"] - run_examples = request.config.getoption("--run-examples") - - if not run_examples: - pytest.skip( - f"Skipping {file_name} since --run-examples not set", - ) + if not request.config.getoption("--run-examples"): + pytest.skip(f"Skipping {name} since --run-examples not set") - # Skip if backend not available + # Backend availability if example_file["requires_tabpfn"]: if not HAS_TABPFN: pytest.skip( - f"Example {file_name} requires TabPFN package, but it's not installed", + f"Example {name} requires the TabPFN package, which is not installed", ) - elif TABPFN_SOURCE == "tabpfn_client": + if TABPFN_SOURCE == "tabpfn_client": pytest.skip( - f"Example {file_name} requires TabPFN package, not compatible with client", + f"Example {name} requires the TabPFN package, not the client", ) + # Optional dependency not installed -> skip (not a failure) + required_module = example_file["requires_module"] + if required_module and importlib.util.find_spec(required_module) is None: + pytest.skip( + f"Example {name} requires '{required_module}', which is not installed" + ) + + # GPU-only example with no CUDA device -> skip + if example_file["gpu_only"] and not _cuda_available(): + pytest.skip( + f"Example {name} requires a CUDA device (exceeds TabPFN's CPU sample limit)", + ) + + strict = request.config.getoption("--example-strict") + if example_file["long"]: + timeout = request.config.getoption("--example-long-timeout") + else: + timeout = request.config.getoption("--example-timeout") + + # Examples are top-to-bottom scripts; run each in its own process so a hang + # can be killed cleanly and state never leaks between examples. The example + # inherits TEST_MODE/FAST_TEST_MODE/TABPFN_EXCLUDE_DEVICES from this process. + env = dict(os.environ) + env["TEST_MODE"] = "1" + try: - # Handle slow examples (including large datasets) differently - if example_file.get("slow", False) or example_file.get("always_timeout", False): - # For slow examples, we'll run them with a short internal timeout - # and expect them to be interrupted - import threading - - def run_with_timeout(path, max_time=5): - """Run import with a timeout using threading approach.""" - result = {"completed": False, "exception": None} - - def target(): - try: - # Import the module - spec = importlib.util.spec_from_file_location(path.stem, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - result["completed"] = True - except Exception as e: # noqa: BLE001 - result["exception"] = e - - # Start the import in a separate thread - thread = threading.Thread(target=target) - thread.daemon = ( - True # Daemon threads are killed when the main thread exits - ) - thread.start() - - # Wait for the thread to complete or timeout - thread.join(timeout=max_time) - - return result - - # Add parent directory to path - parent_dir = str(file_path.parent) - if parent_dir not in sys.path: - sys.path.insert(0, parent_dir) - - # Run with a 5 second timeout - run_result = run_with_timeout(file_path, max_time=5) - - if run_result["completed"]: - # If it completed within the timeout, that's fine - print( - f"Note: Slow example {file_name} completed successfully within 5 seconds", - ) - elif run_result["exception"]: - # If it failed for reasons other than timeout - pytest.xfail(f"Example {file_name} failed: {run_result['exception']}") - else: - # Expected timeout after running for 5 seconds - pytest.xfail( - f"Example {file_name} ran for 5 seconds and was stopped as expected", - ) - else: - # Fast examples should complete normally - import_module_from_path(file_path, timeout=None) - except TimeoutError as e: - # Unexpected timeout in fast examples is a failure - pytest.fail(f"Example {file_name} timed out: {e!s}") - - -def pytest_addoption(parser): - """Add command-line options to pytest.""" - parser.addoption( - "--run-examples", - action="store_true", - default=False, - help="Run all example files (including slow ones that will be expected to timeout)", - ) + proc = subprocess.run( # noqa: S603 - trusted, repo-local example scripts + [sys.executable, str(path)], + cwd=str(path.parent), + env=env, + timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + except subprocess.TimeoutExpired: + if example_file["long"]: + print( + f"{name}: ran {timeout}s without error " + f"(long-running; not run to completion by design)", + ) + return + if strict: + pytest.fail(f"Example {name} did not finish within {timeout}s") + print( + f"{name}: ran {timeout}s without error " + f"(smoke mode; completion not verified)", + ) + return + + if proc.returncode != 0: + output = (proc.stdout or b"").decode("utf-8", "replace") + tail = "\n".join(output.strip().splitlines()[-25:]) + pytest.fail( + f"Example {name} exited with code {proc.returncode}:\n{tail}", + ) if __name__ == "__main__": From 83b4dbc5443377e204da84df106e48351dd29e6b Mon Sep 17 00:00:00 2001 From: Adrian Hayler Date: Thu, 25 Jun 2026 18:42:13 +0200 Subject: [PATCH 2/4] docs: add changelog entries for #327 Co-Authored-By: Claude Opus 4.8 (1M context) --- changelog/327.added.md | 1 + changelog/327.fixed.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog/327.added.md create mode 100644 changelog/327.fixed.md diff --git a/changelog/327.added.md b/changelog/327.added.md new file mode 100644 index 00000000..72c697f6 --- /dev/null +++ b/changelog/327.added.md @@ -0,0 +1 @@ +Example scripts in `examples/` are now exercised in CI via a fast per-PR smoke test (path-filtered to example changes), guarding against examples that stop running. diff --git a/changelog/327.fixed.md b/changelog/327.fixed.md new file mode 100644 index 00000000..9ef07759 --- /dev/null +++ b/changelog/327.fixed.md @@ -0,0 +1 @@ +Repaired example scripts that had stopped running: raised the `interpretability` extra's `shapiq` floor to `>=1.2.0` (needed for `class_index` and `TabPFNExplainer`) so the SHAP/shapiq examples work again, and fixed the `unsupervised.experiments` import in the DAG example. From 4dc126ca024a2f4178bcdc8c81f21c64584f791c Mon Sep 17 00:00:00 2001 From: Adrian Hayler Date: Thu, 25 Jun 2026 18:48:53 +0200 Subject: [PATCH 3/4] test: address Gemini review on the example harness - Prepend the in-repo src/ to the example subprocess PYTHONPATH so it imports this checkout's tabpfn_extensions even without an editable install. - Widen the failure-output tail from 25 to 100 lines for easier CI debugging. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_examples.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 714717a0..7559dbb4 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -185,6 +185,15 @@ def test_example(request, example_file): # inherits TEST_MODE/FAST_TEST_MODE/TABPFN_EXCLUDE_DEVICES from this process. env = dict(os.environ) env["TEST_MODE"] = "1" + # Prepend the in-repo src/ so the example subprocess imports this checkout's + # tabpfn_extensions, not some other installed copy (and works even if the + # package isn't installed in editable mode). + src_dir = str(Path(__file__).parent.parent / "src") + env["PYTHONPATH"] = ( + f"{src_dir}{os.pathsep}{env['PYTHONPATH']}" + if env.get("PYTHONPATH") + else src_dir + ) try: proc = subprocess.run( # noqa: S603 - trusted, repo-local example scripts @@ -213,7 +222,7 @@ def test_example(request, example_file): if proc.returncode != 0: output = (proc.stdout or b"").decode("utf-8", "replace") - tail = "\n".join(output.strip().splitlines()[-25:]) + tail = "\n".join(output.strip().splitlines()[-100:]) pytest.fail( f"Example {name} exited with code {proc.returncode}:\n{tail}", ) From e7178e81dd94931eed27fd44167ce2db669df5b5 Mon Sep 17 00:00:00 2001 From: Adrian Hayler Date: Mon, 13 Jul 2026 18:21:05 +0200 Subject: [PATCH 4/4] test: address Oscar's review - Fold the example smoke gate into pull_request.yml as an always-run job (drop the separate path-filtered workflow); it now also runs on src/ changes. - Inline torch.cuda.is_available() instead of the _cuda_available() wrapper. - Drop the subprocess PYTHONPATH prepend; rely on the installed package. - Trim the shapiq/shap dependency comments. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/example_tests.yml | 79 ----------------------------- .github/workflows/pull_request.yml | 54 ++++++++++++++++++++ pyproject.toml | 16 ++---- tests/test_examples.py | 22 +------- 4 files changed, 60 insertions(+), 111 deletions(-) delete mode 100644 .github/workflows/example_tests.yml diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml deleted file mode 100644 index 586810ab..00000000 --- a/.github/workflows/example_tests.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Example smoke tests -run-name: "Example smoke on ${{ github.head_ref || github.ref_name }}" - -# Fast per-PR smoke gate for the scripts in examples/. Each example is run as a -# subprocess; the test fails only if it *errors*. An example that does not finish -# within the timeout still passes -- we only assert it starts and runs without -# crashing. Running every example to completion (where a timeout is a failure) is -# the job of the scheduled GPU run; see PRI-330. -# -# Path-filtered so this only runs when an example or the harness changes (~most -# PRs don't touch examples/). -on: - pull_request: - branches: - - main - paths: - - "examples/**" - - "tests/test_examples.py" - - "tests/conftest.py" - - "pyproject.toml" - - ".github/workflows/example_tests.yml" - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: example-smoke-${{ github.ref }} - cancel-in-progress: true - -jobs: - example_smoke: - name: Example smoke (CPU) - runs-on: ubuntu-latest - env: - TABPFN_MODEL_CACHE_DIR: ${{ github.workspace }}/model_cache - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "3.12" - - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - enable-cache: true - - # Highest resolution on purpose: examples are user-facing tutorials, so we - # test them against the deps a fresh install actually gets. The "examples" - # group adds shap (used only by shap_example.py for plotting). - - name: Install dependencies - run: uv sync --all-extras --group examples --group ci - - - name: Restore model cache - id: restore-model-cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ${{ github.workspace }}/model_cache - key: model-cache-${{ hashFiles('model_cache/**') }} - restore-keys: | - model-cache- - enableCrossOsArchive: true - - - name: Download models from Hugging Face - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - TABPFN_TOKEN: ${{ secrets.TABPFN_TOKEN }} - run: uv run --no-sync python scripts/download_all_models.py - - # Smoke mode (no --example-strict): a timeout is a pass. -n auto runs the - # examples in parallel to keep wall-clock down. - - name: Run example smoke tests - env: - TABPFN_EXCLUDE_DEVICES: mps - run: | - FAST_TEST_MODE=1 uv run --no-sync pytest tests/test_examples.py \ - --run-examples -n auto --example-timeout 120 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7960dde3..06062060 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -202,6 +202,60 @@ jobs: key: model-cache-${{ hashFiles('model_cache/**') }} enableCrossOsArchive: true + example_smoke: + name: Example smoke (CPU) + # Fast smoke gate for the scripts in examples/. Each example runs as a + # subprocess; the test fails only if it *errors*. An example that doesn't + # finish within the timeout still passes -- we only assert it starts and runs + # without crashing. Running every example to completion (timeout = failure) is + # the job of the scheduled GPU run; see PRI-330. + runs-on: ubuntu-latest + env: + TABPFN_MODEL_CACHE_DIR: ${{ github.workspace }}/model_cache + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + + # Highest resolution on purpose: examples are user-facing tutorials, so we + # test them against the deps a fresh install gets. The "examples" group adds + # shap (used only by shap_example.py for plotting). + - name: Install dependencies + run: uv sync --all-extras --group examples --group ci + + - name: Restore model cache + id: restore-model-cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/model_cache + key: model-cache-${{ hashFiles('model_cache/**') }} + restore-keys: | + model-cache- + enableCrossOsArchive: true + + - name: Download models from Hugging Face + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + TABPFN_TOKEN: ${{ secrets.TABPFN_TOKEN }} + run: uv run --no-sync python scripts/download_all_models.py + + # Smoke mode (no --example-strict): a timeout is a pass. -n auto runs the + # examples in parallel to keep wall-clock down. + - name: Run example smoke tests + env: + TABPFN_EXCLUDE_DEVICES: mps + run: | + FAST_TEST_MODE=1 uv run --no-sync pytest tests/test_examples.py \ + --run-examples -n auto --example-timeout 120 + build_verify: name: Build wheel + sdist runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index 09bc977b..ac24f67a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,11 +56,8 @@ source = "https://github.com/PriorLabs/tabpfn-extensions" [project.optional-dependencies] interpretability = [ - # 1.2.0 is the lowest release providing every shapiq feature this package - # uses: `class_index` on TabularExplainer/Explainer (added in 1.1.1) and - # `shapiq.TabPFNExplainer` (added in 1.2.0, used by get_tabpfn_explainer). - # With the old `>=1.1.0` floor, CI's lowest-direct resolution installed - # 1.1.0 and the examples crashed (missing class_index / TabPFNExplainer). + # 1.2.0 is the first release with shapiq.TabPFNExplainer (get_tabpfn_explainer) + # and class_index on TabularExplainer. "shapiq>=1.2.0", "seaborn>=0.12.2", ] @@ -108,13 +105,8 @@ dev = [ # not need these, only the example demos do, and the examples ship in the repo (not # the wheel), so anyone running them works from a clone where this group is available. examples = [ - # interpretability/shap_example.py bridges shapiq values into a shap.Explanation - # purely for plotting (shap.summary_plot / shap.plots.*). The library dropped shap - # as a runtime dep in #283 (RES-1467) since shapiq supersedes the adapter; it - # survives only as an example plotting convenience. - # Floor is 0.46.0, not the older 0.41.0: 0.41.0 pins an ancient numba (0.47.0) - # with no cp310+ wheel, which fails to build from source. 0.46.0 leaves numba - # unconstrained so a modern, wheel-backed numba is used. + # Only used by interpretability/shap_example.py for plotting. + # Floor is 0.46.0 (0.41.0 pins an unbuildable numba). "shap>=0.46.0", ] diff --git a/tests/test_examples.py b/tests/test_examples.py index 7559dbb4..68a69f44 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -38,6 +38,7 @@ from pathlib import Path import pytest +import torch # Enable test mode so examples shrink their workload where they support it. os.environ["TEST_MODE"] = "1" @@ -123,16 +124,6 @@ def get_example_files() -> list[dict]: return files -def _cuda_available() -> bool: - """Whether a CUDA device is visible to torch (best-effort).""" - try: - import torch - - return torch.cuda.is_available() - except Exception: # noqa: BLE001 - return False - - @pytest.mark.example @pytest.mark.parametrize("example_file", _example_params()) def test_example(request, example_file): @@ -169,7 +160,7 @@ def test_example(request, example_file): ) # GPU-only example with no CUDA device -> skip - if example_file["gpu_only"] and not _cuda_available(): + if example_file["gpu_only"] and not torch.cuda.is_available(): pytest.skip( f"Example {name} requires a CUDA device (exceeds TabPFN's CPU sample limit)", ) @@ -185,15 +176,6 @@ def test_example(request, example_file): # inherits TEST_MODE/FAST_TEST_MODE/TABPFN_EXCLUDE_DEVICES from this process. env = dict(os.environ) env["TEST_MODE"] = "1" - # Prepend the in-repo src/ so the example subprocess imports this checkout's - # tabpfn_extensions, not some other installed copy (and works even if the - # package isn't installed in editable mode). - src_dir = str(Path(__file__).parent.parent / "src") - env["PYTHONPATH"] = ( - f"{src_dir}{os.pathsep}{env['PYTHONPATH']}" - if env.get("PYTHONPATH") - else src_dir - ) try: proc = subprocess.run( # noqa: S603 - trusted, repo-local example scripts