test: run example files as CI tests; fix bugs they surface (PRI-328)

.github/workflows/pull_request.yml

@@ -202,6 +202,60 @@ jobs:
           key: model-cache-${{ hashFiles('model_cache/**') }}
           enableCrossOsArchive: true
 
+  example_smoke:
+    name: Example smoke (CPU)
+    # Fast smoke gate for the scripts in examples/. Each example runs as a
+    # subprocess; the test fails only if it *errors*. An example that doesn't
+    # finish within the timeout still passes -- we only assert it starts and runs
+    # without crashing. Running every example to completion (timeout = failure) is
+    # the job of the scheduled GPU run; see PRI-330.
+    runs-on: ubuntu-latest
+    env:
+      TABPFN_MODEL_CACHE_DIR: ${{ github.workspace }}/model_cache
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          enable-cache: true
+
+      # Highest resolution on purpose: examples are user-facing tutorials, so we
+      # test them against the deps a fresh install gets. The "examples" group adds
+      # shap (used only by shap_example.py for plotting).
+      - name: Install dependencies
+        run: uv sync --all-extras --group examples --group ci
+
+      - name: Restore model cache
+        id: restore-model-cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ github.workspace }}/model_cache
+          key: model-cache-${{ hashFiles('model_cache/**') }}
+          restore-keys: |
+            model-cache-
+          enableCrossOsArchive: true
+
+      - name: Download models from Hugging Face
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          TABPFN_TOKEN: ${{ secrets.TABPFN_TOKEN }}
+        run: uv run --no-sync python scripts/download_all_models.py
+
+      # Smoke mode (no --example-strict): a timeout is a pass. -n auto runs the
+      # examples in parallel to keep wall-clock down.
+      - name: Run example smoke tests
+        env:
+          TABPFN_EXCLUDE_DEVICES: mps
+        run: |
+          FAST_TEST_MODE=1 uv run --no-sync pytest tests/test_examples.py \
+            --run-examples -n auto --example-timeout 120
+
   build_verify:
     name: Build wheel + sdist
     runs-on: ubuntu-latest

changelog/327.added.md

@@ -0,0 +1 @@
+Example scripts in `examples/` are now exercised in CI via a fast per-PR smoke test (path-filtered to example changes), guarding against examples that stop running.

changelog/327.fixed.md

@@ -0,0 +1 @@
+Repaired example scripts that had stopped running: raised the `interpretability` extra's `shapiq` floor to `>=1.2.0` (needed for `class_index` and `TabPFNExplainer`) so the SHAP/shapiq examples work again, and fixed the `unsupervised.experiments` import in the DAG example.

examples/tabebm/tabebm_augment_real_world_data.py

@@ -3,6 +3,19 @@
 
 This script replicates the functionality of the notebook but uses sklearn
 instead of TabCamel for data loading and preprocessing.
+
+.. warning::
+   TabEBM does not currently work with recent TabPFN versions: it relies on
+   ``tabpfn.config.ModelInterfaceConfig`` and ``PREPROCESS_TRANSFORMS`` to
+   disable preprocessing for SGLD sampling, both of which were removed/renamed
+   in TabPFN's ``inference_config`` API. Tracked in
+   https://github.com/PriorLabs/tabpfn-extensions/issues/225 -- this example
+   will fail to import until that is resolved.
+
+   As a workaround you can run it with the TabPFN version TabEBM was built
+   against, where ``tabpfn.config`` still exists::
+
+       pip install "tabpfn>=2.1.1,<3"
 """
 
 import warnings

examples/unsupervised/generate_data_following_dag.py

@@ -16,6 +16,7 @@
 from sklearn.model_selection import train_test_split
 
 from tabpfn_extensions import TabPFNClassifier, TabPFNRegressor, unsupervised
+from tabpfn_extensions.unsupervised import experiments
 
 df = load_wine(return_X_y=False)
 X, y = df["data"], df["target"]
@@ -54,7 +55,7 @@
     tabpfn_reg=reg,
 )
 
-exp_synthetic = unsupervised.experiments.GenerateSyntheticDataExperiment(
+exp_synthetic = experiments.GenerateSyntheticDataExperiment(
     task_type="unsupervised",
 )
 

pyproject.toml

@@ -56,10 +56,9 @@ source = "https://github.com/PriorLabs/tabpfn-extensions"
 
 [project.optional-dependencies]
 interpretability = [
-    # 1.1.0 is the lowest release where `imputer="baseline"` + `index="SV"` +
-    # `max_order=1` (the default config used by get_tabpfn_imputation_explainer)
-    # works without raising.
-    "shapiq>=1.1.0",
+    # 1.2.0 is the first release with shapiq.TabPFNExplainer (get_tabpfn_explainer)
+    # and class_index on TabularExplainer.
+    "shapiq>=1.2.0",
     "seaborn>=0.12.2",
 ]
 post_hoc_ensembles = [
@@ -101,6 +100,15 @@ dev = [
     "build>=1.3.0",
     "twine>=6.2.0",
 ]
+# Extra deps needed only to *run* the example scripts (see examples/), used by the
+# example-file CI job. Deliberately not a published extra: the library itself does
+# not need these, only the example demos do, and the examples ship in the repo (not
+# the wheel), so anyone running them works from a clone where this group is available.
+examples = [
+    # Only used by interpretability/shap_example.py for plotting.
+    # Floor is 0.46.0 (0.41.0 pins an unbuildable numba).
+    "shap>=0.46.0",
+]
 
 [tool.uv]
 exclude-newer = "7 days"
@@ -114,6 +122,9 @@ log_cli = false
 log_level = "DEBUG"
 xfail_strict = true
 addopts = "--durations=10 -vv"
+markers = [
+    "example: runs an example script from examples/ (deselect with -m 'not example')",
+]
 
 # https://github.com/astral-sh/ruff
 [tool.ruff]

tests/conftest.py

@@ -121,6 +121,26 @@ def pytest_addoption(parser):
         default="all",
         help="Specify which TabPFN backend to use: 'tabpfn', 'tabpfn_client', or 'all' (use both if available)",
     )
+    parser.addoption(
+        "--example-timeout",
+        action="store",
+        type=int,
+        default=120,
+        help="Per-example timeout in seconds for normal examples (see tests/test_examples.py)",
+    )
+    parser.addoption(
+        "--example-long-timeout",
+        action="store",
+        type=int,
+        default=120,
+        help="Per-example timeout in seconds for long-running examples (a timeout never fails these)",
+    )
+    parser.addoption(
+        "--example-strict",
+        action="store_true",
+        default=False,
+        help="Treat a normal example that doesn't finish within the timeout as a failure (weekly full run)",
+    )
 
 
 # Define markers for tests