Fully working random effects model; update viz for clarity.

Author: njhenry

.claude/CLAUDE.md

@@ -49,6 +49,7 @@ Style: Black (format-on-save in VSCode). Lint: flake8 + pylint, configured in `p
 - [docs/data-sources.md](docs/data-sources.md) — URLs, auth, schema quirks for every source
 - [docs/taxonomy-setup.md](docs/taxonomy-setup.md) — crosswalk CSVs, build_taxonomy.py, frontend sync
 - [docs/data-versioning.md](docs/data-versioning.md) — `versions:` block, path resolution, external references
+- [docs/turnover-model-methodology.md](docs/turnover-model-methodology.md) — statistical derivation of the POI turnover model with ZIE extension
 
 ## Running to-do
 

.claude/skills/model-history-pipeline/SKILL.md

@@ -31,7 +31,7 @@ End-to-end: Geofabrik full-history PBFs → observations table → fitted λ →
    ```bash
    python scripts/models/osm_turnover.py
    ```
-   Writes `fitted_params.csv`, `param_draws.csv`, `predictions.csv`, `fitted_model.pkl` to `{date}_by_shared_label` (the unified random-effects model) or `{date}_constant` (single-rate baseline) under `directories.model_output.path`.
+   Writes `fitted_params.csv`, `param_draws.csv`, `predictions.csv` to `{date}_by_shared_label` (the unified random-effects model) or `{date}_constant` (single-rate baseline) under `directories.model_output.path`.
 
 4. **Apply predictions to the OSM snapshot** → `osm_snapshot_rated.parquet`
    ```bash

.claude/skills/verify-pipeline-run/SKILL.md

@@ -32,7 +32,6 @@ Flag >5% drops. Known regression patterns:
   fitted_params.csv     # λ and σ per group
   param_draws.csv       # uncertainty bounds
   predictions.csv       # predictions per POI
-  fitted_model.pkl      # pickled JAX ModelFitter
 ```
 
 Checks:

config.yaml

@@ -126,7 +126,7 @@ osm_data:
     - last_obs_timestamp
     - last_tag_timestamp
   apply_model:
-    model_stub: '20260416'
+    model_stub: '20260421'
 
 # Settings for scripts/models/osm_turnover.py (JAX turnover model)
 osm_turnover_model:
@@ -180,7 +180,6 @@ directories:
       predictions: predictions.csv
       diagnostics: diagnostics.csv
       inference_data: inference_data.nc
-      fitted_model: fitted_model.pkl
   snapshot_foursquare:
     versioned: true
     path: ~/data/openpois/snapshots/foursquare

docs/workflows.rst

@@ -107,7 +107,7 @@ See :mod:`openpois.osm.format_observations`.
 Fits an empirical Bayes JAX model (constant or random-effects by type)
 estimating the Poisson change rate λ per group via BlackJAX NUTS. Outputs
 ``fitted_params.csv`` and ``predictions.csv`` (and optionally
-``param_draws.csv`` / ``fitted_model.pkl``).
+``param_draws.csv``).
 
 See :mod:`openpois.models.model_fitter`, :mod:`openpois.models.osm_models`,
 and :mod:`openpois.models.setup`.

scripts/models/osm_turnover.py

@@ -38,11 +38,9 @@
     diagnostics.csv     — per-parameter R-hat / bulk-ESS (multi-chain only)
     inference_data.nc   — ArviZ InferenceData (optional, if arviz installed)
     param_draws.csv     — posterior draws (if save_full_model = true)
-    fitted_model.pkl    — pickled ModelFitter (if save_full_model = true)
 """
 
 import argparse
-import pickle
 
 import jax.numpy as jnp
 import numpy as np
@@ -211,7 +209,3 @@ def flatten_param_draws(
             "model_output",
             "param_draws",
         )
-        with open(
-            config.get_file_path("model_output", "fitted_model"), "wb",
-        ) as fh:
-            pickle.dump(fitter, fh)

scripts/osm_data/data_viz.py

@@ -27,9 +27,11 @@
     osm_changes_all.png                     — overall survival curve
     osm_changes_all_preds.png               — overall curve with constant-model
                                               prediction overlay
-    osm_changes_by_shared_label.png         — per-label facet grid
-    osm_changes_shared_label_preds_<label>.png — per-label curves with
-                                                 shared-label model predictions
+    osm_changes_by_shared_label.png         — per-label facet grid (top N)
+    by_type/osm_changes_<label>.png         — per-label curves with
+                                              shared-label model predictions,
+                                              one file per shared_label with a
+                                              fitted prediction
 """
 
 import numpy as np
@@ -68,20 +70,29 @@
 
 
 def fig_save(
-    fig: gg.ggplot, stub: str, width: float = 10, height: float = 6, **kwargs
+    fig: gg.ggplot,
+    stub: str,
+    width: float = 10,
+    height: float = 6,
+    subdir: str | None = None,
+    **kwargs,
 ) -> None:
     """
-    Save a ggplot figure as a PNG file to VIZ_DIR.
+    Save a ggplot figure as a PNG file to VIZ_DIR (or a subdirectory of it).
 
     Args:
         fig: The ggplot figure to save.
         stub: Output filename stem (without extension).
         width: Figure width in inches.
         height: Figure height in inches.
+        subdir: Optional subdirectory under VIZ_DIR. Created if it doesn't
+            exist.
         **kwargs: Additional keyword arguments forwarded to fig.save().
     """
+    out_dir = VIZ_DIR if subdir is None else VIZ_DIR / subdir
+    out_dir.mkdir(parents = True, exist_ok = True)
     fig.save(
-        filename = VIZ_DIR / f"{stub}.png",
+        filename = out_dir / f"{stub}.png",
         width = width,
         height = height,
         units = 'in',
@@ -209,12 +220,9 @@ def get_preds_df(version: str) -> pd.DataFrame | None:
     fig_save(fig = fig, stub = "osm_changes_by_shared_label")
 
     if preds.get('shared_label') is not None:
-        top_n_labels = (
-            to_plot_df["shared_label"].value_counts().head(TOP_N_TYPES).index
-        )
-        pred_groups = preds['shared_label']['group_name'].unique().tolist()
-        keep_preds = list(set(top_n_labels) & set(pred_groups))
-        for pred_label in keep_preds:
+        observed_labels = set(to_plot_df["shared_label"].dropna().unique())
+        pred_groups = set(preds['shared_label']['group_name'].unique())
+        for pred_label in sorted(pred_groups & observed_labels):
             print(f"Plotting shared_label = {pred_label}")
             fig = change_plot_create(
                 observations = to_plot_df.query(
@@ -235,5 +243,6 @@ def get_preds_df(version: str) -> pd.DataFrame | None:
             )
             fig_save(
                 fig,
-                stub = f"osm_changes_shared_label_preds_{pred_label}",
+                stub = f"osm_changes_{pred_label}",
+                subdir = "by_type",
             )

src/openpois/osm/change_plots.py

@@ -166,9 +166,10 @@ def change_plot_create(
         )
         fig = fig + gg.geom_ribbon(
             data = p_renamed,
-            fill = 'red', alpha = 0.25, linetype = 'dashed', color = '#444444'
+            fill = 'darkred', alpha = 0.25, linetype = 'dashed',
+            color = 'darkred',
         ) + gg.geom_line(
-            data = p_renamed, color = '#444444',
+            data = p_renamed, color = 'darkred',
             mapping = gg.aes(x = 'year', y = 'y')
         )
     return fig