Add new dimension to conflation: change detection for deleted points.

Author: njhenry

config.yaml

@@ -1,11 +1,15 @@
 # Versioned directories (used with config.get_dir_path())
 versions:
-  osm_data: "20260416"
+  osm_data: "20260515"
   model_output: "20260422_by_shared_label"
   snapshot_osm: "20260417"
   snapshot_overture: "20260423"
   conflation: "20260423"
   source_coop: "2026-04-23-v0"   # Source Cooperative upload folder (YYYY-MM-DD-v<IDX>); bump v<IDX> only for same-day re-uploads
+  # Ghost POI dataset reconstructed from OSM history (one row per
+  # detected previous-state event). Pinned to the same value as
+  # ``osm_data`` since it is derived from the same history parquets.
+  ghost_osm: "20260515"
 
 # Settings for downloading data
 download:
@@ -188,6 +192,11 @@ directories:
       partitioned: conflated_partitioned
       pmtiles: conflated.pmtiles
       summary_by_label: summary_by_label.csv
+  ghost_osm:
+    versioned: true
+    path: ~/data/openpois/ghost_osm
+    files:
+      ghosts: ghosts.parquet
   testing:
     versioned: false
     path: ~/data/openpois/testing
@@ -222,6 +231,23 @@ conflation:
     ymin: 47.50
     xmax: -122.25
     ymax: 47.70
+  # Change-detection feature: use OSM history to penalize Overture POIs
+  # that co-locate with a "ghost" — a previous state of an OSM element
+  # (primary-tag deletion, lifecycle-prefix addition, or substantial
+  # rename). Disabled by default for clean A/B testing.
+  change_detection:
+    enabled: false
+    # Minimum composite score for an Overture × ghost shadow match.
+    # Same scale as the main matcher's min_match_score.
+    min_shadow_match_score: 0.50
+    # rapidfuzz.fuzz.token_set_ratio threshold below which an OSM name
+    # change is considered a "substantial rename" rather than a typo
+    # fix. Range 0-100. Lower = stricter (fewer events emitted).
+    name_change_similarity_threshold: 50
+    # Fallback delta for ghosts whose shared_label isn't in the fitted
+    # model's per-group params. Equals sigmoid(logit_delta_0) for the
+    # current 20260422_by_shared_label fit (logit_delta_0 = -2.72).
+    default_delta: 0.062
 
 # Settings for publishing snapshots to Source Cooperative
 # (https://source.coop/henryspatialanalysis/openpois). Source Coop is

scripts/conflation/apply_change_detection.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+"""
+Apply the change-detection penalty to a baseline conflated dataset.
+
+Reads:
+  - the baseline ``conflated.parquet`` (no change detection)
+  - ``ghosts.parquet`` (from ``scripts/conflation/build_ghosts.py``)
+  - ``fitted_params.csv`` for the active ``model_output`` version
+
+Writes a new conflated parquet (suffix ``_cd`` by default) whose
+unmatched-Overture rows have had ``conf_mean`` re-weighted by
+``δ_group`` for any spatial+name+taxonomy match against a ghost. Audit
+columns are appended (``shadow_*`` + ``original_conf_mean``) so the
+demoted rows can be inspected by hand.
+
+Usage:
+    python scripts/conflation/apply_change_detection.py \
+        --baseline-suffix=baseline --output-suffix=cd [--test]
+
+Both ``--baseline-suffix`` and ``--output-suffix`` are inserted into
+the conflated filename before ``.parquet`` (e.g. ``conflated_cd.parquet``).
+"""
+from __future__ import annotations
+
+import argparse
+import time
+from pathlib import Path
+
+from config_versioned import Config
+
+from openpois.conflation.change_detection import apply_shadow_match
+
+
+def _suffixed_path(base_path: Path, suffix: str | None) -> Path:
+    """Insert ``suffix`` before the parquet extension."""
+    if not suffix:
+        return base_path
+    return base_path.with_name(
+        f"{base_path.stem}_{suffix}{base_path.suffix}"
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description = (
+            "Apply change-detection penalty to a baseline conflated "
+            "dataset using OSM-history-derived ghost POIs."
+        )
+    )
+    parser.add_argument(
+        "--baseline-suffix",
+        default = "baseline",
+        help = (
+            "Suffix inserted into the input parquet filename "
+            "(default: 'baseline' → conflated_baseline.parquet). "
+            "Pass an empty string to read conflated.parquet directly."
+        ),
+    )
+    parser.add_argument(
+        "--output-suffix",
+        default = "cd",
+        help = (
+            "Suffix inserted into the output parquet filename "
+            "(default: 'cd' → conflated_cd.parquet)."
+        ),
+    )
+    parser.add_argument(
+        "--test",
+        action = "store_true",
+        help = (
+            "Restrict ghosts to the configured conflation.test_bbox. "
+            "Use when the baseline was produced with --test."
+        ),
+    )
+    args = parser.parse_args()
+
+    config = Config("~/repos/openpois/config.yaml")
+
+    conflated_base = config.get_file_path("conflation", "conflated")
+    baseline_path = _suffixed_path(
+        conflated_base, args.baseline_suffix,
+    )
+    output_path = _suffixed_path(
+        conflated_base, args.output_suffix,
+    )
+    ghosts_path = config.get_file_path("ghost_osm", "ghosts")
+
+    model_dir = Path(config.get_dir_path("model_output"))
+    fitted_params_path = model_dir / config.get(
+        "directories", "model_output", "files", "fitted_params",
+    )
+
+    cd_cfg = config.get("conflation", "change_detection")
+    min_match_score = float(cd_cfg["min_shadow_match_score"])
+    default_delta = float(cd_cfg["default_delta"])
+
+    max_radius_m = float(config.get("conflation", "max_radius_m"))
+    default_radius_m = float(
+        config.get("conflation", "default_radius_m")
+    )
+    distance_weight = float(config.get("conflation", "distance_weight"))
+    name_weight = float(config.get("conflation", "name_weight"))
+    type_weight = float(config.get("conflation", "type_weight"))
+    identifier_weight = float(
+        config.get("conflation", "identifier_weight")
+    )
+
+    test_bbox = (
+        config.get("conflation", "test_bbox") if args.test else None
+    )
+
+    print(f"Baseline: {baseline_path}")
+    print(f"Ghosts:   {ghosts_path}")
+    print(f"Fitted params: {fitted_params_path}")
+    print(f"Output:   {output_path}")
+    print(
+        f"min_match_score={min_match_score} "
+        f"max_radius_m={max_radius_m} "
+        f"default_delta={default_delta}"
+    )
+    if args.test:
+        print(f"Test bbox: {test_bbox}")
+
+    t0 = time.time()
+    summary = apply_shadow_match(
+        conflated_path = baseline_path,
+        ghosts_path = ghosts_path,
+        fitted_params_path = fitted_params_path,
+        output_path = output_path,
+        min_match_score = min_match_score,
+        max_radius_m = max_radius_m,
+        default_radius_m = default_radius_m,
+        distance_weight = distance_weight,
+        name_weight = name_weight,
+        type_weight = type_weight,
+        identifier_weight = identifier_weight,
+        default_delta = default_delta,
+        test_bbox = test_bbox,
+    )
+    elapsed = time.time() - t0
+
+    print(f"\nApplied change-detection in {elapsed:.0f}s")
+    print(f"  Total conflated rows:       {summary['n_total']:,}")
+    print(
+        f"  Unmatched Overture rows:    "
+        f"{summary['n_unmatched_overture']:,}"
+    )
+    print(f"  Ghosts considered:          {summary['n_ghosts']:,}")
+    print(
+        f"  Shadow matches:             "
+        f"{summary['n_shadow_matches']:,}"
+    )
+    print(
+        f"  Mean penalty factor (Δ/old): "
+        f"{summary['mean_penalty_factor']:.4f}"
+    )
+    print(f"  Output: {output_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/conflation/build_ghosts.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+"""
+Build the ghost-OSM POI dataset from OSM history.
+
+A ghost is a previous state of an OSM node that we believe no longer
+reflects ground truth (primary tag deleted, lifecycle prefix added, or
+substantial rename). The output Parquet feeds the change-detection
+pass in ``scripts/conflation/conflate.py``.
+
+Config keys used (config.yaml):
+    versions.osm_data, versions.ghost_osm        — pinned together
+    directories.osm_data.osm_versions
+    directories.osm_data.osm_changes
+    directories.ghost_osm.ghosts
+    download.osm.filter_keys                     — POI tag keys
+    conflation.change_detection.name_change_similarity_threshold
+
+Usage:
+    python scripts/conflation/build_ghosts.py
+"""
+from __future__ import annotations
+
+import time
+
+from config_versioned import Config
+
+from openpois.conflation.ghost_osm import build_ghosts
+
+
+def main() -> None:
+    config = Config("~/repos/openpois/config.yaml")
+
+    versions_path = config.get_file_path("osm_data", "osm_versions")
+    changes_path = config.get_file_path("osm_data", "osm_changes")
+    output_path = config.get_file_path("ghost_osm", "ghosts")
+
+    filter_keys = config.get("download", "osm", "filter_keys")
+    name_threshold = float(
+        config.get(
+            "conflation", "change_detection",
+            "name_change_similarity_threshold",
+        )
+    )
+
+    print(f"Versions path: {versions_path}")
+    print(f"Changes path:  {changes_path}")
+    print(f"Output path:   {output_path}")
+    print(f"POI keys:      {filter_keys}")
+    print(f"Name similarity threshold: {name_threshold}")
+
+    t0 = time.time()
+    ghosts = build_ghosts(
+        versions_path = versions_path,
+        changes_path = changes_path,
+        poi_keys = filter_keys,
+        name_change_similarity_threshold = name_threshold,
+    )
+    elapsed = time.time() - t0
+    print(f"\nBuilt {len(ghosts):,} ghosts in {elapsed:.0f}s")
+
+    if len(ghosts):
+        event_counts = (
+            ghosts["event_type"].value_counts().to_dict()
+        )
+        print("Event-type breakdown:")
+        for et, n in sorted(event_counts.items(), key = lambda kv: -kv[1]):
+            print(f"  {et}: {n:,}")
+
+        sl_total = int((ghosts["shared_label"] != "").sum())
+        print(
+            f"shared_label assigned: {sl_total:,}/{len(ghosts):,} "
+            f"({100 * sl_total / max(len(ghosts), 1):.1f}%)"
+        )
+
+    output_path.parent.mkdir(parents = True, exist_ok = True)
+    ghosts.to_parquet(output_path, compression = "zstd")
+    print(f"\nWrote {output_path}")
+    config.write_self("ghost_osm")
+
+
+if __name__ == "__main__":
+    main()

scripts/conflation/conflate.py

@@ -274,7 +274,23 @@ def _load_gdf(
             "a debug/baseline option."
         ),
     )
+    parser.add_argument(
+        "--output-suffix",
+        default = "",
+        help = (
+            "If set, inserts ``_<suffix>`` before .parquet in the "
+            "output filename (e.g. --output-suffix=baseline writes "
+            "conflated_baseline.parquet). Used by the change-detection "
+            "A/B testing workflow."
+        ),
+    )
     args = parser.parse_args()
+
+    if args.output_suffix:
+        OUTPUT_PATH = OUTPUT_PATH.with_name(
+            f"{OUTPUT_PATH.stem}_{args.output_suffix}"
+            f"{OUTPUT_PATH.suffix}"
+        )
     t0 = time.time()
 
     test_bbox = TEST_BBOX if args.test else None