Move globals from exploratory scripts to config.yaml.

Author: njhenry

config.yaml

@@ -0,0 +1,65 @@
+# Shared settings
+osm_keys:
+  - amenity
+  - shop
+  - healthcare
+  - leisure
+end_date: 2025-12-31
+
+# Settings for exploratory/osm_data/download.py
+download:
+  timeout: 1000
+  bbox:
+    ymin: 47.41
+    xmin: -122.48
+    ymax: 47.79
+    xmax: -122.16
+  start_date: 2016-01-01
+  date_interval_days: 7
+
+# Settings for exploratory/osm_data/format_tabular.py
+format_tabular:
+  tag_key: shop
+
+# Settings for exploratory/osm_data/data_viz.py
+data_viz:
+  tag_key: name
+  top_n_types: 10
+  timestamp_cols:
+    - obs_timestamp
+    - last_obs_timestamp
+    - last_tag_timestamp
+
+# Settings for exploratory/models/pytorch_simple.py
+pytorch_simple:
+  tag_key: name
+  group_key: null
+  group_values:
+    - park
+  n_draws: 250
+  save_full_model: false
+
+# Directory definitions (used with cfg.get_dir_path())
+directories:
+  osm_download:
+    versioned: false
+    path: ~/data/osm_example_data
+    files:
+      osm_elements: osm_elements.csv
+      osm_versions: osm_versions.csv
+      osm_changes: osm_changes.csv
+      osm_failed_elements: osm_failed_elements.csv
+  osm_data:
+    versioned: true
+    path: ~/data/openpois
+    files:
+      osm_changes: osm_changes.csv
+      osm_versions: osm_versions.csv
+  model_output:
+    versioned: true
+    path: ~/data/openpois
+    files: {}
+
+versions:
+  osm_data: "20260129"
+  model_output: "20260212"

exploratory/models/pytorch_simple.py

@@ -11,20 +11,21 @@
 import pandas as pd
 import torch
 from pathlib import Path
+from config_versioned import Config
 
 from openpois.models.model_fitter import ModelFitter
 from openpois.models.setup import pytorch_setup, prepare_data_for_model
 
 # Globals
-DATA_VERSION = "20260129"
-MODEL_VERSION = "20260212"
-DATA_DIR = Path("~/data/openpois").expanduser() / DATA_VERSION
-MODEL_DIR = Path("~/data/openpois").expanduser() / MODEL_VERSION
-TAG_KEY = "name"
-GROUP_KEY = None
-GROUP_VALUES = ["park"]
-N_DRAWS = 250
-SAVE_FULL_MODEL = False
+_cfg = Config("~/repos/openpois/config.yaml")
+
+DATA_DIR = _cfg.get_dir_path("osm_data")
+MODEL_DIR = _cfg.get_dir_path("model_output")
+TAG_KEY = _cfg.get("pytorch_simple", "tag_key")
+GROUP_KEY = _cfg.get("pytorch_simple", "group_key")
+GROUP_VALUES = _cfg.get("pytorch_simple", "group_values")
+N_DRAWS = _cfg.get("pytorch_simple", "n_draws")
+SAVE_FULL_MODEL = _cfg.get("pytorch_simple", "save_full_model")
 
 
 if __name__ == "__main__":

exploratory/osm_data/data_viz.py

@@ -9,6 +9,7 @@
 import numpy as np
 import pandas as pd
 from pathlib import Path
+from config_versioned import Config
 
 import matplotlib
 matplotlib.use("Agg")
@@ -20,12 +21,13 @@
 # Configuration constants
 # ----------------------------------------------------------------------------------------
 
-DATA_VERSION = "20260129"
-SAVE_DIR = Path("~/data/openpois").expanduser() / DATA_VERSION
+_cfg = Config("~/repos/openpois/config.yaml")
+
+SAVE_DIR = _cfg.get_dir_path("osm_data")
 VIZ_DIR = SAVE_DIR / "viz"
-OSM_KEYS = ["amenity", "shop", "healthcare", "leisure"]
-TAG_KEY = "name"
-END_DATE = pd.Timestamp('2025-12-31', tz = 'UTC')
+OSM_KEYS = _cfg.get("osm_keys")
+TAG_KEY = _cfg.get("data_viz", "tag_key")
+END_DATE = pd.Timestamp(_cfg.get("end_date"), tz='UTC')
 
 max_days = 365 * 10
 VIZ_DIR.mkdir(parents = True, exist_ok = True)
@@ -60,7 +62,7 @@ def fig_save(
     # Read observations
     # Drop the first observation for each POI (when the POI was first added) - the last
     #   observation timestamp will be missing for these rows
-    timestamp_cols = ['obs_timestamp', 'last_obs_timestamp', 'last_tag_timestamp']
+    timestamp_cols = _cfg.get("data_viz", "timestamp_cols")
     observations_df = (pd.read_csv(SAVE_DIR / f"osm_observations_{TAG_KEY}.csv")
         .dropna(subset = timestamp_cols)
     )
@@ -110,7 +112,7 @@ def fig_save(
     fig_save(fig, stub = f"osm_changes_{TAG_KEY}_all")
 
     # Create multi-panel plots for the top tags in each OSM category
-    TOP_N_TYPES = 10
+    TOP_N_TYPES = _cfg.get("data_viz", "top_n_types")
     for subtype in OSM_KEYS:
         fig = change_multiplot_create(
             observations = to_plot_df,

exploratory/osm_data/download.py

@@ -7,7 +7,7 @@
 3. Save the results to CSV files.
 """
 import datetime
-import os
+from config_versioned import Config
 from openpois.osm.download import (
     build_date_range,
     collect_element_ids,
@@ -18,15 +18,17 @@
 # Configuration constants
 # -----------------------------------------------------------------------------
 
-TIMEOUT = 1000
-BBOX = {"ymin": 47.41, "xmin": -122.48, "ymax": 47.79, "xmax": -122.16}
-START_DATE = datetime.datetime(2016, 1, 1)  # Earliest option is September 13, 2012
-END_DATE = datetime.datetime(2025, 12, 31)  # Latest
-DATE_INTERVAL = datetime.timedelta(days = 7)
-OSM_KEYS = ["amenity", "shop", "healthcare", "leisure"]
-SAVE_DIR = "~/data/osm_example_data"
+_cfg = Config("~/repos/openpois/config.yaml")
 
-os.makedirs(SAVE_DIR, exist_ok = True)
+TIMEOUT = _cfg.get("download", "timeout")
+BBOX = _cfg.get("download", "bbox")
+START_DATE = datetime.datetime.combine(_cfg.get("download", "start_date"), datetime.time.min)  # Earliest option is September 13, 2012
+END_DATE = datetime.datetime.combine(_cfg.get("end_date"), datetime.time.min)  # Latest
+DATE_INTERVAL = datetime.timedelta(days=_cfg.get("download", "date_interval_days"))
+OSM_KEYS = _cfg.get("osm_keys")
+SAVE_DIR = _cfg.get_dir_path("osm_download")
+
+SAVE_DIR.mkdir(parents=True, exist_ok=True)
 
 
 # -----------------------------------------------------------------------------
@@ -50,12 +52,11 @@
     )
 
     # Save elements table
-    elements_table.to_csv(
-        os.path.join(SAVE_DIR, "osm_elements.csv"),
-        index = False,
+    _cfg.write(elements_table, "osm_download", "osm_elements")
+    print(
+        f"Succeeded on {len(succeed_dates)} dates, "
+        f"failed on {len(failed_dates)}"
     )
-    print(f"Saved {len(elements_table)} elements to osm_elements.csv")
-    print(f"Succeeded on {len(succeed_dates)} dates, failed on {len(failed_dates)}")
 
     # Download element histories
     versions_df, changes_df, failed_rows = download_element_histories(
@@ -65,19 +66,8 @@
     )
 
     # Save results
-    versions_df.to_csv(
-        os.path.join(SAVE_DIR, "osm_versions.csv"),
-        index=False,
-    )
-    changes_df.to_csv(
-        os.path.join(SAVE_DIR, "osm_changes.csv"),
-        index = False,
-    )
+    _cfg.write(versions_df, "osm_download", "osm_versions")
+    _cfg.write(changes_df, "osm_download", "osm_changes")
     print(f"Saved {len(versions_df)} versions and {len(changes_df)} changes")
-
-    print(f"Failed on {len(failed_rows)} elements")
-    failed_elements = elements_table.iloc[failed_rows, :]
-    failed_elements.to_csv(
-        os.path.join(SAVE_DIR, "osm_failed_elements.csv"),
-        index = False,
-    )
+    _cfg.write(failed_rows, "osm_download", "osm_failed_elements")
+    print(f"Failed on {len(failed_rows)} elements")

exploratory/osm_data/format_tabular.py

@@ -9,7 +9,7 @@
 """
 
 import pandas as pd
-from pathlib import Path
+from config_versioned import Config
 
 from openpois.osm.format_observations import format_observations
 
@@ -18,10 +18,11 @@
 # Configuration constants
 # ----------------------------------------------------------------------------------------
 
-DATA_VERSION = "20260129"
-SAVE_DIR = Path("~/data/openpois").expanduser() / DATA_VERSION
-OSM_KEYS = ["amenity", "shop", "healthcare", "leisure"]
-TAG_KEY = "shop"
+_cfg = Config("~/repos/openpois/config.yaml")
+
+SAVE_DIR = _cfg.get_dir_path("osm_data")
+OSM_KEYS = _cfg.get("osm_keys")
+TAG_KEY = _cfg.get("format_tabular", "tag_key")
 
 
 # ----------------------------------------------------------------------------------------