Add a mirrored latest/ folder to Source.Coop upload.

Author: njhenry

scripts/publish/upload_to_source_coop.py

@@ -14,6 +14,8 @@
             osm-pmtiles/osm.pmtiles
             conflated-parquet/geohash_prefix=*/part-*.parquet
             conflated-pmtiles/conflated.pmtiles
+        latest/
+            (server-side mirror of the most recently published version)
 
 Credentials
 -----------
@@ -41,6 +43,7 @@
 from openpois.io.credentials import load_source_coop_credentials
 from openpois.io.source_coop import (
     make_client,
+    mirror_prefix,
     public_url,
     upload_bytes,
     upload_directory,
@@ -76,6 +79,13 @@ def parse_args() -> argparse.Namespace:
             "default because these rarely change."
         ),
     )
+    parser.add_argument(
+        "--skip-latest-mirror", action = "store_true",
+        help = (
+            "Skip mirroring the uploaded version to {repo_prefix}/latest/. "
+            "Enabled by default so /latest always tracks the newest release."
+        ),
+    )
     parser.add_argument(
         "--dry-run", action = "store_true",
         help = "Print every remote key that would be written and exit.",
@@ -170,6 +180,30 @@ def main() -> None:
         dry_run = args.dry_run,
     )
 
+    # -------------------------------------------------------------------------
+    # Mirror the published version to {repo_prefix}/latest/
+    # -------------------------------------------------------------------------
+    latest_prefix = f"{repo_prefix}/latest"
+    if not args.skip_latest_mirror:
+        print()
+        print(f"Mirroring {version_prefix}/ → {latest_prefix}/ …")
+        if args.dry_run:
+            print(
+                "  [dry-run] skipping remote listing — copy/delete counts "
+                "will only reflect real remote state during a live run."
+            )
+        summary = mirror_prefix(
+            client = client,
+            bucket = bucket,
+            src_prefix = f"{version_prefix}/",
+            dst_prefix = f"{latest_prefix}/",
+            dry_run = args.dry_run,
+        )
+        print(
+            f"  copied {summary['copied']} object(s), "
+            f"deleted {summary['deleted']} stale object(s)."
+        )
+
     # -------------------------------------------------------------------------
     # Top-level README + LICENSE (opt-in)
     # -------------------------------------------------------------------------
@@ -194,6 +228,8 @@ def main() -> None:
     print()
     print(f"Version landing page: https://source.coop/{repo_prefix}/")
     print(f"Version data root:    {public_url(f'{version_prefix}/')}")
+    if not args.skip_latest_mirror:
+        print(f"Latest data root:     {public_url(f'{latest_prefix}/')}")
 
 
 if __name__ == "__main__":

src/openpois/io/source_coop.py

@@ -143,3 +143,93 @@ def upload_bytes(
 def public_url(key: str) -> str:
     """Compose the public read URL for a given object key."""
     return f"{DEFAULT_READ_HOST}/{key.lstrip('/')}"
+
+
+def list_keys(client, bucket: str, prefix: str) -> list[str]:
+    """Return every object key in ``bucket`` under ``prefix`` (paginated)."""
+    keys: list[str] = []
+    paginator = client.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket = bucket, Prefix = prefix):
+        for obj in page.get("Contents", []) or []:
+            keys.append(obj["Key"])
+    return keys
+
+
+def delete_keys(
+    client,
+    bucket: str,
+    keys: list[str],
+    dry_run: bool = False,
+) -> None:
+    """Delete ``keys`` from ``bucket`` in batches of up to 1000 per request."""
+    if not keys:
+        return
+    prefix = "[dry-run] " if dry_run else ""
+    print(f"{prefix}deleting {len(keys)} object(s) from s3://{bucket}/")
+    if dry_run:
+        return
+    for i in range(0, len(keys), 1000):
+        chunk = keys[i : i + 1000]
+        client.delete_objects(
+            Bucket = bucket,
+            Delete = {
+                "Objects": [{"Key": k} for k in chunk],
+                "Quiet": True,
+            },
+        )
+
+
+def mirror_prefix(
+    client,
+    bucket: str,
+    src_prefix: str,
+    dst_prefix: str,
+    dry_run: bool = False,
+) -> dict:
+    """Server-side copy every object under ``src_prefix`` to ``dst_prefix``.
+
+    Any objects currently under ``dst_prefix`` whose relative path is not
+    present in ``src_prefix`` are deleted first, so ``dst_prefix`` becomes a
+    faithful mirror of ``src_prefix``. Returns a summary dict with the
+    counts and the resolved src/dst prefixes.
+    """
+    src_prefix = src_prefix if src_prefix.endswith("/") else src_prefix + "/"
+    dst_prefix = dst_prefix if dst_prefix.endswith("/") else dst_prefix + "/"
+
+    # Without a live client we can't enumerate either side; report zeros.
+    if client is None:
+        src_keys: list[str] = []
+        dst_keys: list[str] = []
+    else:
+        src_keys = list_keys(client, bucket, src_prefix)
+        dst_keys = list_keys(client, bucket, dst_prefix)
+
+    src_relatives = {k[len(src_prefix) :] for k in src_keys}
+    stale = [k for k in dst_keys if k[len(dst_prefix) :] not in src_relatives]
+
+    delete_keys(client, bucket, stale, dry_run = dry_run)
+
+    iterator = (
+        tqdm(src_keys, desc = f"↪ {dst_prefix}", unit = "obj")
+        if src_keys else src_keys
+    )
+    for src_key in iterator:
+        rel = src_key[len(src_prefix) :]
+        dst_key = f"{dst_prefix}{rel}"
+        if dry_run:
+            print(f"[dry-run] copy s3://{bucket}/{src_key} → s3://{bucket}/{dst_key}")
+            continue
+        client.copy_object(
+            Bucket = bucket,
+            Key = dst_key,
+            CopySource = {"Bucket": bucket, "Key": src_key},
+            ACL = "bucket-owner-full-control",
+            MetadataDirective = "COPY",
+        )
+
+    return {
+        "src_prefix": src_prefix,
+        "dst_prefix": dst_prefix,
+        "copied": len(src_keys),
+        "deleted": len(stale),
+    }

src/openpois/publish/templates/top_readme.md

@@ -21,6 +21,10 @@ Each refresh writes a new versioned folder. Inside every version:
 └── conflated-pmtiles/conflated.pmtiles
 ```
 
+`latest/` is a server-side mirror of the most recently published version —
+use it for live demos and tutorials, and pin a dated folder
+(e.g. `2026-04-23-v0/`) when you need a stable, reproducible reference.
+
 Browse all versions at
 <https://source.coop/henryspatialanalysis/openpois>.
 
@@ -54,7 +58,7 @@ Read a specific version directly from Source Cooperative (no authentication):
 import pyarrow.dataset as ds
 
 BASE = "https://data.source.coop/henryspatialanalysis/openpois"
-VERSION = "2026-04-23-v0"   # replace with the latest version folder
+VERSION = "latest"   # or pin a specific dated folder, e.g. "2026-04-23-v0"
 
 pois = ds.dataset(
     f"{BASE}/{VERSION}/conflated-parquet/",
@@ -71,7 +75,7 @@ print(f"{pois.count_rows():,} POIs")
 import duckdb
 
 BASE = "https://data.source.coop/henryspatialanalysis/openpois"
-VERSION = "2026-04-23-v0"
+VERSION = "latest"   # or pin a specific dated folder, e.g. "2026-04-23-v0"
 
 con = duckdb.connect()
 con.execute("INSTALL httpfs; LOAD httpfs;")
@@ -92,7 +96,7 @@ print(df)
 import geopandas as gpd
 
 BASE = "https://data.source.coop/henryspatialanalysis/openpois"
-VERSION = "2026-04-23-v0"
+VERSION = "latest"   # or pin a specific dated folder, e.g. "2026-04-23-v0"
 
 # geohash_prefix=9q is roughly the US West coast
 gdf = gpd.read_parquet(
@@ -123,7 +127,7 @@ PMTiles are authored at zoom 14, so zoom in past z14 to see points.
 <script src="https://unpkg.com/pmtiles@3/dist/pmtiles.js"></script>
 <script>
   const BASE = "https://data.source.coop/henryspatialanalysis/openpois";
-  const VERSION = "2026-04-23-v0";
+  const VERSION = "latest";   // or pin a specific dated folder, e.g. "2026-04-23-v0"
 
   const protocol = new pmtiles.Protocol();
   maplibregl.addProtocol("pmtiles", protocol.tile);