Merge pull request #791 from OpenHistoricalMap/staging

Production release - ohmx update , planet-dump sequence number and planet-stats deploy

Author: Rub21

.github/workflows/frontend-planet-stats.yaml

@@ -0,0 +1,62 @@
+name: Build and Publish OHM Stats Dashboard
+
+on:
+  push:
+    branches:
+      - "main"
+      - "staging"
+  workflow_dispatch:
+
+env:
+  DASHBOARD_REPO: danvk/ohm
+  DASHBOARD_GITSHA: 58c9bca6894a76158242b2025bfcba29ada6ddd8
+  CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+
+jobs:
+  build:
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.13.0
+        with:
+          access_token: ${{ github.token }}
+
+      - name: Set environment variables - Staging
+        if: github.ref == 'refs/heads/staging'
+        uses: allenevans/set-env@v4.0.0
+        with:
+          CLOUDFLARE_PROJECT: ohm-planet-stats-dash-staging
+          DASHBOARD_SERVER: https://s3.amazonaws.com/planet-staging.openhistoricalmap.org/planet-stats
+
+      - name: Set environment variables - Production
+        if: github.ref == 'refs/heads/main'
+        uses: allenevans/set-env@v4.0.0
+        with:
+          CLOUDFLARE_PROJECT: ohm-planet-stats-dash-production
+          DASHBOARD_SERVER: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet-stats
+
+      - name: Checkout dashboard repo
+        uses: actions/checkout@v6
+        with:
+          repository: ${{ env.DASHBOARD_REPO }}
+          ref: ${{ env.DASHBOARD_GITSHA }}
+          path: ohm
+
+      - name: Prepare publish dir
+        run: |
+          mkdir -p site/dashboard
+          cp ohm/index.html site/
+          cp -r ohm/dashboard/. site/dashboard/
+          sed -i "s|export const SERVER = .*;|export const SERVER = \"${DASHBOARD_SERVER}\";|" site/dashboard/metrics.js
+          echo "SERVER set to:"; grep "export const SERVER" site/dashboard/metrics.js
+          ls -la site site/dashboard
+
+      - name: Deploy to Cloudflare Pages
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.DEPLOY_CLOUDFLARE_PAGES_API_TOKEN }}
+          accountId: ${{ env.CLOUDFLARE_ACCOUNT_ID }}
+          command: pages deploy site --project-name=${{ env.CLOUDFLARE_PROJECT }} --branch=${{ github.ref_name }}

images/ohmx-adiff-builder/Dockerfile

@@ -1,39 +1,47 @@
-FROM ghcr.io/bdon/osmexpress:latest
+FROM ghcr.io/bdon/osmexpress:045a515132e91a3679ce3df27329937c9b8b221a
 
 ENV PATH="/root/.cargo/bin:${PATH}"
 
-RUN apk add --no-cache python3 py3-pip git bash xmlstarlet curl aws-cli py3-lxml
+RUN apk add --no-cache python3 py3-pip git bash curl aws-cli py3-lxml make libxml2 libgcc
 
-RUN apk add --no-cache --virtual .build-deps build-base patch python3-dev make cmake zlib-dev expat-dev bzip2-dev boost-dev rust cargo && \
+RUN apk add --no-cache --virtual .build-deps \
+      build-base patch python3-dev cmake zlib-dev expat-dev bzip2-dev boost-dev \
+      capnproto-dev clang-dev libxml2-dev pkgconfig && \
     \
     pip install --break-system-packages osmx requests osmium && \
     \
-    echo "--- Clonando y modificando osm-cli para OpenHistoricalMap ---" && \
+    # Alpine's rust/cargo lag behind crate MSRVs (osmpbf/home now need rustc 1.88+),
+    # so pull a current stable toolchain via rustup instead of the apk packages.
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
+      | sh -s -- -y --profile minimal --default-toolchain stable && \
+    export LIBCLANG_PATH=/usr/lib && \
+    # musl defaults to static linking; bindgen's build script must dlopen
+    # libclang, which a static binary can't do. Link dynamically instead.
+    export RUSTFLAGS="-C target-feature=-crt-static" && \
+    \
+    echo "--- Build osm-cli (jake-low) with OpenHistoricalMap replication URLs ---" && \
     git clone https://github.com/jake-low/osm-cli.git /tmp/osm-cli && \
     cd /tmp/osm-cli && \
-    echo "Cambiando URL en src/replication.rs" && \
     sed -i 's|https://planet.openstreetmap.org/replication/minute|https://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute|g' src/replication.rs && \
-    echo "Cambiando URL en src/main.rs" && \
     sed -i 's|https://www.openstreetmap.org|https://www.openhistoricalmap.org|g' src/main.rs && \
-    echo "Compilando osm-cli desde la fuente modificada..." && \
-    cargo install --path . && \
-    cd / && \
-    rm -rf /tmp/osm-cli && \
+    cargo install --root /usr/local --path . && \
+    cd / && rm -rf /tmp/osm-cli && \
+    \
+    echo "--- Build osmx-rs (jake-low) for augmented-diff --split ---" && \
+    cargo install --root /usr/local --git https://github.com/jake-low/osmx-rs.git osmx-rs && \
     \
-    apk del .build-deps && \
-    apk add make
+    # Binaries now live in /usr/local/bin, so the whole toolchain can go.
+    rustup self uninstall -y && \
+    apk del .build-deps
 
 RUN curl https://mise.jdx.dev/install.sh | sh
 ENV PATH="/root/.local/bin:${PATH}"
 
 WORKDIR /app
 
-# RUN echo "Update repo 08/08/2025"
-
-RUN git clone https://github.com/OpenHistoricalMap/osmx-adiff-builder.git /app
-
-COPY *.sh .
-COPY *.py .
+# Pipeline scripts come from OSMCha/osmx-adiff-builder (Rust path), customized
+# for OpenHistoricalMap and copied in directly instead of cloned.
+COPY scripts/*.sh scripts/*.py scripts/*.mk /app/
 
 RUN chmod +x /app/*.sh && chmod +x /app/*.py
 

images/ohmx-adiff-builder/config.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Shared configuration. Sourced by start.sh and process_min_range.sh.
+export WORKDIR=${WORKDIR:-/data}
+
+# Binaries: osmx (C++) for expand/update/query, osmx-rs (Rust) for augmented diffs.
+export OSMX_BIN=${OSMX_BIN:-osmx}
+export OSMX_RS_BIN=${OSMX_RS_BIN:-osmx-rs}
+
+# Database
+export OSMX_DB_DIR=$WORKDIR/db
+export OSMX_DB_PATH=$OSMX_DB_DIR/osmx.db
+export PLANET_FILE_PATH=$WORKDIR/planet.osm.pbf
+
+# Services
+export REPLICATION_URL="${REPLICATION_URL:-https://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute}"
+export API_URL=${API_URL:-https://api.openhistoricalmap.org}
+
+# Working directories
+export SPLIT_ADIFFS_DIR=$WORKDIR/stage-data/split-adiffs   # per-changeset fragments
+export CHANGESET_DIR=$WORKDIR/stage-data/changesets        # merge.mk stamp files
+export BUCKET_DIR=$WORKDIR/stage-data/bucket-data          # merged adiffs ready to upload
+export BAD_CHANGESETS_DIR=$WORKDIR/stage-data/bad_changesets
+
+# Upload target: s3://$AWS_S3_BUCKET/$S3_PREFIX/<changeset>.adiff
+export S3_PREFIX=${S3_PREFIX:-ohm-augmented-diffs/changesets}
+
+# Liveness (watchdog in start.sh + healthcheck.sh)
+export HEARTBEAT_DIR=${HEARTBEAT_DIR:-/tmp/heartbeat}
+export HEARTBEAT_STALE_SECONDS=${HEARTBEAT_STALE_SECONDS:-600}
+
+# Kill one update.sh run if it exceeds this (must be < HEARTBEAT_STALE_SECONDS)
+export UPDATE_TIMEOUT=${UPDATE_TIMEOUT:-540}