Update ohmx and planet-dump and backup (#789)

* Update ohmx

* Update configs

* Update api endpoint

* Set a images id with gitsha for osmexpress

* Fix ohmx-adiff-builder build: rustc 1.88 via rustup, capnp from capnproto-dev

* Update rustc 1.87→1.88

* Fix dependencies for the ohmx continaer

* Update scripts and values

* Read sequence number from the pbf file

* Reduce logs files for ohmx merge process

* Update osm-seed version

* Disable data processors for staging

Author: Rub21

images/ohmx-adiff-builder/Dockerfile

@@ -1,39 +1,47 @@
-FROM ghcr.io/bdon/osmexpress:latest
+FROM ghcr.io/bdon/osmexpress:045a515132e91a3679ce3df27329937c9b8b221a
 
 ENV PATH="/root/.cargo/bin:${PATH}"
 
-RUN apk add --no-cache python3 py3-pip git bash xmlstarlet curl aws-cli py3-lxml
+RUN apk add --no-cache python3 py3-pip git bash curl aws-cli py3-lxml make libxml2 libgcc
 
-RUN apk add --no-cache --virtual .build-deps build-base patch python3-dev make cmake zlib-dev expat-dev bzip2-dev boost-dev rust cargo && \
+RUN apk add --no-cache --virtual .build-deps \
+      build-base patch python3-dev cmake zlib-dev expat-dev bzip2-dev boost-dev \
+      capnproto-dev clang-dev libxml2-dev pkgconfig && \
     \
     pip install --break-system-packages osmx requests osmium && \
     \
-    echo "--- Clonando y modificando osm-cli para OpenHistoricalMap ---" && \
+    # Alpine's rust/cargo lag behind crate MSRVs (osmpbf/home now need rustc 1.88+),
+    # so pull a current stable toolchain via rustup instead of the apk packages.
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
+      | sh -s -- -y --profile minimal --default-toolchain stable && \
+    export LIBCLANG_PATH=/usr/lib && \
+    # musl defaults to static linking; bindgen's build script must dlopen
+    # libclang, which a static binary can't do. Link dynamically instead.
+    export RUSTFLAGS="-C target-feature=-crt-static" && \
+    \
+    echo "--- Build osm-cli (jake-low) with OpenHistoricalMap replication URLs ---" && \
     git clone https://github.com/jake-low/osm-cli.git /tmp/osm-cli && \
     cd /tmp/osm-cli && \
-    echo "Cambiando URL en src/replication.rs" && \
     sed -i 's|https://planet.openstreetmap.org/replication/minute|https://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute|g' src/replication.rs && \
-    echo "Cambiando URL en src/main.rs" && \
     sed -i 's|https://www.openstreetmap.org|https://www.openhistoricalmap.org|g' src/main.rs && \
-    echo "Compilando osm-cli desde la fuente modificada..." && \
-    cargo install --path . && \
-    cd / && \
-    rm -rf /tmp/osm-cli && \
+    cargo install --root /usr/local --path . && \
+    cd / && rm -rf /tmp/osm-cli && \
+    \
+    echo "--- Build osmx-rs (jake-low) for augmented-diff --split ---" && \
+    cargo install --root /usr/local --git https://github.com/jake-low/osmx-rs.git osmx-rs && \
     \
-    apk del .build-deps && \
-    apk add make
+    # Binaries now live in /usr/local/bin, so the whole toolchain can go.
+    rustup self uninstall -y && \
+    apk del .build-deps
 
 RUN curl https://mise.jdx.dev/install.sh | sh
 ENV PATH="/root/.local/bin:${PATH}"
 
 WORKDIR /app
 
-# RUN echo "Update repo 08/08/2025"
-
-RUN git clone https://github.com/OpenHistoricalMap/osmx-adiff-builder.git /app
-
-COPY *.sh .
-COPY *.py .
+# Pipeline scripts come from OSMCha/osmx-adiff-builder (Rust path), customized
+# for OpenHistoricalMap and copied in directly instead of cloned.
+COPY scripts/*.sh scripts/*.py scripts/*.mk /app/
 
 RUN chmod +x /app/*.sh && chmod +x /app/*.py
 

images/ohmx-adiff-builder/config.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Shared configuration. Sourced by start.sh and process_min_range.sh.
+export WORKDIR=${WORKDIR:-/data}
+
+# Binaries: osmx (C++) for expand/update/query, osmx-rs (Rust) for augmented diffs.
+export OSMX_BIN=${OSMX_BIN:-osmx}
+export OSMX_RS_BIN=${OSMX_RS_BIN:-osmx-rs}
+
+# Database
+export OSMX_DB_DIR=$WORKDIR/db
+export OSMX_DB_PATH=$OSMX_DB_DIR/osmx.db
+export PLANET_FILE_PATH=$WORKDIR/planet.osm.pbf
+
+# Services
+export REPLICATION_URL="${REPLICATION_URL:-https://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute}"
+export API_URL=${API_URL:-https://api.openhistoricalmap.org}
+
+# Working directories
+export SPLIT_ADIFFS_DIR=$WORKDIR/stage-data/split-adiffs   # per-changeset fragments
+export CHANGESET_DIR=$WORKDIR/stage-data/changesets        # merge.mk stamp files
+export BUCKET_DIR=$WORKDIR/stage-data/bucket-data          # merged adiffs ready to upload
+export BAD_CHANGESETS_DIR=$WORKDIR/stage-data/bad_changesets
+
+# Upload target: s3://$AWS_S3_BUCKET/$S3_PREFIX/<changeset>.adiff
+export S3_PREFIX=${S3_PREFIX:-ohm-augmented-diffs/changesets}
+
+# Liveness (watchdog in start.sh + healthcheck.sh)
+export HEARTBEAT_DIR=${HEARTBEAT_DIR:-/tmp/heartbeat}
+export HEARTBEAT_STALE_SECONDS=${HEARTBEAT_STALE_SECONDS:-600}
+
+# Kill one update.sh run if it exceeds this (must be < HEARTBEAT_STALE_SECONDS)
+export UPDATE_TIMEOUT=${UPDATE_TIMEOUT:-540}

images/ohmx-adiff-builder/functions.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# Usage: gc.sh
+# 
+# Deletes old intermediate files in stage-data/ that aren't needed anymore.
+# The openstreetmap.org server automatically closes changesets after 24h,
+# so if a changeset hasn't been modified in at least that long, we can
+# safely assume that it won't change again in the future.
+
+SPLIT_ADIFFS_DIR=${1:-'stage-data/split-adiffs'}
+CHANGESET_DIR=${2:-'stage-data/changesets'}
+
+
+find "$CHANGESET_DIR" -type f -name "*.adiff.md5" -mtime +3 | while read stampfile; do
+  changeset_id=$(basename "$stampfile" .adiff.md5)
+  echo "removing files for changeset $changeset_id"
+  # atomically move the split files to a temporary location before deleting
+  # them (prevents merge_adiffs.py being run during the deletion, which could
+  # result in an incomplete adiff being generated)
+  tmpdir=$(mktemp -d)
+  mv $SPLIT_ADIFFS_DIR/$changeset_id/ $tmpdir
+  rm -rf $tmpdir
+
+  # also delete the stamp file
+  rm $CHANGESET_DIR/$changeset_id.adiff.md5
+done

images/ohmx-adiff-builder/process_min_range.sh

@@ -13,7 +13,7 @@ if [ ! -d "$HEARTBEAT_DIR" ]; then
 fi
 
 now=$(date +%s)
-for name in create_diff_files process_diff_files upload_diff_files; do
+for name in make_diffs publish; do
   hb="$HEARTBEAT_DIR/$name"
   if [ ! -f "$hb" ]; then
     echo "unhealthy: no heartbeat for $name"