ohmx-adiff for osmcha

.github/workflows/deploy-ohmx-adiff.yaml

@@ -0,0 +1,130 @@
+name: Deploy ohm adiff generator in Hetzner Cloud
+
+on:
+  push:
+    branches:
+      - main
+      - ohmexpress 
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up SSH key
+      uses: webfactory/ssh-agent@v0.8.0
+      with:
+        ssh-private-key: ${{ secrets.HETZNER_SSH_PRIVATE_KEY }}
+
+    - name: Add EC2 host to known_hosts
+      run: |
+        ssh-keyscan -H ${{ secrets.HETZNER_HOST }} >> ~/.ssh/known_hosts
+
+    - name: Create .env file with all variables
+      run: |
+        # Get branch name
+        BRANCH_NAME="${GITHUB_REF##*/}"
+        echo "BRANCH_NAME=${BRANCH_NAME}" > .env
+        echo "REPO=${{ github.repository }}" >> .env
+        echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID}}" >> .env
+        echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY}}" >> .env
+
+        case "$BRANCH_NAME" in
+          staging|ohmexpress)
+            ENVIRONMENT="staging"
+            REMOTE_APP_DIR="/staging/ohmx_addif"
+            echo "ENVIRONMENT=$ENVIRONMENT" >> .env
+            echo "REMOTE_APP_DIR=$REMOTE_APP_DIR" >> .env
+            ;;
+          main)
+            ENVIRONMENT="production"
+            REMOTE_APP_DIR="/production/ohmx_addif"
+            echo "ENVIRONMENT=$ENVIRONMENT" >> .env
+            echo "REMOTE_APP_DIR=$REMOTE_APP_DIR" >> .env
+            ;;
+          *)
+            echo "Unknown branch: $BRANCH_NAME"
+            exit 1
+            ;;
+        esac
+        echo ".env file created for branch: $BRANCH_NAME"
+        echo "REMOTE_APP_DIR=$REMOTE_APP_DIR" >> $GITHUB_ENV
+        echo "ENVIRONMENT=$ENVIRONMENT" >> $GITHUB_ENV
+        echo "------------------------------------------"
+
+    - name: Deploy ohmx_adiff app
+      run: |
+        scp -o StrictHostKeyChecking=no .env root@${{ secrets.HETZNER_HOST }}:/tmp/.env.ohmx_adiff
+        ssh root@${{ secrets.HETZNER_HOST }} <<'EOF'
+          set -e
+          echo "Starting app deployment..."
+
+          # Load variables from temporary .env
+          set -o allexport
+          source /tmp/.env.ohmx_adiff
+          set +o allexport
+
+          echo "-----------------------------------------"
+          echo "Clone or update $BRANCH_NAME"
+          if [ ! -d "$REMOTE_APP_DIR/.git" ]; then
+            echo "Repo not found. Cloning..."
+            git clone -b $BRANCH_NAME https://github.com/$REPO.git $REMOTE_APP_DIR
+            cd $REMOTE_APP_DIR
+          else
+            cd $REMOTE_APP_DIR
+            git reset --hard HEAD
+            git fetch origin
+            git checkout $BRANCH_NAME
+            git pull origin $BRANCH_NAME
+          fi
+
+          TARGET_DIR="$REMOTE_APP_DIR/hetzner/ohmx_adiff"
+          ENV_FILE="$TARGET_DIR/.env.ohmx_adiff"
+
+          echo "-----------------------------------------"
+          echo "Checking if .env.ohmx_adiff has changed"
+
+          if [ -f "$ENV_FILE" ]; then
+            if cmp -s /tmp/.env.ohmx_adiff "$ENV_FILE"; then
+              echo ".env.ohmx_adiff is identical to the previous version"
+              ENV_CHANGED=false
+            else
+              echo ".env.ohmx_adiff has changed. Updating..."
+              cp /tmp/.env.ohmx_adiff "$ENV_FILE"
+              ENV_CHANGED=true
+            fi
+          else
+            echo "No previous .env.ohmx_adiff found. Copying new one..."
+            cp /tmp/.env.ohmx_adiff "$ENV_FILE"
+            ENV_CHANGED=true
+          fi
+
+          echo "-----------------------------------------"
+          echo "Checking for changes in $TARGET_DIR/ohmx_adiff.$ENVIRONMENT.yml"
+
+          # Get list of changed files since last pull
+          CHANGED_FILES=$(git diff --name-only HEAD@{1} HEAD || echo "")
+          echo "Changed files since last update:"
+          echo "$CHANGED_FILES"
+
+          TARGET_FILE="hetzner/ohmx_adiff/ohmx_adiff.$ENVIRONMENT.yml"
+          if echo "$CHANGED_FILES" | grep -q "^$TARGET_FILE\$"; then
+            echo "$TARGET_FILE changed"
+            FILE_CHANGED=true
+          else
+            echo "No changes in $TARGET_FILE"
+            FILE_CHANGED=false
+          fi
+
+          echo "-----------------------------------------"
+          if [ "$ENV_CHANGED" = true ] || [ "$FILE_CHANGED" = true ]; then
+            echo "Changes detected. Building and redeploying..."
+            docker compose -f $TARGET_FILE build --pull --no-cache
+            docker compose -f $TARGET_FILE up -d --force-recreate
+            echo "Deployment finished in $REMOTE_APP_DIR"
+          else
+            echo "No relevant changes. Skipping Docker build and redeploy."
+          fi
+
+          echo "-----------------------------------------"
+          docker ps | grep ohmx_adiff || echo "ohmx_adiff container not running"
+        EOF

compose/ohm-express.yml

@@ -0,0 +1,24 @@
+services:
+  ohmx-adiff:
+    image: rub21/ohmx-adiff:v6
+    container_name: ohmx-adiff
+    build:
+      context: ../images/ohmx-adiff-builder
+      dockerfile: Dockerfile
+    environment:
+      - API_URL=https://api.openhistoricalmap.org
+      - PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250807_0102.osm.pbf
+      - MINUTE_REPLICATION_URL=https://planet.openhistoricalmap.org/?prefix=replication/minute/
+      - OSMX_INITIAL_SEQNUM=1683000
+      - AWS_S3_BUCKET=planet.openhistoricalmap.org
+    env_file:
+      - ../envs/.env.ohmx_addif
+    volumes:
+      - ohmx_db:/data
+      - ./../images/ohmx-adiff-builder/start.sh:/app/start.sh
+      - ./../images/ohmx-adiff-builder/utils:/app/utils
+      - ./../../osmx-adiff-builder/:/app
+volumes:
+  ohmx_db:
+    driver: local
+    name: ohmx_db

hetzner/ohmx_adiff/ohmx_adiff.production.yml

@@ -0,0 +1,18 @@
+services:
+  ohmx_adiff_producion:
+    image: rub21/ohmx-adiff:v6
+    container_name: ohmx_adiff_producion
+    environment:
+      - API_URL=https://api.openhistoricalmap.org
+      - PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250807_0102.osm.pbf
+      - MINUTE_REPLICATION_URL=https://planet.openhistoricalmap.org/?prefix=replication/minute/
+      - OSMX_INITIAL_SEQNUM=1810000
+      - AWS_S3_BUCKET=planet.openhistoricalmap.org
+    env_file:
+      - .env.ohmx_adiff
+    volumes:
+      - ohmx_db:/data
+volumes:
+  ohmx_db:
+    driver: local
+    name: ohmx_db

hetzner/ohmx_adiff/ohmx_adiff.staging.yml

@@ -0,0 +1,18 @@
+services:
+  ohmx_adiff_staging:
+    image: rub21/ohmx-adiff:v6
+    container_name: ohmx_adiff_staging
+    environment:
+      - API_URL=https://api.openhistoricalmap.org
+      - PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250807_0102.osm.pbf
+      - MINUTE_REPLICATION_URL=https://planet.openhistoricalmap.org/?prefix=replication/minute/
+      - OSMX_INITIAL_SEQNUM=1810000
+      - AWS_S3_BUCKET=planet.openhistoricalmap.org
+    env_file:
+      - .env.ohmx_adiff
+    volumes:
+      - ohmx_db:/data
+volumes:
+  ohmx_db:
+    driver: local
+    name: ohmx_db

images/ohmx-adiff-builder/Dockerfile

@@ -0,0 +1,41 @@
+FROM ghcr.io/bdon/osmexpress:latest
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+RUN apk add --no-cache python3 py3-pip git bash xmlstarlet curl aws-cli
+
+RUN apk add --no-cache --virtual .build-deps build-base patch python3-dev make cmake zlib-dev expat-dev bzip2-dev boost-dev rust cargo && \
+    \
+    pip install --break-system-packages osmx py3-requests osmium && \
+    \
+    echo "--- Clonando y modificando osm-cli para OpenHistoricalMap ---" && \
+    git clone https://github.com/jake-low/osm-cli.git /tmp/osm-cli && \
+    cd /tmp/osm-cli && \
+    echo "Cambiando URL en src/replication.rs" && \
+    sed -i 's|https://planet.openstreetmap.org/replication/minute|https://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute|g' src/replication.rs && \
+    echo "Cambiando URL en src/main.rs" && \
+    sed -i 's|https://www.openstreetmap.org|https://www.openhistoricalmap.org|g' src/main.rs && \
+    echo "Compilando osm-cli desde la fuente modificada..." && \
+    cargo install --path . && \
+    cd / && \
+    rm -rf /tmp/osm-cli && \
+    \
+    apk del .build-deps && \
+    apk add make
+
+RUN curl https://mise.jdx.dev/install.sh | sh
+ENV PATH="/root/.local/bin:${PATH}"
+
+WORKDIR /app
+
+RUN echo "Update repo 08/08/2025"
+
+RUN git clone https://github.com/OpenHistoricalMap/osmx-adiff-builder.git /app
+
+COPY start.sh .
+COPY update.sh .
+
+RUN chmod +x /app/*.sh && chmod +x /app/*.py
+
+ENTRYPOINT ["/bin/bash", "-c"]
+CMD ["/app/start.sh"]

images/ohmx-adiff-builder/start.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+# set -e
+
+WORKDIR=/data
+
+## Database path
+OSMX_DB_DIR=$WORKDIR/db/
+OSMX_DB_PATH=$OSMX_DB_DIR/osmx.db
+PLANET_FILE_PATH=$WORKDIR/planet.osm.pbf
+
+## URLs services
+REPLICATION_URL="${REPLICATION_URL:-https://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute}"
+API_URL=${API_URL:-https://api.openstreetmap.org}
+
+## Reqeuiried directories
+REPLICATION_ADIFFS_DIR=$WORKDIR/stage-data/replication-adiffs
+SPLIT_ADIFFS_DIR=$WORKDIR/stage-data/split-adiffs
+CHANGESET_DIR=$WORKDIR/stage-data/changesets
+BUCKET_DIR=$WORKDIR/stage-data/bucket-data
+UPLOAD_TRACK_FILE=$WORKDIR/stage-data/uploaded_files.md5
+BAD_CHANGESETS_DIR=$WORKDIR/stage-data/bad_changesets
+
+## Sequence number
+OSMX_INITIAL_SEQNUM=${OSMX_INITIAL_SEQNUM:-0}
+
+## Process diff files from the last 60 min
+FILTER_ADIFF_FILES=60
+
+mkdir -p $OSMX_DB_DIR $REPLICATION_ADIFFS_DIR $SPLIT_ADIFFS_DIR $CHANGESET_DIR $BUCKET_DIR $BAD_CHANGESETS_DIR
+
+create_database() {
+  if [ ! -f "$OSMX_DB_PATH" ]; then
+    # Download the planet file if it doesn't exist and a URL is provided
+    if [ ! -f "$PLANET_FILE_PATH" ] && [ -n "$PLANET_PBF_URL" ]; then
+      wget -O "$PLANET_FILE_PATH" "$PLANET_PBF_URL"
+    elif [ ! -f "$PLANET_FILE_PATH" ]; then
+      echo "ERROR: Planet file not found at $PLANET_FILE_PATH and PLANET_PBF_URL is not set."
+      exit 1
+    fi
+    osmx expand "$PLANET_FILE_PATH" "$OSMX_DB_PATH"
+    echo "Database created successfully."
+
+    ## Update database with initial sequence number and get adiff files, starts at OSMX_INITIAL_SEQNU
+    ./update.sh $OSMX_DB_PATH $REPLICATION_ADIFFS_DIR $BAD_CHANGESETS_DIR $OSMX_INITIAL_SEQNUM
+    ## Start generating files with no initial sequence number
+    create_diff_files
+
+  else
+    echo "Database already exists. Skipping creation."
+  fi
+}
+
+
+create_diff_files() {
+  while true; do
+    echo "Running update.sh at $(date)..."
+    if ! ./update.sh "$OSMX_DB_PATH" "$REPLICATION_ADIFFS_DIR" "$BAD_CHANGESETS_DIR"; then
+      echo "update.sh failed at $(date), restarting..."
+    else
+      echo "update.sh completed successfully at $(date)"
+    fi
+
+    sleep 60
+  done
+}
+
+process_diff_files() {
+  while true; do
+    echo "Processing diff files at $(date)..."
+    if ! ./process.sh \
+      "$REPLICATION_ADIFFS_DIR" \
+      "$SPLIT_ADIFFS_DIR" \
+      "$CHANGESET_DIR" \
+      "$BUCKET_DIR" \
+      "$API_URL" \
+      "$FILTER_ADIFF_FILES"; then
+      echo "process.sh failed at $(date), restarting..."
+    else
+      echo "process.sh completed successfully at $(date)"
+    fi
+
+    sleep 60
+  done
+}
+
+upload_diff_files() {
+  mkdir -p "$(dirname "$UPLOAD_TRACK_FILE")"
+  touch "$UPLOAD_TRACK_FILE"
+
+  declare -A uploaded_md5s
+  while read -r line; do
+    file=$(echo "$line" | awk '{print $1}')
+    hash=$(echo "$line" | awk '{print $2}')
+    uploaded_md5s["$file"]="$hash"
+  done < "$UPLOAD_TRACK_FILE"
+
+  while true; do
+    echo "Uploading files at $(date)..."
+    find "$BUCKET_DIR" -type f -name '*.adiff' -mmin -60 | while read -r filepath; do
+      filename=$(basename "$filepath")
+      current_md5=$(md5sum "$filepath" | awk '{print $1}')
+
+      if [[ -n "${uploaded_md5s[$filename]}" ]]; then
+        if [[ "${uploaded_md5s[$filename]}" == "$current_md5" ]]; then
+          echo "Skipping unchanged: $filename"
+          continue
+        else
+          echo "File changed: $filename — reuploading"
+        fi
+      else
+        echo "New file: $filename — uploading"
+      fi
+
+      aws s3 cp "$filepath" "s3://$AWS_S3_BUCKET/ohm-augmented-diffs/changesets/$filename" \
+        --content-type "application/xml" \
+        --content-encoding "gzip"
+
+      uploaded_md5s["$filename"]="$current_md5"
+    done
+
+    # Actualiza archivo de control
+    : > "$UPLOAD_TRACK_FILE"
+    for fname in "${!uploaded_md5s[@]}"; do
+      echo "$fname ${uploaded_md5s[$fname]}" >> "$UPLOAD_TRACK_FILE"
+    done
+
+    sleep 15
+  done
+}
+
+## Start process diff files
+create_database
+create_diff_files &
+process_diff_files &
+upload_diff_files