Add script to update and handle issue for ohmx

Author: Rub21

images/ohmx-adiff-builder/Dockerfile

@@ -33,6 +33,7 @@ RUN echo "Update repo 08/08/2025"
 RUN git clone https://github.com/OpenHistoricalMap/osmx-adiff-builder.git /app
 
 COPY start.sh .
+COPY update.sh .
 
 RUN chmod +x /app/*.sh && chmod +x /app/*.py
 

images/ohmx-adiff-builder/start.sh

@@ -18,14 +18,15 @@ SPLIT_ADIFFS_DIR=$WORKDIR/stage-data/split-adiffs
 CHANGESET_DIR=$WORKDIR/stage-data/changesets
 BUCKET_DIR=$WORKDIR/stage-data/bucket-data
 UPLOAD_TRACK_FILE=$WORKDIR/stage-data/uploaded_files.md5
+BAD_CHANGESETS_DIR=$WORKDIR/stage-data/bad_changesets
 
 ## Sequence number
 OSMX_INITIAL_SEQNUM=${OSMX_INITIAL_SEQNUM:-0}
 
 ## Process diff files from the last 60 min
 FILTER_ADIFF_FILES=60
 
-mkdir -p $OSMX_DB_DIR $REPLICATION_ADIFFS_DIR $SPLIT_ADIFFS_DIR $CHANGESET_DIR $BUCKET_DIR
+mkdir -p $OSMX_DB_DIR $REPLICATION_ADIFFS_DIR $SPLIT_ADIFFS_DIR $CHANGESET_DIR $BUCKET_DIR $BAD_CHANGESETS_DIR
 
 create_database() {
   if [ ! -f "$OSMX_DB_PATH" ]; then
@@ -40,7 +41,7 @@ create_database() {
     echo "Database created successfully."
 
     ## Update database with initial sequence number and get adiff files, starts at OSMX_INITIAL_SEQNU
-    ./update.sh $OSMX_DB_PATH $REPLICATION_ADIFFS_DIR $OSMX_INITIAL_SEQNUM
+    ./update.sh $OSMX_DB_PATH $REPLICATION_ADIFFS_DIR $BAD_CHANGESETS_DIR $OSMX_INITIAL_SEQNUM
     ## Start generating files with no initial sequence number
     create_diff_files
 
@@ -53,7 +54,7 @@ create_database() {
 create_diff_files() {
   while true; do
     echo "Running update.sh at $(date)..."
-    if ! ./update.sh "$OSMX_DB_PATH" "$REPLICATION_ADIFFS_DIR"; then
+    if ! ./update.sh "$OSMX_DB_PATH" "$REPLICATION_ADIFFS_DIR" "$BAD_CHANGESETS_DIR"; then
       echo "update.sh failed at $(date), restarting..."
     else
       echo "update.sh completed successfully at $(date)"

images/ohmx-adiff-builder/update.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -euo pipefail
+
+# This script is base  on https://github.com/OSMCha/osmx-adiff-builder/blob/main/update.sh
+
+export PATH="$PATH:$PWD"
+
+if [ "$#" -lt 3 ]; then
+    echo "Usage: $0 <osmx_db_path> <replication_adiffs_dir> <bad_changesets_dir> [initial_seqnum]"
+    exit 1
+fi
+
+OSMX_DB_PATH=$1
+REPLICATION_ADIFFS_DIR=$2
+BAD_CHANGESETS_DIR=$3
+INITIAL_SEQNUM=${4:-}
+
+eval "$(mise activate bash --shims)"
+
+if [ -n "${INITIAL_SEQNUM:-}" ]; then
+  seqno_start="$INITIAL_SEQNUM"
+else
+  seqno_start="$(osmx query "$OSMX_DB_PATH" seqnum)"
+fi
+
+echo ">>> Starting from seqno=$seqno_start"
+
+osm replication minute --seqno "$seqno_start" \
+| while read -r seqno timestamp url; do
+    # Exit if seqno is empty
+    if [ -z "${seqno:-}" ]; then
+      echo ">>> seqno is empty, end of stream"; break
+    fi
+
+    echo ">>> [$seqno] fetching $url"
+    # Robust curl with retries and timeout
+    if ! curl -fsSL --retry 3 --retry-delay 2 --max-time 120 "$url" | gzip -d > "${seqno}.osc"; then
+      echo "!!! [$seqno] failed to download or decompress. skipping..."
+      continue
+    fi
+
+    echo ">>> [$seqno] building adiff"
+    tmpfile="$(mktemp)"
+    if augmented_diff.py "$OSMX_DB_PATH" "${seqno}.osc" \
+       | xmlstarlet format > "$tmpfile"; then
+        mv "$tmpfile" "$REPLICATION_ADIFFS_DIR/${seqno}.adiff"
+        echo ">>> [$seqno] adiff OK -> ${REPLICATION_ADIFFS_DIR}/${seqno}.adiff"
+    else
+        rm -f "$tmpfile"
+        mv "${seqno}.osc" "$BAD_CHANGESETS_DIR/${seqno}.osc"
+        echo "!!! [$seqno] error generating adiff (possible invalid XML). OSC moved to $BAD_CHANGESETS_DIR. Continuing..."
+        continue
+    fi
+
+    echo ">>> [$seqno] osmx update"
+    osmx update "$OSMX_DB_PATH" "${seqno}.osc" "$seqno" "$timestamp" --commit || {
+      echo "!!! [$seqno] osmx update failed"; exit 1;
+    }
+    rm -f "${seqno}.osc"
+    echo ">>> [$seqno] done"
+done