Skip to content

Commit 3399903

Browse files
Rub21Rub21
authored
Merge pull request #591 from OpenHistoricalMap/ohmexpress
ohmx-adiff for osmcha
2 parents bb04141 + 682bd30 commit 3399903

File tree

7 files changed

+427
-0
lines changed

7 files changed

+427
-0
lines changed

‎.github/workflows/deploy-ohmx-adiff.yaml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: Deploy ohm adiff generator in Hetzner Cloud
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- ohmexpress
8+
jobs:
9+
deploy:
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- name: Set up SSH key
14+
uses: webfactory/[email protected]
15+
with:
16+
ssh-private-key: ${{ secrets.HETZNER_SSH_PRIVATE_KEY }}
17+
18+
- name: Add EC2 host to known_hosts
19+
run: |
20+
ssh-keyscan -H ${{ secrets.HETZNER_HOST }} >> ~/.ssh/known_hosts
21+
22+
- name: Create .env file with all variables
23+
run: |
24+
# Get branch name
25+
BRANCH_NAME="${GITHUB_REF##*/}"
26+
echo "BRANCH_NAME=${BRANCH_NAME}" > .env
27+
echo "REPO=${{ github.repository }}" >> .env
28+
echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID}}" >> .env
29+
echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY}}" >> .env
30+
31+
case "$BRANCH_NAME" in
32+
staging|ohmexpress)
33+
ENVIRONMENT="staging"
34+
REMOTE_APP_DIR="/staging/ohmx_addif"
35+
echo "ENVIRONMENT=$ENVIRONMENT" >> .env
36+
echo "REMOTE_APP_DIR=$REMOTE_APP_DIR" >> .env
37+
;;
38+
main)
39+
ENVIRONMENT="production"
40+
REMOTE_APP_DIR="/production/ohmx_addif"
41+
echo "ENVIRONMENT=$ENVIRONMENT" >> .env
42+
echo "REMOTE_APP_DIR=$REMOTE_APP_DIR" >> .env
43+
;;
44+
*)
45+
echo "Unknown branch: $BRANCH_NAME"
46+
exit 1
47+
;;
48+
esac
49+
echo ".env file created for branch: $BRANCH_NAME"
50+
echo "REMOTE_APP_DIR=$REMOTE_APP_DIR" >> $GITHUB_ENV
51+
echo "ENVIRONMENT=$ENVIRONMENT" >> $GITHUB_ENV
52+
echo "------------------------------------------"
53+
54+
- name: Deploy ohmx_adiff app
55+
run: |
56+
scp -o StrictHostKeyChecking=no .env root@${{ secrets.HETZNER_HOST }}:/tmp/.env.ohmx_adiff
57+
ssh root@${{ secrets.HETZNER_HOST }} <<'EOF'
58+
set -e
59+
echo "Starting app deployment..."
60+
61+
# Load variables from temporary .env
62+
set -o allexport
63+
source /tmp/.env.ohmx_adiff
64+
set +o allexport
65+
66+
echo "-----------------------------------------"
67+
echo "Clone or update $BRANCH_NAME"
68+
if [ ! -d "$REMOTE_APP_DIR/.git" ]; then
69+
echo "Repo not found. Cloning..."
70+
git clone -b $BRANCH_NAME https://github.com/$REPO.git $REMOTE_APP_DIR
71+
cd $REMOTE_APP_DIR
72+
else
73+
cd $REMOTE_APP_DIR
74+
git reset --hard HEAD
75+
git fetch origin
76+
git checkout $BRANCH_NAME
77+
git pull origin $BRANCH_NAME
78+
fi
79+
80+
TARGET_DIR="$REMOTE_APP_DIR/hetzner/ohmx_adiff"
81+
ENV_FILE="$TARGET_DIR/.env.ohmx_adiff"
82+
83+
echo "-----------------------------------------"
84+
echo "Checking if .env.ohmx_adiff has changed"
85+
86+
if [ -f "$ENV_FILE" ]; then
87+
if cmp -s /tmp/.env.ohmx_adiff "$ENV_FILE"; then
88+
echo ".env.ohmx_adiff is identical to the previous version"
89+
ENV_CHANGED=false
90+
else
91+
echo ".env.ohmx_adiff has changed. Updating..."
92+
cp /tmp/.env.ohmx_adiff "$ENV_FILE"
93+
ENV_CHANGED=true
94+
fi
95+
else
96+
echo "No previous .env.ohmx_adiff found. Copying new one..."
97+
cp /tmp/.env.ohmx_adiff "$ENV_FILE"
98+
ENV_CHANGED=true
99+
fi
100+
101+
echo "-----------------------------------------"
102+
echo "Checking for changes in $TARGET_DIR/ohmx_adiff.$ENVIRONMENT.yml"
103+
104+
# Get list of changed files since last pull
105+
CHANGED_FILES=$(git diff --name-only HEAD@{1} HEAD || echo "")
106+
echo "Changed files since last update:"
107+
echo "$CHANGED_FILES"
108+
109+
TARGET_FILE="hetzner/ohmx_adiff/ohmx_adiff.$ENVIRONMENT.yml"
110+
if echo "$CHANGED_FILES" | grep -q "^$TARGET_FILE\$"; then
111+
echo "$TARGET_FILE changed"
112+
FILE_CHANGED=true
113+
else
114+
echo "No changes in $TARGET_FILE"
115+
FILE_CHANGED=false
116+
fi
117+
118+
echo "-----------------------------------------"
119+
if [ "$ENV_CHANGED" = true ] || [ "$FILE_CHANGED" = true ]; then
120+
echo "Changes detected. Building and redeploying..."
121+
docker compose -f $TARGET_FILE build --pull --no-cache
122+
docker compose -f $TARGET_FILE up -d --force-recreate
123+
echo "Deployment finished in $REMOTE_APP_DIR"
124+
else
125+
echo "No relevant changes. Skipping Docker build and redeploy."
126+
fi
127+
128+
echo "-----------------------------------------"
129+
docker ps | grep ohmx_adiff || echo "ohmx_adiff container not running"
130+
EOF

‎compose/ohm-express.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
services:
2+
ohmx-adiff:
3+
image: rub21/ohmx-adiff:v6
4+
container_name: ohmx-adiff
5+
build:
6+
context: ../images/ohmx-adiff-builder
7+
dockerfile: Dockerfile
8+
environment:
9+
- API_URL=https://api.openhistoricalmap.org
10+
- PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250807_0102.osm.pbf
11+
- MINUTE_REPLICATION_URL=https://planet.openhistoricalmap.org/?prefix=replication/minute/
12+
- OSMX_INITIAL_SEQNUM=1683000
13+
- AWS_S3_BUCKET=planet.openhistoricalmap.org
14+
env_file:
15+
- ../envs/.env.ohmx_addif
16+
volumes:
17+
- ohmx_db:/data
18+
- ./../images/ohmx-adiff-builder/start.sh:/app/start.sh
19+
- ./../images/ohmx-adiff-builder/utils:/app/utils
20+
- ./../../osmx-adiff-builder/:/app
21+
volumes:
22+
ohmx_db:
23+
driver: local
24+
name: ohmx_db

‎hetzner/ohmx_adiff/ohmx_adiff.production.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
services:
2+
ohmx_adiff_producion:
3+
image: rub21/ohmx-adiff:v6
4+
container_name: ohmx_adiff_producion
5+
environment:
6+
- API_URL=https://api.openhistoricalmap.org
7+
- PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250807_0102.osm.pbf
8+
- MINUTE_REPLICATION_URL=https://planet.openhistoricalmap.org/?prefix=replication/minute/
9+
- OSMX_INITIAL_SEQNUM=1810000
10+
- AWS_S3_BUCKET=planet.openhistoricalmap.org
11+
env_file:
12+
- .env.ohmx_adiff
13+
volumes:
14+
- ohmx_db:/data
15+
volumes:
16+
ohmx_db:
17+
driver: local
18+
name: ohmx_db

‎hetzner/ohmx_adiff/ohmx_adiff.staging.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
services:
2+
ohmx_adiff_staging:
3+
image: rub21/ohmx-adiff:v6
4+
container_name: ohmx_adiff_staging
5+
environment:
6+
- API_URL=https://api.openhistoricalmap.org
7+
- PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250807_0102.osm.pbf
8+
- MINUTE_REPLICATION_URL=https://planet.openhistoricalmap.org/?prefix=replication/minute/
9+
- OSMX_INITIAL_SEQNUM=1810000
10+
- AWS_S3_BUCKET=planet.openhistoricalmap.org
11+
env_file:
12+
- .env.ohmx_adiff
13+
volumes:
14+
- ohmx_db:/data
15+
volumes:
16+
ohmx_db:
17+
driver: local
18+
name: ohmx_db

‎images/ohmx-adiff-builder/Dockerfile‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
FROM ghcr.io/bdon/osmexpress:latest
2+
3+
ENV PATH="/root/.cargo/bin:${PATH}"
4+
5+
RUN apk add --no-cache python3 py3-pip git bash xmlstarlet curl aws-cli
6+
7+
RUN apk add --no-cache --virtual .build-deps build-base patch python3-dev make cmake zlib-dev expat-dev bzip2-dev boost-dev rust cargo && \
8+
\
9+
pip install --break-system-packages osmx py3-requests osmium && \
10+
\
11+
echo "--- Clonando y modificando osm-cli para OpenHistoricalMap ---" && \
12+
git clone https://github.com/jake-low/osm-cli.git /tmp/osm-cli && \
13+
cd /tmp/osm-cli && \
14+
echo "Cambiando URL en src/replication.rs" && \
15+
sed -i 's|https://planet.openstreetmap.org/replication/minute|https://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute|g' src/replication.rs && \
16+
echo "Cambiando URL en src/main.rs" && \
17+
sed -i 's|https://www.openstreetmap.org|https://www.openhistoricalmap.org|g' src/main.rs && \
18+
echo "Compilando osm-cli desde la fuente modificada..." && \
19+
cargo install --path . && \
20+
cd / && \
21+
rm -rf /tmp/osm-cli && \
22+
\
23+
apk del .build-deps && \
24+
apk add make
25+
26+
RUN curl https://mise.jdx.dev/install.sh | sh
27+
ENV PATH="/root/.local/bin:${PATH}"
28+
29+
WORKDIR /app
30+
31+
RUN echo "Update repo 08/08/2025"
32+
33+
RUN git clone https://github.com/OpenHistoricalMap/osmx-adiff-builder.git /app
34+
35+
COPY start.sh .
36+
COPY update.sh .
37+
38+
RUN chmod +x /app/*.sh && chmod +x /app/*.py
39+
40+
ENTRYPOINT ["/bin/bash", "-c"]
41+
CMD ["/app/start.sh"]

‎images/ohmx-adiff-builder/start.sh‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/bin/bash
2+
# set -e
3+
4+
WORKDIR=/data
5+
6+
## Database path
7+
OSMX_DB_DIR=$WORKDIR/db/
8+
OSMX_DB_PATH=$OSMX_DB_DIR/osmx.db
9+
PLANET_FILE_PATH=$WORKDIR/planet.osm.pbf
10+
11+
## URLs services
12+
REPLICATION_URL="${REPLICATION_URL:-https://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute}"
13+
API_URL=${API_URL:-https://api.openstreetmap.org}
14+
15+
## Reqeuiried directories
16+
REPLICATION_ADIFFS_DIR=$WORKDIR/stage-data/replication-adiffs
17+
SPLIT_ADIFFS_DIR=$WORKDIR/stage-data/split-adiffs
18+
CHANGESET_DIR=$WORKDIR/stage-data/changesets
19+
BUCKET_DIR=$WORKDIR/stage-data/bucket-data
20+
UPLOAD_TRACK_FILE=$WORKDIR/stage-data/uploaded_files.md5
21+
BAD_CHANGESETS_DIR=$WORKDIR/stage-data/bad_changesets
22+
23+
## Sequence number
24+
OSMX_INITIAL_SEQNUM=${OSMX_INITIAL_SEQNUM:-0}
25+
26+
## Process diff files from the last 60 min
27+
FILTER_ADIFF_FILES=60
28+
29+
mkdir -p $OSMX_DB_DIR $REPLICATION_ADIFFS_DIR $SPLIT_ADIFFS_DIR $CHANGESET_DIR $BUCKET_DIR $BAD_CHANGESETS_DIR
30+
31+
create_database() {
32+
if [ ! -f "$OSMX_DB_PATH" ]; then
33+
# Download the planet file if it doesn't exist and a URL is provided
34+
if [ ! -f "$PLANET_FILE_PATH" ] && [ -n "$PLANET_PBF_URL" ]; then
35+
wget -O "$PLANET_FILE_PATH" "$PLANET_PBF_URL"
36+
elif [ ! -f "$PLANET_FILE_PATH" ]; then
37+
echo "ERROR: Planet file not found at $PLANET_FILE_PATH and PLANET_PBF_URL is not set."
38+
exit 1
39+
fi
40+
osmx expand "$PLANET_FILE_PATH" "$OSMX_DB_PATH"
41+
echo "Database created successfully."
42+
43+
## Update database with initial sequence number and get adiff files, starts at OSMX_INITIAL_SEQNU
44+
./update.sh $OSMX_DB_PATH $REPLICATION_ADIFFS_DIR $BAD_CHANGESETS_DIR $OSMX_INITIAL_SEQNUM
45+
## Start generating files with no initial sequence number
46+
create_diff_files
47+
48+
else
49+
echo "Database already exists. Skipping creation."
50+
fi
51+
}
52+
53+
54+
create_diff_files() {
55+
while true; do
56+
echo "Running update.sh at $(date)..."
57+
if ! ./update.sh "$OSMX_DB_PATH" "$REPLICATION_ADIFFS_DIR" "$BAD_CHANGESETS_DIR"; then
58+
echo "update.sh failed at $(date), restarting..."
59+
else
60+
echo "update.sh completed successfully at $(date)"
61+
fi
62+
63+
sleep 60
64+
done
65+
}
66+
67+
process_diff_files() {
68+
while true; do
69+
echo "Processing diff files at $(date)..."
70+
if ! ./process.sh \
71+
"$REPLICATION_ADIFFS_DIR" \
72+
"$SPLIT_ADIFFS_DIR" \
73+
"$CHANGESET_DIR" \
74+
"$BUCKET_DIR" \
75+
"$API_URL" \
76+
"$FILTER_ADIFF_FILES"; then
77+
echo "process.sh failed at $(date), restarting..."
78+
else
79+
echo "process.sh completed successfully at $(date)"
80+
fi
81+
82+
sleep 60
83+
done
84+
}
85+
86+
upload_diff_files() {
87+
mkdir -p "$(dirname "$UPLOAD_TRACK_FILE")"
88+
touch "$UPLOAD_TRACK_FILE"
89+
90+
declare -A uploaded_md5s
91+
while read -r line; do
92+
file=$(echo "$line" | awk '{print $1}')
93+
hash=$(echo "$line" | awk '{print $2}')
94+
uploaded_md5s["$file"]="$hash"
95+
done < "$UPLOAD_TRACK_FILE"
96+
97+
while true; do
98+
echo "Uploading files at $(date)..."
99+
find "$BUCKET_DIR" -type f -name '*.adiff' -mmin -60 | while read -r filepath; do
100+
filename=$(basename "$filepath")
101+
current_md5=$(md5sum "$filepath" | awk '{print $1}')
102+
103+
if [[ -n "${uploaded_md5s[$filename]}" ]]; then
104+
if [[ "${uploaded_md5s[$filename]}" == "$current_md5" ]]; then
105+
echo "Skipping unchanged: $filename"
106+
continue
107+
else
108+
echo "File changed: $filename — reuploading"
109+
fi
110+
else
111+
echo "New file: $filename — uploading"
112+
fi
113+
114+
aws s3 cp "$filepath" "s3://$AWS_S3_BUCKET/ohm-augmented-diffs/changesets/$filename" \
115+
--content-type "application/xml" \
116+
--content-encoding "gzip"
117+
118+
uploaded_md5s["$filename"]="$current_md5"
119+
done
120+
121+
# Actualiza archivo de control
122+
: > "$UPLOAD_TRACK_FILE"
123+
for fname in "${!uploaded_md5s[@]}"; do
124+
echo "$fname ${uploaded_md5s[$fname]}" >> "$UPLOAD_TRACK_FILE"
125+
done
126+
127+
sleep 15
128+
done
129+
}
130+
131+
## Start process diff files
132+
create_database
133+
create_diff_files &
134+
process_diff_files &
135+
upload_diff_files

0 commit comments

Comments
 (0)