Merge pull request #1 from siameseoriental/DO-101

siameseoriental · web-flow · commit a9333e948382 · 2023-08-24T13:52:29.000+02:00
DO-101
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -0,0 +1,62 @@
+name: Docker Image CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+      - 'master'
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - 'main'
+      - 'master'
+
+env:
+  REGISTRY: docker.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+
+      # Login against a Docker registry
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.DOCKERHUB_LOGIN }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,48 @@
+# syntax=docker/dockerfile:1.2
+
+FROM debian:stable as base
+
+MAINTAINER NumDes <info@numdes.com>
+
+LABEL org.opencontainers.image.vendor="Numerical Design LLC"
+LABEL org.opencontainers.image.description="Docker image for universal postgres backups"
+
+ARG GOCRONVER=v0.0.10
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+    curl \
+    ca-certificates \
+    postgresql-client \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && curl --fail --retry 4 --retry-all-errors -o /usr/local/bin/go-cron.gz -L https://github.com/prodrigestivill/go-cron/releases/download/$GOCRONVER/go-cron-linux-amd64.gz \
+    && gzip -vnd /usr/local/bin/go-cron.gz && chmod a+x /usr/local/bin/go-cron
+
+RUN curl --location --output /usr/local/bin/mcli "https://dl.min.io/client/mc/release/linux-amd64/mc" && \
+    chmod +x /usr/local/bin/mcli
+RUN mcli -v
+
+ENV POSTGRES_DB="**None**" \
+    POSTGRES_HOST="**None**" \
+    POSTGRES_PORT=5432 \
+    POSTGRES_USER="**None**" \
+    POSTGRES_PASSWORD="**None**" \
+    POSTGRES_EXTRA_OPTS="--blobs" \
+    SCHEDULE="@daily" \
+    HEALTHCHECK_PORT=8080 \
+    S3_ACCESS_KEY_ID="**None**" \
+    S3_SECRET_ACCESS_KEY="**None**" \
+    S3_BUCKET="**None**" \
+    S3_ENDPOINT="**None**"
+
+COPY backup.sh /backup.sh
+RUN chmod +x backup.sh
+
+COPY hooks /hooks
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD ["exec /usr/local/bin/go-cron -s \"$SCHEDULE\" -p \"$HEALTHCHECK_PORT\" -- /backup.sh"]
+
+HEALTHCHECK --interval=5m --timeout=3s \
+  CMD curl -f "http://localhost:$HEALTHCHECK_PORT/" || exit 1
diff --git a/README.md b/README.md
@@ -2,6 +2,105 @@
 Docker image for universal postgres backups
 
 # Roadmap
-- [ ] Add support for S3
+- [X] Add support for S3
+- [X] Add CI/CD to publish image to DockerHub
 - [ ] Add retention policy settings by env vars
-- [ ] Notify about backup status by HTTP-request
+- [X] Notify about backup status by HTTP-request
+- [ ] Add docker-compose example
+
+## Docker build
+```shell
+docker build . -t numdes/nd_postgres_backup:v*.*.*
+```
+
+# Usage
+## Backup manually:
+```shell
+docker run --rm -it \
+    -e POSTGRES_HOST="FQDN-OR-IP" \
+    -e POSTGRES_DB="DB-NAME" \
+    -e POSTGRES_USER="DB-USER" \
+    -e POSTGRES_PASSWORD="PASS" \
+    -e S3_ENDPOINT=http://YOUR-S3 \
+    -e S3_ACCESS_KEY_ID="KEY-ID" \
+    -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \
+    -e S3_BUCKET="BUCKET-NAME" \
+    -e PRIVATE_NOTIFICATION_URL=http://webhook \
+    -e TELEGRAM_CHAT_ID=point_to_notify_group \
+    -e POSTGRES_PORT=if_not_5432 \
+    --entrypoint /bin/bash \
+    numdes/nd_postgres_backup:v*.*.*
+```
+To run backup, in active container shell call `backup.sh` script
+```shell
+./backup.sh
+```
+
+## Backup using `go-cron`
+```shell
+docker run -d \
+    -e POSTGRES_HOST="FQDN-OR-IP" \
+    -e POSTGRES_DB="DB-NAME" \
+    -e POSTGRES_USER="DB-USER" \
+    -e POSTGRES_PASSWORD="PASS" \
+    -e S3_ENDPOINT=http://YOUR-S3 \
+    -e S3_ACCESS_KEY_ID="KEY-ID" \
+    -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \
+    -e S3_BUCKET="BUCKET-NAME" \
+    -e PRIVATE_NOTIFICATION_URL=http://webhook \
+    -e TELEGRAM_CHAT_ID=point_to_notify_group \
+    -e POSTGRES_PORT=if_not_5432 \
+    -e SCHEDULE=Chosen_schedule
+    numdes/nd_postgres_backup:v*.*.*
+```
+:wave: By default `SCHEDULE` variable is set to `@daily` in case if you need other scheduling options, please refer to `go-cron` *[Documentation](https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc#hdr-Predefined_schedules)*.
+
+## Variables
+### `Gitlab Actions` *[variables](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository)*:
+| Name              |  Description                                        |
+|-------------------|-----------------------------------------------------|
+|DOCKERHUB_LOGIN    | `Actions` Repository secret                         |
+|DOCKERHUB_PASSWORD | `Actions` Repository secret                         |
+
+### Notification environmental variables
+| Name                      |  Description                                        |
+|---------------------------|-----------------------------------------------------|
+|TELEGRAM_CHAT_ID           | Notifying group                                     |
+|PRIVATE_NOTIFICATION_URL   | Private notifier URL                                |
+|TELEGRAM_BOT_TOKEN         | Only used to call Telegram's public API             |
+
+### Environmental variables
+| Name                   | Default value   | Description                               |
+|------------------------|  :------   |------------------------------------------------|
+| POSTGRES_DB            |  -         | Database name                                  |
+| POSTGRES_HOST          |  -         | PostgreSQL IP address or hostname              |
+| POSTGRES_PORT          | 5432       | Connection TCP port                            |
+| POSTGRES_USER          |  -         | Database user                                  |
+| POSTGRES_PASSWORD      |  -         | Database user password                         |
+| POSTGRES_EXTRA_OPTS    | --blobs    | Extra options `pg_dump` run                    |
+| SCHEDULE               | @daily     | `go-cron` schedule. See [this](#backup-using-go-cron) |
+| HEALTHCHECK_PORT       | 8080       | Port listening for cron-schedule health check. |
+| S3_ACCESS_KEY_ID       |  -         | Key or username with RW access to bucket       |
+| S3_SECRET_ACCESS_KEY   |  -         | Secret or password for `S3_ACCESS_KEY_ID`      |
+| S3_BUCKET              |  -         | Name of bucket created for backups             |
+| S3_ENDPOINT            |  -         | URL of S3 storage                              |
+
+### Notification selection
+
+It is possible to use either private Telegram bot if you have it or Telegram public API.
+
+In scenario with private bot `PRIVATE_NOTIFICATION_URL` must be set alongside with `TELEGRAM_CHAT_ID`.
+
+In scenario with Telegram's public API `TELEGRAM_BOT_TOKEN` must be set as it is received (`Use this token to access the HTTP API:`) from `@BotFather` Telegram Bot. Variable `TELEGRAM_CHAT_ID` must be a proper Telegram ID of bot
+
+In `docker ...` command need to replace:
+```
+    -e PRIVATE_NOTIFICATION_URL=http://webhook \
+    -e TELEGRAM_CHAT_ID=point_to_notify_group \
+```
+to
+```
+    -e TELEGRAM_BOT_TOKEN='XXXXXXX:XXXXxxxxXXXXxxx' \
+    -e TELEGRAM_CHAT_ID=000000000 \
+```
+- If `TELEGRAM_CHAT_ID` has a proper format (Only digits not less than 5 not more than 32) and `TELEGRAM_BOT_TOKEN` is set, script will try to send notification through Telegram's public API.
diff --git a/backup.sh b/backup.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+#
+# Script made for backup PostgreSQL database from local (${POSTGRES_HOST}=127.0.0.1)
+# or remote host. Created backup stores in S3 storage. On completion script calls
+# notification scripts from hooks/ directory to send report to given Telegram Chat
+# based on variables set private or public notification method will be selected 
+
+set -euo pipefail
+IFS=$'\n\t'
+
+# Will be name of directory in backet yyyy-mm-dd_HH:MM:SS
+timestamp="$(date +%F_%T)"
+
+export PGPASSWORD=${POSTGRES_PASSWORD}
+
+# Will create base backup
+echo "Creating backup of ${POSTGRES_DB} database. From ${POSTGRES_HOST} and port \
+is ${POSTGRES_PORT}. Username: ${POSTGRES_USER}. With following extra options: \
+${POSTGRES_EXTRA_OPTS}"
+pg_dump --username="${POSTGRES_USER}" \
+        --host="${POSTGRES_HOST}" \
+        --port="${POSTGRES_PORT}" \
+        --dbname="${POSTGRES_DB}" \
+        "${POSTGRES_EXTRA_OPTS}" \
+        > "${POSTGRES_DB}".sql
+
+# Declaring variables for informational purposes
+copy_file_name="${POSTGRES_DB}.tar.gz"
+copy_path="${S3_BUCKET}/${POSTGRES_DB}/${timestamp}"
+mcli_copy_path="${copy_path}/${copy_file_name}"
+info_copy_path="${S3_ENDPOINT}/${copy_path}"
+
+# Do compression
+tar --create \
+    --gzip \
+    --verbose \
+    --file "${copy_file_name}" \
+    "${POSTGRES_DB}.sql"
+
+# Count file size
+send_file_size="$(ls -lh "${copy_file_name}" | awk '{print $5}')"
+
+echo "Created ${copy_file_name} with file size: ${send_file_size}"
+
+# Set S3 connection configuration
+mcli alias set backup "${S3_ENDPOINT}" "${S3_ACCESS_KEY_ID}" "${S3_SECRET_ACCESS_KEY}"
+
+echo "Starting to copy ${copy_file_name} to ${info_copy_path}..."
+
+# Copying backup to S3
+mcli cp "${copy_file_name}" backup/"${mcli_copy_path}"
+
+# Do nettoyage
+echo "Maid is here... Doing cleaning..."
+rm --force "${POSTGRES_DB}".*
+
+# Do anounce
+run-parts --reverse \
+          --arg "${copy_file_name}" \
+          --arg "${send_file_size}" \
+          --arg "${info_copy_path}" \
+          /hooks
diff --git a/hooks/send_private_notification b/hooks/send_private_notification
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+#
+# Use http request to send notification to Telegram chat through local Telegram bot
+
+set -eo pipefail
+IFS=$'\n\t'
+
+copy_file_name="${1}"
+send_file_size="${2}"
+info_copy_path="${3}"
+
+message_text="Backed up ${copy_file_name} with file size: ${send_file_size} \
+to ${info_copy_path}"
+
+if [[ -n "${PRIVATE_NOTIFICATION_URL}" ]]; then
+    curl -XPOST \
+      --url "${PRIVATE_NOTIFICATION_URL}" \
+      --header 'Content-Type: application/json' \
+      --data "{\"key\": \"${TELEGRAM_CHAT_ID}\", \"text\": \"${message_text}\"}" \
+      --max-time 10 \
+      --retry 5
+fi
diff --git a/hooks/send_telegram_message b/hooks/send_telegram_message
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# Use http request to send notification to Telegram chat through external Telegram API
+
+set -eo pipefail
+IFS=$'\n\t'
+
+copy_file_name="${1}"
+send_file_size="${2}"
+info_copy_path="${3}"
+
+message_text="Backed up ${copy_file_name} with file size: ${send_file_size} \
+to ${info_copy_path}"
+
+if [[ -n "${TELEGRAM_BOT_TOKEN}" ]]; then
+  if [[ "${TELEGRAM_CHAT_ID}" =~ ^[0-9]{5,32}$ ]]; then
+    curl -s \
+      --data "text=${message_text}" \
+      --data "chat_id=${TELEGRAM_CHAT_ID}" \
+      'https://api.telegram.org/bot'"${TELEGRAM_BOT_TOKEN}"'/sendMessage' > /dev/null
+  else
+    echo "Telegram chatID doesn't matched to standard pattern. Probably \
+    TELEGRAM_BOT_TOKEN variable was set by mistake"
+  fi
+fi
