Skip to content
CI

Bump the major group across 1 directory with 7 updates #4192

Sign in to view logs

Bump the major group across 1 directory with 7 updates

Bump the major group across 1 directory with 7 updates #4192

Workflow file for this run

.github/workflows/ci.yml at 07b80a0
name: CI
permissions:
contents: read
packages: read # Required to pull container images from GHCR
on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main ]
release:
types: [ published ]
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
jobs:
test-sveltekit:
name: Test SvelteKit Project
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./web
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for git tags (needed for vergen version)
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Run linter
run: bun run lint
- name: Run type check
run: bun run check
- name: Build SvelteKit project
env:
# Sentry source map upload (requires secrets to be configured in GitHub)
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
run: bun run build
test-e2e:
name: E2E Tests (Playwright)
runs-on: self-hosted
container:
image: ghcr.io/hut8/soar-ci:latest
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
volumes:
- /opt/ci-cache:/ci-cache
services:
postgres:
image: ghcr.io/hut8/postgis-timescaledb:latest
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
env:
POSTGRES_PASSWORD: postgres
POSTGRES_USER: postgres
POSTGRES_DB: soar_test
options: >-
--health-cmd="pg_isready -U postgres"
--health-interval=10s
--health-timeout=5s
--health-retries=5
mailpit:
image: axllent/mailpit:v1.20
defaults:
run:
shell: bash
working-directory: ./web
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for git tags (needed for vergen version)
- name: Setup persistent cargo cache
run: |
mkdir -p /ci-cache/${RUNNER_NAME}/cargo-registry /ci-cache/${RUNNER_NAME}/cargo-git
ln -sfn /ci-cache/${RUNNER_NAME}/cargo-registry /usr/local/cargo/registry
ln -sfn /ci-cache/${RUNNER_NAME}/cargo-git /usr/local/cargo/git
echo "CARGO_TARGET_DIR=/ci-cache/${RUNNER_NAME}/target-e2e" >> $GITHUB_ENV
- name: Install unzip (required by setup-bun in container)
run: apt-get update -qq && apt-get install -y -qq unzip
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Install web dependencies
run: bun install --frozen-lockfile
- name: Build SvelteKit project
run: bun run build
- name: Install Playwright browsers (if not pre-installed in image)
run: bunx playwright install --with-deps chromium
- name: Wait for Mailpit to be ready
working-directory: .
run: |
timeout 30s bash -c 'until curl -s http://mailpit:8025/api/v1/info > /dev/null 2>&1; do sleep 2; done'
- name: Setup test database
working-directory: .
env:
DATABASE_URL: postgres://postgres:postgres@postgres:5432/soar_test
PGPASSWORD: postgres
TEST_USER_EMAIL: test@example.com
TEST_USER_PASSWORD: testpassword123
SEED_COUNT: 20
run: |
# Create PostGIS extension
psql -h postgres -U postgres -d soar_test -c "CREATE EXTENSION IF NOT EXISTS postgis;"
# Run migrations (includes h3 extension setup)
diesel migration run
# Build the project to get the seed-test-data command
# IMPORTANT: Must use --cfg tokio_unstable for console-subscriber
# Use debug build for E2E - faster to compile, perf doesn't matter for tests
RUSTFLAGS="--cfg tokio_unstable" cargo build
# Seed test data
$CARGO_TARGET_DIR/debug/soar seed-test-data
- name: Start Rust backend server
working-directory: .
env:
DATABASE_URL: postgres://postgres:postgres@postgres:5432/soar_test
SENTRY_DSN: ''
SOAR_ENV: test
JWT_SECRET: test-jwt-secret-for-e2e-tests-only
SMTP_SERVER: mailpit
SMTP_PORT: 1025
SMTP_USERNAME: test
SMTP_PASSWORD: test
FROM_EMAIL: test@soar.local
FROM_NAME: SOAR Test
BASE_URL: http://localhost:4173
run: |
# Start backend server in background
$CARGO_TARGET_DIR/debug/soar web --port 61225 --interface localhost > backend.log 2>&1 &
BACKEND_PID=$!
echo "Backend PID: $BACKEND_PID"
echo $BACKEND_PID > backend.pid
# Wait for backend to be ready (max 60 seconds)
for i in {1..60}; do
if curl -s http://localhost:61225/health > /dev/null 2>&1; then
echo "Backend server is ready!"
break
fi
if [ $i -eq 60 ]; then
echo "Backend server failed to start within 60 seconds"
cat backend.log
exit 1
fi
echo "Waiting for backend... ($i/60)"
sleep 1
done
- name: Run E2E tests
env:
TEST_USER_EMAIL: test@example.com
TEST_USER_PASSWORD: testpassword123
DATABASE_URL: postgres://postgres:postgres@postgres:5432/soar_test
# Point Playwright at the Rust backend which serves both
# embedded static files (from web/build/) and the API (/data/*)
PLAYWRIGHT_BASE_URL: http://localhost:61225
# Mailpit is a service container, accessible by hostname not localhost
MAILPIT_URL: http://mailpit:8025
run: bun run test
- name: Stop backend server
if: always()
working-directory: .
run: |
if [ -f backend.pid ]; then
kill $(cat backend.pid) || true
fi
- name: Upload E2E test artifacts
if: always()
uses: actions/upload-artifact@v7
with:
name: e2e-test-artifacts
path: |
web/playwright-report/
web/test-results/
backend.log
retention-days: 30
test-rust:
name: Test Rust Project
runs-on: self-hosted
container:
image: ghcr.io/hut8/soar-ci:latest
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
volumes:
- /opt/ci-cache:/ci-cache
services:
postgres:
image: ghcr.io/hut8/postgis-timescaledb:latest
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
env:
POSTGRES_PASSWORD: postgres
POSTGRES_USER: postgres
POSTGRES_DB: soar_test
options: >-
--health-cmd="pg_isready -U postgres"
--health-interval=10s
--health-timeout=5s
--health-retries=5
defaults:
run:
shell: bash
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for git tags (needed for vergen version)
- name: Setup persistent cargo cache
run: |
mkdir -p /ci-cache/${RUNNER_NAME}/cargo-registry /ci-cache/${RUNNER_NAME}/cargo-git
ln -sfn /ci-cache/${RUNNER_NAME}/cargo-registry /usr/local/cargo/registry
ln -sfn /ci-cache/${RUNNER_NAME}/cargo-git /usr/local/cargo/git
echo "CARGO_TARGET_DIR=/ci-cache/${RUNNER_NAME}/target-test" >> $GITHUB_ENV
- name: Install unzip (required by setup-bun in container)
run: apt-get update -qq && apt-get install -y -qq unzip
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Install web dependencies
working-directory: ./web
run: bun install --frozen-lockfile
- name: Setup test database
env:
PGPASSWORD: postgres
run: |
# Create PostGIS extension (already available in postgis/postgis image)
psql -h postgres -U postgres -d soar_test -c "CREATE EXTENSION IF NOT EXISTS postgis;"
# H3 extensions will be created by migrations
- name: Check Rust formatting
run: cargo fmt --check
- name: Check migrations for unsafe operations
run: diesel-guard check migrations/
- name: Run migrations
env:
DATABASE_URL: postgres://postgres:postgres@postgres:5432/soar_test
run: diesel migration run
- name: Setup test template database
env:
PGPASSWORD: postgres
PGHOST: postgres
PGPORT: 5432
PGUSER: postgres
run: ./scripts/setup-test-template.sh
- name: Run Clippy
env:
DATABASE_URL: postgres://postgres:postgres@postgres:5432/soar_test
run: cargo clippy --all-targets --all-features -- -D warnings
- name: Run Rust tests
env:
TEST_DATABASE_URL: postgres://postgres:postgres@postgres:5432/soar_test
ELEVATION_DATA_PATH: /tmp/elevation
ELEVATION_S3_BUCKET: elevation-tiles-prod
ELEVATION_S3_PREFIX: skadi
AWS_REGION: us-east-1
run: cargo nextest run
build-release:
name: Build Release Binary (Native Static musl)
runs-on: self-hosted
container:
image: ghcr.io/hut8/soar-ci:latest
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
volumes:
- /opt/ci-cache:/ci-cache
defaults:
run:
shell: bash
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for git tags (needed for vergen version)
- name: Setup persistent cargo cache
run: |
mkdir -p /ci-cache/${RUNNER_NAME}/cargo-registry /ci-cache/${RUNNER_NAME}/cargo-git
ln -sfn /ci-cache/${RUNNER_NAME}/cargo-registry /usr/local/cargo/registry
ln -sfn /ci-cache/${RUNNER_NAME}/cargo-git /usr/local/cargo/git
echo "CARGO_TARGET_DIR=/ci-cache/${RUNNER_NAME}/target-release-musl" >> $GITHUB_ENV
- name: Install unzip (required by setup-bun in container)
run: apt-get update -qq && apt-get install -y -qq unzip
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Install web dependencies
working-directory: ./web
run: bun install --frozen-lockfile
- name: Build SvelteKit project
working-directory: ./web
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
run: bun run build
- name: Build static release binary
env:
SKIP_WEB_BUILD: "1" # Frontend already built via bun run build above
# CRITICAL: tokio_unstable is required for tokio-console support
# Without this flag, --enable-tokio-console will cause exit code 101 with no output
# CRITICAL: force-frame-pointers=yes is required for Pyroscope profiling
# Without frame pointers, flame graphs show unhelpful "[unknown]" frames
RUSTFLAGS: "--cfg tokio_unstable -C target-feature=+crt-static -C link-arg=-static -C force-frame-pointers=yes"
run: cargo build --release --target x86_64-unknown-linux-musl --features bundled-postgres
- name: Verify static linking
run: |
echo "Checking if binary is statically linked..."
file $CARGO_TARGET_DIR/x86_64-unknown-linux-musl/release/soar
# Check for dynamic dependencies (should show "statically linked")
if ldd $CARGO_TARGET_DIR/x86_64-unknown-linux-musl/release/soar 2>&1 | grep -q "not a dynamic executable"; then
echo "Binary is statically linked (no dynamic dependencies)"
else
echo "Binary has dynamic dependencies:"
ldd $CARGO_TARGET_DIR/x86_64-unknown-linux-musl/release/soar || true
fi
- name: Create binary archive
run: |
rm -rf release
mkdir -p release
cp $CARGO_TARGET_DIR/x86_64-unknown-linux-musl/release/soar release/
cp README.md release/ || echo "No README.md found"
tar -czf soar-linux-x64.tar.gz -C release .
- name: Upload release binary
uses: actions/upload-artifact@v7
with:
name: soar-linux-x64
path: soar-linux-x64.tar.gz
retention-days: 30
- name: Show binary info
run: |
echo "Binary size:"
ls -lh $CARGO_TARGET_DIR/x86_64-unknown-linux-musl/release/soar
echo ""
echo "Binary info:"
file $CARGO_TARGET_DIR/x86_64-unknown-linux-musl/release/soar
echo ""
echo "Stripped binary size:"
cp $CARGO_TARGET_DIR/x86_64-unknown-linux-musl/release/soar /tmp/soar-stripped
strip /tmp/soar-stripped
ls -lh /tmp/soar-stripped
security-audit:
name: Security Audit
runs-on: self-hosted
container:
image: ghcr.io/hut8/soar-ci:latest
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
volumes:
- /opt/ci-cache:/ci-cache
defaults:
run:
shell: bash
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for git tags (needed for vergen version)
- name: Setup persistent cargo cache
run: |
mkdir -p /ci-cache/${RUNNER_NAME}/cargo-registry /ci-cache/${RUNNER_NAME}/cargo-git
ln -sfn /ci-cache/${RUNNER_NAME}/cargo-registry /usr/local/cargo/registry
ln -sfn /ci-cache/${RUNNER_NAME}/cargo-git /usr/local/cargo/git
- name: Run security audit
run: cargo audit
- name: Check for outdated dependencies
run: cargo outdated --exit-code 1 || echo "Some dependencies are outdated (non-critical)"
build-android:
name: Build Android App
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./android
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: '17'
distribution: 'temurin'
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4
- name: Build debug APK
run: ./gradlew assembleDebug
- name: Run Android lint
run: ./gradlew lint
- name: Upload debug APK
uses: actions/upload-artifact@v7
with:
name: soar-tracker-debug
path: android/app/build/outputs/apk/debug/*.apk
retention-days: 30
deploy-staging:
name: Deploy to Staging
runs-on: ubuntu-latest
needs: [test-sveltekit, test-e2e, test-rust, build-release, security-audit]
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for git tags (needed for vergen version)
- name: Download release binary
uses: actions/download-artifact@v8
with:
name: soar-linux-x64
path: ./
- name: Extract binary
run: |
tar -xzf soar-linux-x64.tar.gz
chmod +x soar
- name: Setup SSH
run: |
mkdir -p ~/.ssh
echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H staging.glider.flights >> ~/.ssh/known_hosts
- name: Prepare deployment package
run: |
# Create timestamped deployment directory name
TIMESTAMP=$(date +%Y%m%d%H%M%S)
echo "DEPLOY_TIMESTAMP=$TIMESTAMP" >> $GITHUB_ENV
# Create local deployment directory
mkdir -p deploy-pkg
cp soar deploy-pkg/
cp infrastructure/soar-deploy deploy-pkg/
cp infrastructure/systemd/*.service deploy-pkg/ || echo "No service files found"
cp infrastructure/systemd/*.timer deploy-pkg/ || echo "No timer files found"
cp infrastructure/systemd/*.target deploy-pkg/ || echo "No target files found"
# Copy systemd directory (includes template files for Grafana SMTP config)
if [ -d infrastructure/systemd ]; then
mkdir -p deploy-pkg/systemd
cp infrastructure/systemd/*.template deploy-pkg/systemd/ 2>/dev/null || echo "No systemd template files found"
echo "Systemd template files included in deployment package"
fi
# Copy Prometheus job configuration files
if [ -d infrastructure/prometheus-jobs ]; then
cp -r infrastructure/prometheus-jobs deploy-pkg/
echo "Prometheus job files included in deployment package"
fi
# Copy Grafana provisioning configuration
if [ -d infrastructure/grafana-provisioning ]; then
cp -r infrastructure/grafana-provisioning deploy-pkg/
echo "Grafana provisioning configuration included in deployment package"
fi
# Build Grafana dashboards from panel files
echo "Building Grafana dashboards..."
python3 infrastructure/dashboards/build.py
echo "Dashboard build complete"
# Copy Grafana dashboard files
if compgen -G "infrastructure/grafana-dashboard-*.json" > /dev/null; then
cp infrastructure/grafana-dashboard-*.json deploy-pkg/
DASHBOARD_COUNT=$(ls -1 infrastructure/grafana-dashboard-*.json | wc -l)
echo "Grafana dashboards included in deployment package: $DASHBOARD_COUNT"
fi
# Copy observability stack configuration files
for config_file in tempo-config.yml loki-config.yml pyroscope-config.yml alloy-config.alloy.template; do
if [ -f "infrastructure/$config_file" ]; then
cp "infrastructure/$config_file" deploy-pkg/
echo "Observability config included: $config_file"
else
echo "Warning: infrastructure/$config_file not found"
fi
done
# Copy backup scripts directory
if [ -d scripts/backup ]; then
mkdir -p deploy-pkg/scripts
cp -r scripts/backup deploy-pkg/scripts/
echo "Backup scripts included in deployment package"
fi
# Create version file with commit SHA (short form for Sentry)
echo "${GITHUB_SHA:0:7}" > deploy-pkg/VERSION
echo "Deployment package prepared for timestamp: $TIMESTAMP"
echo "Commit SHA (short): ${GITHUB_SHA:0:7}"
ls -lh deploy-pkg/
if [ -d deploy-pkg/prometheus-jobs ]; then
echo "Prometheus jobs:"
ls -lh deploy-pkg/prometheus-jobs/
fi
if [ -d deploy-pkg/grafana-provisioning ]; then
echo "Grafana provisioning:"
ls -lh deploy-pkg/grafana-provisioning/dashboards/
fi
if [ -d deploy-pkg/scripts/backup ]; then
echo "Backup scripts:"
ls -lh deploy-pkg/scripts/backup/
fi
- name: Upload deployment package
run: |
DEPLOY_DIR="/tmp/soar/deploy/${{ env.DEPLOY_TIMESTAMP }}"
echo "Creating deployment directory on server: $DEPLOY_DIR"
ssh soar@staging.glider.flights "mkdir -p $DEPLOY_DIR"
echo "Uploading deployment package..."
rsync -az --info=progress2 deploy-pkg/ soar@staging.glider.flights:$DEPLOY_DIR/
echo "Deployment package uploaded successfully"
- name: Execute deployment to staging
run: |
DEPLOY_DIR="/tmp/soar/deploy/${{ env.DEPLOY_TIMESTAMP }}"
DEPLOY_ID="staging-${{ env.DEPLOY_TIMESTAMP }}"
LOG_FILE="/var/lib/soar/deploy/logs/${DEPLOY_ID}.log"
STATUS_FILE="/var/lib/soar/deploy/status/${DEPLOY_ID}.status"
echo "Executing deployment script for staging..."
echo "Deployment ID: $DEPLOY_ID"
echo "Log file: $LOG_FILE"
# Run deployment in background with nohup to survive SSH disconnection
# soar-deploy creates log/status directories and redirects its own output
ssh soar@staging.glider.flights "nohup sudo /usr/local/bin/soar-deploy staging $DEPLOY_DIR &"
# Give the background process a moment to start
sleep 3
echo "Deployment started in background, polling for completion..."
# Poll for completion (check for status file every 10 seconds, timeout after 30 minutes)
MAX_WAIT=1800
ELAPSED=0
POLL_INTERVAL=10
while [ $ELAPSED -lt $MAX_WAIT ]; do
# Check if status file exists
if ssh soar@staging.glider.flights "test -f $STATUS_FILE" 2>/dev/null; then
echo "Deployment completed, fetching results..."
# Get the exit code from status file
EXIT_CODE=$(ssh soar@staging.glider.flights "cat $STATUS_FILE")
echo "Deployment exit code: $EXIT_CODE"
# Show the deployment log
echo ""
echo "=== Deployment Log ==="
ssh soar@staging.glider.flights "cat $LOG_FILE" || echo "(Failed to retrieve log)"
echo "=== End Deployment Log ==="
echo ""
# Exit with the same code as the deployment
exit $EXIT_CODE
fi
# Show progress
if [ $((ELAPSED % 60)) -eq 0 ] && [ $ELAPSED -gt 0 ]; then
echo "Still waiting for deployment... (${ELAPSED}s elapsed)"
# Show last few lines of log if available
ssh soar@staging.glider.flights "tail -5 $LOG_FILE 2>/dev/null" || true
fi
sleep $POLL_INTERVAL
ELAPSED=$((ELAPSED + POLL_INTERVAL))
done
echo "Deployment timed out after ${MAX_WAIT} seconds"
echo "Deployment may still be running on the server"
echo "Check: ssh soar@staging.glider.flights 'cat $LOG_FILE'"
exit 1
- name: Cleanup SSH
if: always()
run: |
rm -f ~/.ssh/id_rsa
deploy-production:
name: Deploy to Production
runs-on: ubuntu-latest
needs: [test-sveltekit, test-e2e, test-rust, build-release, security-audit]
if: github.event_name == 'release' && github.event.action == 'published'
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0 # Fetch all history for git tags (needed for vergen version)
- name: Download release binary
uses: actions/download-artifact@v8
with:
name: soar-linux-x64
path: ./
- name: Extract binary
run: |
tar -xzf soar-linux-x64.tar.gz
chmod +x soar
- name: Setup SSH
run: |
mkdir -p ~/.ssh
echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H glider.flights >> ~/.ssh/known_hosts
- name: Prepare deployment package
run: |
# Create timestamped deployment directory name
TIMESTAMP=$(date +%Y%m%d%H%M%S)
echo "DEPLOY_TIMESTAMP=$TIMESTAMP" >> $GITHUB_ENV
# Create local deployment directory
mkdir -p deploy-pkg
cp soar deploy-pkg/
cp infrastructure/soar-deploy deploy-pkg/
cp infrastructure/systemd/*.service deploy-pkg/ || echo "No service files found"
cp infrastructure/systemd/*.timer deploy-pkg/ || echo "No timer files found"
cp infrastructure/systemd/*.target deploy-pkg/ || echo "No target files found"
# Copy systemd directory (includes template files for Grafana SMTP config)
if [ -d infrastructure/systemd ]; then
mkdir -p deploy-pkg/systemd
cp infrastructure/systemd/*.template deploy-pkg/systemd/ 2>/dev/null || echo "No systemd template files found"
echo "Systemd template files included in deployment package"
fi
# Copy Prometheus job configuration files
if [ -d infrastructure/prometheus-jobs ]; then
cp -r infrastructure/prometheus-jobs deploy-pkg/
echo "Prometheus job files included in deployment package"
fi
# Copy Grafana provisioning configuration
if [ -d infrastructure/grafana-provisioning ]; then
cp -r infrastructure/grafana-provisioning deploy-pkg/
echo "Grafana provisioning configuration included in deployment package"
fi
# Build Grafana dashboards from panel files
echo "Building Grafana dashboards..."
python3 infrastructure/dashboards/build.py
echo "Dashboard build complete"
# Copy Grafana dashboard files
if compgen -G "infrastructure/grafana-dashboard-*.json" > /dev/null; then
cp infrastructure/grafana-dashboard-*.json deploy-pkg/
DASHBOARD_COUNT=$(ls -1 infrastructure/grafana-dashboard-*.json | wc -l)
echo "Grafana dashboards included in deployment package: $DASHBOARD_COUNT"
fi
# Copy observability stack configuration files
for config_file in tempo-config.yml loki-config.yml pyroscope-config.yml alloy-config.alloy.template; do
if [ -f "infrastructure/$config_file" ]; then
cp "infrastructure/$config_file" deploy-pkg/
echo "Observability config included: $config_file"
else
echo "Warning: infrastructure/$config_file not found"
fi
done
# Copy backup scripts directory
if [ -d scripts/backup ]; then
mkdir -p deploy-pkg/scripts
cp -r scripts/backup deploy-pkg/scripts/
echo "Backup scripts included in deployment package"
fi
# Create version file with release tag
echo "${GITHUB_REF#refs/tags/}" > deploy-pkg/VERSION
echo "Deployment package prepared for timestamp: $TIMESTAMP"
echo "Release tag: ${GITHUB_REF#refs/tags/}"
ls -lh deploy-pkg/
if [ -d deploy-pkg/prometheus-jobs ]; then
echo "Prometheus jobs:"
ls -lh deploy-pkg/prometheus-jobs/
fi
if [ -d deploy-pkg/grafana-provisioning ]; then
echo "Grafana provisioning:"
ls -lh deploy-pkg/grafana-provisioning/dashboards/
fi
if [ -d deploy-pkg/scripts/backup ]; then
echo "Backup scripts:"
ls -lh deploy-pkg/scripts/backup/
fi
- name: Upload deployment package
run: |
DEPLOY_DIR="/tmp/soar/deploy/${{ env.DEPLOY_TIMESTAMP }}"
echo "Creating deployment directory on server: $DEPLOY_DIR"
ssh soar@glider.flights "mkdir -p $DEPLOY_DIR"
echo "Uploading deployment package..."
rsync -az --info=progress2 deploy-pkg/ soar@glider.flights:$DEPLOY_DIR/
echo "Deployment package uploaded successfully"
- name: Execute deployment to production
run: |
DEPLOY_DIR="/tmp/soar/deploy/${{ env.DEPLOY_TIMESTAMP }}"
DEPLOY_ID="production-${{ env.DEPLOY_TIMESTAMP }}"
LOG_FILE="/var/lib/soar/deploy/logs/${DEPLOY_ID}.log"
STATUS_FILE="/var/lib/soar/deploy/status/${DEPLOY_ID}.status"
echo "Executing deployment script for production..."
echo "Deployment ID: $DEPLOY_ID"
echo "Log file: $LOG_FILE"
# Run deployment in background with nohup to survive SSH disconnection
# soar-deploy creates log/status directories and redirects its own output
ssh soar@glider.flights "nohup sudo /usr/local/bin/soar-deploy production $DEPLOY_DIR &"
# Give the background process a moment to start
sleep 3
echo "Deployment started in background, polling for completion..."
# Poll for completion (check for status file every 10 seconds, timeout after 30 minutes)
MAX_WAIT=1800
ELAPSED=0
POLL_INTERVAL=10
while [ $ELAPSED -lt $MAX_WAIT ]; do
# Check if status file exists
if ssh soar@glider.flights "test -f $STATUS_FILE" 2>/dev/null; then
echo "Deployment completed, fetching results..."
# Get the exit code from status file
EXIT_CODE=$(ssh soar@glider.flights "cat $STATUS_FILE")
echo "Deployment exit code: $EXIT_CODE"
# Show the deployment log
echo ""
echo "=== Deployment Log ==="
ssh soar@glider.flights "cat $LOG_FILE" || echo "(Failed to retrieve log)"
echo "=== End Deployment Log ==="
echo ""
# Exit with the same code as the deployment
exit $EXIT_CODE
fi
# Show progress
if [ $((ELAPSED % 60)) -eq 0 ] && [ $ELAPSED -gt 0 ]; then
echo "Still waiting for deployment... (${ELAPSED}s elapsed)"
# Show last few lines of log if available
ssh soar@glider.flights "tail -5 $LOG_FILE 2>/dev/null" || true
fi
sleep $POLL_INTERVAL
ELAPSED=$((ELAPSED + POLL_INTERVAL))
done
echo "Deployment timed out after ${MAX_WAIT} seconds"
echo "Deployment may still be running on the server"
echo "Check: ssh soar@glider.flights 'cat $LOG_FILE'"
exit 1
- name: Cleanup SSH
if: always()
run: |
rm -f ~/.ssh/id_rsa
build-release-arm64:
name: Build Release Binary (ARM64 Static musl)
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Bun
uses: oven-sh/setup-bun@v2
- name: Install web dependencies
working-directory: ./web
run: bun install --frozen-lockfile
- name: Build SvelteKit project
working-directory: ./web
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
run: bun run build
- name: Install Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
target: aarch64-unknown-linux-musl
cache: false # Disable built-in cache; cross builds inside Docker so host-only cache is useless
- name: Setup Rust cache
uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
shared-key: "release-build-arm64-musl"
save-if: ${{ github.ref == 'refs/heads/main' }}
cache-all-crates: true
- name: Install cross
uses: taiki-e/install-action@v2
with:
tool: cross@0.2.5
- name: Log in to GHCR (for custom cross image)
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build static release binary (ARM64)
env:
CROSS_CONTAINER_ENGINE: docker
# Mount host cargo cache into cross container so cached crate downloads are reused
CROSS_CONTAINER_OPTS: "-v /home/runner/.cargo/registry:/root/.cargo/registry -v /home/runner/.cargo/git:/root/.cargo/git"
SKIP_WEB_BUILD: "1"
RUSTFLAGS: "--cfg tokio_unstable -C target-feature=+crt-static -C link-arg=-static -C force-frame-pointers=yes"
run: cross build --release --target aarch64-unknown-linux-musl --features bundled-postgres
- name: Create binary archive
run: |
rm -rf release
mkdir -p release
cp target/aarch64-unknown-linux-musl/release/soar release/
tar -czf soar-linux-arm64.tar.gz -C release .
- name: Upload release binary
uses: actions/upload-artifact@v7
with:
name: soar-linux-arm64
path: soar-linux-arm64.tar.gz
retention-days: 30
- name: Show binary info
run: |
ls -lh target/aarch64-unknown-linux-musl/release/soar
file target/aarch64-unknown-linux-musl/release/soar
deploy-radar-staging:
name: Deploy Radar (Staging)
runs-on: ubuntu-latest
needs: [build-release-arm64]
# Deploy staging binary to radar on main push (same trigger as deploy-staging)
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Download ARM64 release binary
uses: actions/download-artifact@v8
with:
name: soar-linux-arm64
path: ./
- name: Extract binary
run: |
tar -xzf soar-linux-arm64.tar.gz
chmod +x soar
- name: Setup SSH
run: |
mkdir -p ~/.ssh
echo "${{ secrets.RADAR_SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H radar.hut8.tools >> ~/.ssh/known_hosts
- name: Prepare deployment package
run: |
TIMESTAMP=$(date +%Y%m%d%H%M%S)
echo "DEPLOY_TIMESTAMP=$TIMESTAMP" >> $GITHUB_ENV
mkdir -p deploy-pkg
cp soar deploy-pkg/
cp infrastructure/soar-deploy deploy-pkg/
cp infrastructure/ingest-radar-staging.toml deploy-pkg/
cp infrastructure/systemd/soar-ingest-radar-staging.service deploy-pkg/
echo "Deployment package prepared for timestamp: $TIMESTAMP"
ls -lh deploy-pkg/
- name: Upload deployment package
run: |
DEPLOY_DIR="/tmp/soar/deploy/${{ env.DEPLOY_TIMESTAMP }}"
echo "Creating deployment directory on radar: $DEPLOY_DIR"
ssh soar@radar.hut8.tools "mkdir -p $DEPLOY_DIR"
echo "Uploading deployment package..."
rsync -az --info=progress2 deploy-pkg/ soar@radar.hut8.tools:$DEPLOY_DIR/
echo "Deployment package uploaded successfully"
- name: Execute deployment to radar
run: |
DEPLOY_DIR="/tmp/soar/deploy/${{ env.DEPLOY_TIMESTAMP }}"
DEPLOY_ID="radar-staging-${{ env.DEPLOY_TIMESTAMP }}"
LOG_FILE="/var/lib/soar/deploy/logs/${DEPLOY_ID}.log"
STATUS_FILE="/var/lib/soar/deploy/status/${DEPLOY_ID}.status"
echo "Executing deployment script for radar-staging..."
echo "Deployment ID: $DEPLOY_ID"
# Bootstrap: install the repo's soar-deploy so /usr/local/bin/soar-deploy
# is current before we invoke it (older versions may reject new env names)
ssh soar@radar.hut8.tools "sudo install -m 755 $DEPLOY_DIR/soar-deploy /usr/local/bin/soar-deploy"
ssh soar@radar.hut8.tools "nohup sudo /usr/local/bin/soar-deploy radar-staging $DEPLOY_DIR &"
sleep 3
echo "Deployment started in background, polling for completion..."
MAX_WAIT=300
ELAPSED=0
POLL_INTERVAL=5
while [ $ELAPSED -lt $MAX_WAIT ]; do
if ssh soar@radar.hut8.tools "test -f $STATUS_FILE" 2>/dev/null; then
echo "Deployment completed, fetching results..."
EXIT_CODE=$(ssh soar@radar.hut8.tools "cat $STATUS_FILE")
echo "Deployment exit code: $EXIT_CODE"
echo ""
echo "=== Deployment Log ==="
ssh soar@radar.hut8.tools "cat $LOG_FILE" || echo "(Failed to retrieve log)"
echo "=== End Deployment Log ==="
echo ""
exit $EXIT_CODE
fi
if [ $((ELAPSED % 30)) -eq 0 ] && [ $ELAPSED -gt 0 ]; then
echo "Still waiting for deployment... (${ELAPSED}s elapsed)"
ssh soar@radar.hut8.tools "tail -5 $LOG_FILE 2>/dev/null" || true
fi
sleep $POLL_INTERVAL
ELAPSED=$((ELAPSED + POLL_INTERVAL))
done
echo "Deployment timed out after ${MAX_WAIT} seconds"
exit 1
- name: Cleanup SSH
if: always()
run: |
rm -f ~/.ssh/id_rsa
deploy-radar-production:
name: Deploy Radar (Production)
runs-on: ubuntu-latest
needs: [build-release-arm64]
# Deploy production binary to radar on release (same trigger as deploy-production)
if: github.event_name == 'release' && github.event.action == 'published'
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Download ARM64 release binary
uses: actions/download-artifact@v8
with:
name: soar-linux-arm64
path: ./
- name: Extract binary
run: |
tar -xzf soar-linux-arm64.tar.gz
chmod +x soar
- name: Setup SSH
run: |
mkdir -p ~/.ssh
echo "${{ secrets.RADAR_SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H radar.hut8.tools >> ~/.ssh/known_hosts
- name: Prepare deployment package
run: |
TIMESTAMP=$(date +%Y%m%d%H%M%S)
echo "DEPLOY_TIMESTAMP=$TIMESTAMP" >> $GITHUB_ENV
mkdir -p deploy-pkg
cp soar deploy-pkg/
cp infrastructure/soar-deploy deploy-pkg/
cp infrastructure/ingest-radar-production.toml deploy-pkg/
cp infrastructure/systemd/soar-ingest-radar-production.service deploy-pkg/
echo "Deployment package prepared for timestamp: $TIMESTAMP"
ls -lh deploy-pkg/
- name: Upload deployment package
run: |
DEPLOY_DIR="/tmp/soar/deploy/${{ env.DEPLOY_TIMESTAMP }}"
echo "Creating deployment directory on radar: $DEPLOY_DIR"
ssh soar@radar.hut8.tools "mkdir -p $DEPLOY_DIR"
echo "Uploading deployment package..."
rsync -az --info=progress2 deploy-pkg/ soar@radar.hut8.tools:$DEPLOY_DIR/
echo "Deployment package uploaded successfully"
- name: Execute deployment to radar
run: |
DEPLOY_DIR="/tmp/soar/deploy/${{ env.DEPLOY_TIMESTAMP }}"
DEPLOY_ID="radar-production-${{ env.DEPLOY_TIMESTAMP }}"
LOG_FILE="/var/lib/soar/deploy/logs/${DEPLOY_ID}.log"
STATUS_FILE="/var/lib/soar/deploy/status/${DEPLOY_ID}.status"
echo "Executing deployment script for radar-production..."
echo "Deployment ID: $DEPLOY_ID"
# Bootstrap: install the repo's soar-deploy so /usr/local/bin/soar-deploy
# is current before we invoke it (older versions may reject new env names)
ssh soar@radar.hut8.tools "sudo install -m 755 $DEPLOY_DIR/soar-deploy /usr/local/bin/soar-deploy"
ssh soar@radar.hut8.tools "nohup sudo /usr/local/bin/soar-deploy radar-production $DEPLOY_DIR &"
sleep 3
echo "Deployment started in background, polling for completion..."
MAX_WAIT=300
ELAPSED=0
POLL_INTERVAL=5
while [ $ELAPSED -lt $MAX_WAIT ]; do
if ssh soar@radar.hut8.tools "test -f $STATUS_FILE" 2>/dev/null; then
echo "Deployment completed, fetching results..."
EXIT_CODE=$(ssh soar@radar.hut8.tools "cat $STATUS_FILE")
echo "Deployment exit code: $EXIT_CODE"
echo ""
echo "=== Deployment Log ==="
ssh soar@radar.hut8.tools "cat $LOG_FILE" || echo "(Failed to retrieve log)"
echo "=== End Deployment Log ==="
echo ""
exit $EXIT_CODE
fi
if [ $((ELAPSED % 30)) -eq 0 ] && [ $ELAPSED -gt 0 ]; then
echo "Still waiting for deployment... (${ELAPSED}s elapsed)"
ssh soar@radar.hut8.tools "tail -5 $LOG_FILE 2>/dev/null" || true
fi
sleep $POLL_INTERVAL
ELAPSED=$((ELAPSED + POLL_INTERVAL))
done
echo "Deployment timed out after ${MAX_WAIT} seconds"
exit 1
- name: Cleanup SSH
if: always()
run: |
rm -f ~/.ssh/id_rsa