build.yml

---
name: Build

on:
  workflow_dispatch:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  schedule:
    - cron: "0 1 * * 6" # Weekly Saturday 01:00 UTC (full build)
    - cron: "0 4 * * 1" # Weekly Monday 04:00 UTC (security-focused)

env:
  CI: true
  COMPOSE_VERSION: "v2.32.0"
  PYTHON_VERSION: "3.13"
  REGISTRY: ghcr.io

permissions:
  contents: read
  packages: write

jobs:
  # ============================================================================
  # CHANGE DETECTION - Determines if only documentation files were modified
  # ============================================================================
  detect-changes:
    runs-on: ubuntu-latest
    outputs:
      code-changed: ${{ steps.filter.outputs.code-changed }}
    steps:
      - name: 🔀 Checkout repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: 🔍 Detect changed file types
        id: filter
        env:
          EVENT_NAME: ${{ github.event_name }}
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.sha }}
          BEFORE_SHA: ${{ github.event.before }}
        run: |
          # Always run full pipeline for scheduled builds and manual dispatches
          if [[ "$EVENT_NAME" == "schedule" ]] || \
             [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then
            echo "code-changed=true" >> "$GITHUB_OUTPUT"
            echo "⚙️ Scheduled/manual build — running full pipeline"
            exit 0
          fi

          # Get changed files compared to base
          if [[ "$EVENT_NAME" == "pull_request" ]]; then
            CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")
          else
            CHANGED_FILES=$(git diff --name-only "$BEFORE_SHA" "$HEAD_SHA")
          fi

          echo "📝 Changed files:"
          echo "$CHANGED_FILES"

          # Check if any non-markdown files were changed
          NON_MD_FILES=$(echo "$CHANGED_FILES" | grep -v '\.md$' || true)

          if [[ -z "$NON_MD_FILES" ]]; then
            echo "code-changed=false" >> "$GITHUB_OUTPUT"
            echo "📄 Only markdown files changed — skipping tests, security, and docker jobs"
          else
            echo "code-changed=true" >> "$GITHUB_OUTPUT"
            echo "💻 Code changes detected — running full pipeline"
          fi

  list-sub-projects:
    needs: [detect-changes]
    if: needs.detect-changes.outputs.code-changed == 'true'
    uses: ./.github/workflows/list-sub-projects.yml

  run-code-quality:
    uses: ./.github/workflows/code-quality.yml
    secrets: inherit

  run-security:
    needs: [detect-changes, run-code-quality]
    if: needs.detect-changes.outputs.code-changed == 'true'
    uses: ./.github/workflows/security.yml
    secrets: inherit
    permissions:
      contents: read
      security-events: write # Required for SARIF uploads to GitHub Security tab

  run-docker-compose-validate:
    needs: [detect-changes, run-code-quality]
    if: needs.detect-changes.outputs.code-changed == 'true'
    uses: ./.github/workflows/docker-compose-validate.yml
    secrets: inherit

  run-docker-validate:
    needs: [list-sub-projects, detect-changes, run-code-quality]
    if: needs.detect-changes.outputs.code-changed == 'true'
    uses: ./.github/workflows/docker-validate.yml
    secrets: inherit
    with:
      simple-matrix: ${{ needs.list-sub-projects.outputs.simple-matrix }}

  # Test workflow runs all service tests in parallel
  # Each service has its own job that runs independently
  run-tests:
    needs: [detect-changes, run-code-quality]
    if: needs.detect-changes.outputs.code-changed == 'true'
    uses: ./.github/workflows/test.yml
    secrets: inherit
    permissions:
      contents: read
      pull-requests: write  # Required for coverage report comments
      statuses: write  # Required for coverage status creation (only used on PRs)

  run-e2e-tests:
    needs: [detect-changes, run-code-quality]
    if: needs.detect-changes.outputs.code-changed == 'true'
    uses: ./.github/workflows/e2e-test.yml
    secrets: inherit
    permissions:
      contents: read
      pull-requests: write  # Required for coverage report comments
      statuses: write  # Required for coverage status creation (only used on PRs)

  # ============================================================================
  # AGGREGATE RESULTS - Waits for all parallel workflows to complete
  # ============================================================================
  aggregate-results:
    runs-on: ubuntu-latest
    needs:
      - detect-changes
      - run-code-quality
      - run-tests
      - run-e2e-tests
      - run-security
      - run-docker-compose-validate
      - run-docker-validate
    if: always()

    steps:
      - name: 📊 Check all pipeline results
        env:
          CODE_CHANGED: ${{ needs.detect-changes.outputs.code-changed }}
        run: |
          echo "Pipeline Results Summary:"
          echo "  Code Changed:            $CODE_CHANGED"
          echo "  Code Quality:            ${{ needs.run-code-quality.result }}"
          echo "  Tests:                   ${{ needs.run-tests.result }}"
          echo "  E2E Tests:               ${{ needs.run-e2e-tests.result }}"
          echo "  Security:                ${{ needs.run-security.result }}"
          echo "  Docker Compose Validate: ${{ needs.run-docker-compose-validate.result }}"
          echo "  Docker Validate:         ${{ needs.run-docker-validate.result }}"

          # Code quality must always pass
          if [[ "${{ needs.run-code-quality.result }}" == "failure" ]]; then
            echo "❌ Code quality failed"
            exit 1
          fi

          # For docs-only changes, skipped jobs are expected
          if [[ "$CODE_CHANGED" == "false" ]]; then
            echo "📄 Docs-only change — skipped jobs are expected"
            echo "✅ All required pipeline workflows passed!"
            exit 0
          fi

          # For code changes, all jobs must pass (not fail)
          if [[ "${{ needs.run-tests.result }}" == "failure" ]] || \
             [[ "${{ needs.run-e2e-tests.result }}" == "failure" ]] || \
             [[ "${{ needs.run-security.result }}" == "failure" ]] || \
             [[ "${{ needs.run-docker-compose-validate.result }}" == "failure" ]] || \
             [[ "${{ needs.run-docker-validate.result }}" == "failure" ]]; then
            echo "❌ One or more pipeline workflows failed"
            exit 1
          fi

          echo "✅ All pipeline workflows passed!"

  build-discogsography:
    # Wait for aggregate-results (which ensures all quality gates pass) before building
    needs: [list-sub-projects, detect-changes, aggregate-results]
    if: needs.detect-changes.outputs.code-changed == 'true'

    runs-on: ubuntu-latest
    timeout-minutes: 30

    strategy:
      matrix:
        include: ${{ fromJson(needs.list-sub-projects.outputs.matrix) }}

    permissions:
      contents: read
      packages: write

    steps:
      - name: ⏱️ Start timer
        id: timer
        run: echo "start_time=$(date +%s)" >> "$GITHUB_OUTPUT"

      - name: 🏷️ Set lowercase image name
        id: image
        run: |
          echo "IMAGE_NAME=$(echo "${{ github.repository }}" | tr "[:upper:]" "[:lower:]")" >> "$GITHUB_ENV"

      - name: 🔀 Checkout repository
        uses: actions/checkout@v6
        with:
          submodules: true

      - name: 🧹 Free disk space
        run: |
          echo "📊 Disk space before cleanup:"
          df -h

          echo "🗑️ Removing unnecessary software..."
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

          echo "🧹 Cleaning Docker..."
          docker system prune -af --volumes || true

          echo "📊 Disk space after cleanup:"
          df -h

      - name: 🔒 Log in to the GitHub Container Registry
        if: github.event_name != 'pull_request'
        uses: docker/login-action@v4
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: 🛡️ Anchore security scan - discogsography/${{ matrix.name }}
        uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 # v7.4.0
        with:
          path: ${{ matrix.name }}

      - name: 🔧 Setup Python and UV
        uses: ./.github/actions/setup-python-uv
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: 💾 Setup Docker build cache
        id: docker-cache
        uses: ./.github/actions/docker-build-cache
        with:
          service-name: ${{ matrix.name }}
          dockerfile-path: ${{ matrix.name }}/Dockerfile
          use-cache: ${{ matrix.use_cache }}

      - name: 🔧 Generate uv.lock for Docker build
        run: uv sync --frozen

      - name: 📊 Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v6
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.name }}
          tags: |
            type=raw,value=latest,enable={{is_default_branch}}
            type=ref,event=branch
            type=ref,event=pr
            type=schedule,pattern={{date 'YYYYMMDD'}}

      - name: 🔧 Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
        with:
          platforms: linux/amd64
          driver-opts: |
            image=moby/buildkit:latest
            network=host

      - name: 🚀 Build and push Docker image to GitHub Container Registry - discogsography/${{ matrix.name }}
        uses: docker/build-push-action@v7
        with:
          context: .
          file: ${{ matrix.name }}/Dockerfile
          platforms: linux/amd64
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          provenance: true
          sbom: true
          cache-from: ${{ steps.docker-cache.outputs.cache-from }}
          cache-to: ${{ steps.docker-cache.outputs.cache-to }}
          build-args: |
            BUILDKIT_INLINE_CACHE=1
            BUILDKIT_CACHE_MOUNT_NS=discogsography
            DOCKER_BUILDKIT=1
            PYTHON_VERSION=${{ env.PYTHON_VERSION }}
            BUILD_DATE=${{ github.event.head_commit.timestamp || github.event.repository.updated_at }}
            BUILD_VERSION=${{ steps.meta.outputs.version }}
            VCS_REF=${{ github.sha }}

      - # Temp fix
        # https://github.com/docker/build-push-action/issues/252
        # https://github.com/moby/buildkit/issues/1896
        name: 💾 Move cache
        if: ${{ matrix.use_cache }}
        run: |
          rm -rf ${{ runner.temp }}/.buildx-cache
          mv ${{ runner.temp }}/.buildx-cache-new ${{ runner.temp }}/.buildx-cache

      - name: 🧹 Cleanup build artifacts
        if: always()
        run: |
          echo "🗑️ Pruning Docker build cache..."
          docker buildx prune -f --filter until=1h || true
          docker system prune -f --filter until=1h || true

          echo "📊 Remaining disk space:"
          df -h

      - name: 📊 Collect metrics
        if: always()
        run: |
          end_time=$(date +%s)
          duration=$((end_time - ${{ steps.timer.outputs.start_time }}))
          echo "::notice title=Build Metrics::Service: ${{ matrix.name }}, Duration: ${duration}s, Cache Used: ${{ matrix.use_cache }}"

          # Check cache hit rate
          if [[ -n "${{ steps.docker-cache.outputs.cache-hit }}" ]]; then
            echo "::notice title=Cache Performance::Docker cache hit for ${{ matrix.name }}"
          fi

      - name: 📢 Send notification to Discord
        uses: sarisia/actions-status-discord@eb045afee445dc055c18d3d90bd0f244fd062708 # v1.16.0
        if: always()
        with:
          title: discogsography/${{ matrix.name }}
          description: |
            Build duration: ${{ steps.timer.outputs.duration || 'N/A' }}s
            Cache used: ${{ matrix.use_cache }}
          webhook: ${{ secrets.DISCORD_WEBHOOK }}