chore: update instructions. #69
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: discogsography | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 1 * * 6" | |
| env: | |
| CI: true | |
| COMPOSE_VERSION: "v2.32.0" | |
| PLAYWRIGHT_BROWSERS_PATH: "0" # Use default location | |
| PYTHON_VERSION: "3.13" | |
| REGISTRY: ghcr.io | |
| jobs: | |
| list-sub-projects: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get all sub-projects | |
| id: set-matrix | |
| run: | | |
| # Only include actual service directories (sorted alphabetically) | |
| services=("dashboard" "extractor" "graphinator" "tableinator") | |
| matrix=$(printf "%s\n" "${services[@]}" | jq -R -s -c 'split("\n")[:-1]') | |
| echo "matrix=$matrix" >> "$GITHUB_OUTPUT" | |
| code-quality: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set lowercase image name | |
| id: image | |
| run: | | |
| echo "IMAGE_NAME=$(echo "${{ github.actor }}/discogsography" | tr "[:upper:]" "[:lower:]")" >> "$GITHUB_ENV" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1 | |
| with: | |
| version: latest | |
| enable-cache: true | |
| cache-dependency-glob: "**/uv.lock" | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/uv | |
| .venv | |
| key: ${{ runner.os }}-uv-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv- | |
| - name: Cache pre-commit | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pre-commit- | |
| - name: Cache pytest | |
| uses: actions/cache@v4 | |
| with: | |
| path: .pytest_cache | |
| key: ${{ runner.os }}-pytest-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-pytest- | |
| - name: Cache arkade tools | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.arkade | |
| key: ${{ runner.os }}-arkade-${{ hashFiles('.github/workflows/build.yml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-arkade- | |
| - name: Cache Playwright browsers | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/ms-playwright | |
| ~/Library/Caches/ms-playwright | |
| key: ${{ runner.os }}-playwright-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-playwright- | |
| - name: Install arkade | |
| uses: alexellis/arkade-get@1eef818e467c387d3f50cfe0d2c565d1cbe82b03 # master | |
| with: | |
| hadolint: latest | |
| - name: Cache docker-compose | |
| uses: actions/cache@v4 | |
| with: | |
| path: /usr/local/bin/docker-compose | |
| key: ${{ runner.os }}-docker-compose-${{ hashFiles('.github/workflows/build.yml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-docker-compose- | |
| - name: Install docker-compose | |
| run: | | |
| if ! command -v docker-compose &> /dev/null; then | |
| echo "Installing docker-compose..." | |
| sudo curl -L "https://github.com/docker/compose/releases/download/${{ env.COMPOSE_VERSION }}/docker-compose-$(uname -s)-$(uname -m)" \ | |
| -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| fi | |
| docker-compose --version | |
| - name: Install dependencies | |
| run: | | |
| uv sync --all-extras --frozen | |
| # Install workspace packages | |
| uv pip install -e dashboard | |
| uv pip install -e common | |
| - name: Install Playwright browsers | |
| run: | | |
| # Set Playwright to run in CI mode | |
| export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0 | |
| uv run playwright install chromium | |
| # Install system dependencies for headless Chrome | |
| uv run playwright install-deps chromium | |
| - name: Run pre-commit hooks | |
| run: | | |
| uv run pre-commit run --all-files | |
| - name: Run tests with coverage | |
| run: | | |
| uv run pytest --cov | |
| - name: Validate docker-compose files | |
| run: | | |
| # Check main docker-compose.yml | |
| docker-compose config --quiet | |
| echo "✅ docker-compose.yml is valid" | |
| # Check production overlay if it exists | |
| if [ -f docker-compose.prod.yml ]; then | |
| docker-compose -f docker-compose.yml -f docker-compose.prod.yml config --quiet | |
| echo "✅ docker-compose.prod.yml overlay is valid" | |
| fi | |
| # Validate all services are defined | |
| services=$(docker-compose config --services) | |
| expected_services="dashboard extractor graphinator neo4j postgres rabbitmq tableinator" | |
| for service in $expected_services; do | |
| if ! echo "$services" | grep -q "^$service$"; then | |
| echo "❌ Missing expected service: $service" | |
| exit 1 | |
| fi | |
| done | |
| echo "✅ All expected services are defined" | |
| build-discogsography: | |
| needs: [list-sub-projects, code-quality] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| sub-project: ${{ fromJson(needs.list-sub-projects.outputs.matrix) }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Set lowercase image name | |
| id: image | |
| run: | | |
| echo "IMAGE_NAME=$(echo "${{ github.actor }}/discogsography" | tr "[:upper:]" "[:lower:]")" >> "$GITHUB_ENV" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Log in to the GitHub Container Registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Anchore security scan - discogsography/${{ matrix.sub-project }} | |
| uses: anchore/scan-action@16910ac423301c6d30554b83a7f71ac6ff4a51f3 # v6.4.0 | |
| with: | |
| path: ${{ matrix.sub-project }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1 | |
| with: | |
| version: latest | |
| enable-cache: true | |
| cache-dependency-glob: "**/uv.lock" | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/uv | |
| .venv | |
| key: ${{ runner.os }}-uv-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv- | |
| - name: Cache Docker layers | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ matrix.sub-project }}-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx-${{ matrix.sub-project }}- | |
| ${{ runner.os }}-buildx- | |
| - name: Generate uv.lock for Docker build | |
| run: uv sync --frozen | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.sub-project }} | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=schedule,pattern={{date 'YYYYMMDD'}} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| platforms: linux/amd64, linux/arm64 | |
| driver-opts: | | |
| image=moby/buildkit:latest | |
| network=host | |
| - name: Build and push Docker image to GitHub Container Registry - discogsography/${{ matrix.sub-project }} | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ${{ matrix.sub-project }}/Dockerfile | |
| platforms: linux/amd64, linux/arm64 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| provenance: true | |
| sbom: true | |
| cache-from: | | |
| type=gha | |
| type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.sub-project }}:buildcache | |
| type=local,src=/tmp/.buildx-cache | |
| cache-to: | | |
| type=gha,mode=max | |
| type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.sub-project }}:buildcache,mode=max | |
| type=local,dest=/tmp/.buildx-cache-new,mode=max | |
| build-args: | | |
| BUILDKIT_INLINE_CACHE=1 | |
| PYTHON_VERSION=${{ env.PYTHON_VERSION }} | |
| BUILD_DATE=${{ github.event.head_commit.timestamp || github.event.repository.updated_at }} | |
| BUILD_VERSION=${{ steps.meta.outputs.version }} | |
| VCS_REF=${{ github.sha }} | |
| - name: Move cache | |
| run: | | |
| rm -rf /tmp/.buildx-cache | |
| mv /tmp/.buildx-cache-new /tmp/.buildx-cache | |
| - name: Send notification to Discord | |
| uses: sarisia/actions-status-discord@5ddd3b114a98457dd80a39b2f00b6a998cd69008 # v1.15.3 | |
| if: always() | |
| with: | |
| title: discogsography/${{ matrix.sub-project }} | |
| webhook: ${{ secrets.DISCORD_WEBHOOK }} |