refactor: eliminate test-e2e.sh script in favor of pytest fixture #73
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: discogsography | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 1 * * 6" | |
| env: | |
| CI: true | |
| COMPOSE_VERSION: "v2.32.0" | |
| PLAYWRIGHT_BROWSERS_PATH: "0" # Use default location | |
| PYTHON_VERSION: "3.13" | |
| REGISTRY: ghcr.io | |
| jobs: | |
| list-sub-projects: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Get all sub-projects | |
| id: set-matrix | |
| run: | | |
| # Only include actual service directories (sorted alphabetically) | |
| services=("dashboard" "extractor" "graphinator" "tableinator") | |
| matrix=$(printf "%s\n" "${services[@]}" | jq -R -s -c 'split("\n")[:-1]') | |
| echo "matrix=$matrix" >> "$GITHUB_OUTPUT" | |
| code-quality: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Set lowercase image name | |
| id: image | |
| run: | | |
| echo "IMAGE_NAME=$(echo "${{ github.actor }}/discogsography" | tr "[:upper:]" "[:lower:]")" >> "$GITHUB_ENV" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1 | |
| with: | |
| version: latest | |
| enable-cache: true | |
| cache-dependency-glob: "**/uv.lock" | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/uv | |
| .venv | |
| key: ${{ runner.os }}-uv-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv- | |
| - name: Cache pre-commit | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pre-commit- | |
| - name: Cache pytest | |
| uses: actions/cache@v4 | |
| with: | |
| path: .pytest_cache | |
| key: ${{ runner.os }}-pytest-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-pytest- | |
| - name: Cache arkade tools | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.arkade | |
| key: ${{ runner.os }}-arkade-${{ hashFiles('.github/workflows/build.yml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-arkade- | |
| - name: Install arkade | |
| uses: alexellis/arkade-get@1eef818e467c387d3f50cfe0d2c565d1cbe82b03 # master | |
| with: | |
| hadolint: latest | |
| - name: Cache docker-compose | |
| uses: actions/cache@v4 | |
| with: | |
| path: /usr/local/bin/docker-compose | |
| key: ${{ runner.os }}-docker-compose-${{ hashFiles('.github/workflows/build.yml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-docker-compose- | |
| - name: Install docker-compose | |
| run: | | |
| if ! command -v docker-compose &> /dev/null; then | |
| echo "Installing docker-compose..." | |
| sudo curl -L "https://github.com/docker/compose/releases/download/${{ env.COMPOSE_VERSION }}/docker-compose-$(uname -s)-$(uname -m)" \ | |
| -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| fi | |
| docker-compose --version | |
| - name: Install dependencies | |
| run: | | |
| uv sync --all-extras --frozen | |
| # Install workspace packages | |
| uv pip install -e dashboard | |
| uv pip install -e common | |
| - name: Run pre-commit hooks | |
| run: | | |
| uv run pre-commit run --all-files | |
| - name: Run tests with coverage (excluding E2E) | |
| run: | | |
| uv run pytest --cov -m "not e2e" | |
| - name: Validate docker-compose files | |
| run: | | |
| # Check main docker-compose.yml | |
| docker-compose config --quiet | |
| echo "✅ docker-compose.yml is valid" | |
| # Check production overlay if it exists | |
| if [ -f docker-compose.prod.yml ]; then | |
| docker-compose -f docker-compose.yml -f docker-compose.prod.yml config --quiet | |
| echo "✅ docker-compose.prod.yml overlay is valid" | |
| fi | |
| # Validate all services are defined | |
| services=$(docker-compose config --services) | |
| expected_services="dashboard extractor graphinator neo4j postgres rabbitmq tableinator" | |
| for service in $expected_services; do | |
| if ! echo "$services" | grep -q "^$service$"; then | |
| echo "❌ Missing expected service: $service" | |
| exit 1 | |
| fi | |
| done | |
| echo "✅ All expected services are defined" | |
| build-discogsography: | |
| needs: [list-sub-projects, code-quality] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| sub-project: ${{ fromJson(needs.list-sub-projects.outputs.matrix) }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Set lowercase image name | |
| id: image | |
| run: | | |
| echo "IMAGE_NAME=$(echo "${{ github.actor }}/discogsography" | tr "[:upper:]" "[:lower:]")" >> "$GITHUB_ENV" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Log in to the GitHub Container Registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GHCR_TOKEN }} | |
| - name: Anchore security scan - discogsography/${{ matrix.sub-project }} | |
| uses: anchore/scan-action@16910ac423301c6d30554b83a7f71ac6ff4a51f3 # v6.4.0 | |
| with: | |
| path: ${{ matrix.sub-project }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1 | |
| with: | |
| version: latest | |
| enable-cache: true | |
| cache-dependency-glob: "**/uv.lock" | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/uv | |
| .venv | |
| key: ${{ runner.os }}-uv-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv- | |
| - name: Cache Docker layers | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ matrix.sub-project }}-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx-${{ matrix.sub-project }}- | |
| ${{ runner.os }}-buildx- | |
| - name: Generate uv.lock for Docker build | |
| run: uv sync --frozen | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.sub-project }} | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=schedule,pattern={{date 'YYYYMMDD'}} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| platforms: linux/amd64, linux/arm64 | |
| driver-opts: | | |
| image=moby/buildkit:latest | |
| network=host | |
| - name: Build and push Docker image to GitHub Container Registry - discogsography/${{ matrix.sub-project }} | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ${{ matrix.sub-project }}/Dockerfile | |
| platforms: linux/amd64, linux/arm64 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| provenance: true | |
| sbom: true | |
| cache-from: | | |
| type=gha | |
| type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.sub-project }}:buildcache | |
| type=local,src=/tmp/.buildx-cache | |
| cache-to: | | |
| type=gha,mode=max | |
| type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ matrix.sub-project }}:buildcache,mode=max | |
| type=local,dest=/tmp/.buildx-cache-new,mode=max | |
| build-args: | | |
| BUILDKIT_INLINE_CACHE=1 | |
| PYTHON_VERSION=${{ env.PYTHON_VERSION }} | |
| BUILD_DATE=${{ github.event.head_commit.timestamp || github.event.repository.updated_at }} | |
| BUILD_VERSION=${{ steps.meta.outputs.version }} | |
| VCS_REF=${{ github.sha }} | |
| - name: Move cache | |
| run: | | |
| rm -rf /tmp/.buildx-cache | |
| mv /tmp/.buildx-cache-new /tmp/.buildx-cache | |
| - name: Send notification to Discord | |
| uses: sarisia/actions-status-discord@5ddd3b114a98457dd80a39b2f00b6a998cd69008 # v1.15.3 | |
| if: always() | |
| with: | |
| title: discogsography/${{ matrix.sub-project }} | |
| webhook: ${{ secrets.DISCORD_WEBHOOK }} |