December data release cleanup (#31) #39
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy to GCP | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| env: | |
| APP_PATH: oda-dashboard # Change for each app | |
| BUCKET: data-apps-one-data | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.13' # Adjust to your required version | |
| cache: 'pip' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| cache: 'npm' | |
| - name: Install Python dependencies | |
| run: uv sync | |
| - name: Install Node dependencies | |
| run: npm ci | |
| - id: date | |
| run: echo "date=$(TZ=America/Los_Angeles date +'%Y-%U')" >> $GITHUB_OUTPUT | |
| - id: cache-data | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| src/.observablehq/cache | |
| src/data/cache | |
| key: data-${{ hashFiles('src/data/*') }}-${{ steps.date.outputs.date }} | |
| - name: Build Observable Framework app | |
| run: uv run npm run build | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: 'projects/650536812276/locations/global/workloadIdentityPools/github/providers/github' | |
| service_account: 'github-deployer@one-data-commons.iam.gserviceaccount.com' | |
| - name: individual-loaders | |
| env: | |
| PYTHONPATH: ${{ github.workspace }} | |
| run: | | |
| uv run python src/data/scripts/sectors_view.py | |
| - name: Cache gcloud SDK | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.config/gcloud | |
| ~/.cache/google-cloud-sdk | |
| key: ${{ runner.os }}-gcloud-${{ hashFiles('**/*.yml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gcloud- | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Upload sectors partitioned dataset | |
| run: | | |
| if [ -d "cdn_files/sectors_view" ]; then | |
| echo "Clearing existing sectors dataset in GCS..." | |
| gsutil -m rm -r gs://${{ env.BUCKET }}/sources/sectors_view/** || true | |
| echo "Uploading sectors partitioned dataset to GCS..." | |
| gsutil -m rsync -r -d cdn_files/sectors_view gs://${{ env.BUCKET }}/sources/sectors_view | |
| else | |
| echo "Warning: cdn_files/sectors_view directory not found" | |
| exit 1 | |
| fi | |
| - name: Deploy to GCS | |
| run: | | |
| gsutil -m rsync -r -c -d dist/ gs://${{ env.BUCKET }}/${{ env.APP_PATH }}/ | |
| - name: Purge Cloudflare cache | |
| if: env.CLOUDFLARE_ZONE_ID != '' && env.CLOUDFLARE_API_TOKEN != '' | |
| env: | |
| CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: | | |
| curl -X POST "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ | |
| -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \ | |
| -H "Content-Type: application/json" \ | |
| --data '{"files":["https://data-apps.one.org/${{ env.APP_PATH }}/"]}' |