Generate and Upload Datasets #758
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate and Upload Datasets | |
| on: | |
| schedule: | |
| # Run every hour | |
| - cron: '0 * * * *' | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: # Allow manual trigger | |
| # Allow GitHub Actions to deploy to Pages | |
| permissions: | |
| contents: read | |
| pages: write | |
| id-token: write | |
| # Allow one concurrent deployment | |
| concurrency: | |
| group: "pages" | |
| cancel-in-progress: true | |
| jobs: | |
| generate-datasets: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install sql2datasets with dependencies | |
| run: | | |
| cd sql2datasets | |
| pip install -e . | |
| - name: Create credentials files | |
| run: | | |
| mkdir -p /tmp/creds | |
| echo "${{ secrets.PGPASS_CONTENT }}" > /tmp/creds/.pgpass | |
| echo "${{ secrets.AWS_CREDS_CONTENT }}" > /tmp/creds/.aws-creds | |
| chmod 600 /tmp/creds/.pgpass | |
| - name: Run sql2csv | |
| run: | | |
| sql2csv \ | |
| --input-sql-dir ${{ github.workspace }}/queries \ | |
| --output-dir /tmp/output \ | |
| --pgsql-creds-file /tmp/creds/.pgpass \ | |
| --aws-creds-file /tmp/creds/.aws-creds | |
| - name: Clean up credentials | |
| if: always() | |
| run: | | |
| rm -rf /tmp/creds | |
| - name: Setup Pages | |
| uses: actions/configure-pages@v4 | |
| - name: Upload site artifact | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: '/tmp/output/site' | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v4 |