production-features-pipeline #1115
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: production-features-pipeline | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 8 * * *' | |
| jobs: | |
| scrape_features: | |
| #runs-on: windows-latest | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout repo content | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| - name: setup python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9.13' | |
| - name: install python packages | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt.main | |
| python -m pip install jupyter nbconvert nbformat scrapingant-client | |
| python -m pip install google-api-python-client google-auth-httplib2 google-auth-oauthlib | |
| - name: execute python workflows from notebook | |
| env: | |
| HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }} | |
| NEPTUNE_API_TOKEN: ${{ secrets.NEPTUNE_API_TOKEN }} | |
| SCRAPINGANT_API_KEY: ${{ secrets.SCRAPINGANT_API_KEY }} | |
| GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} | |
| run: | |
| jupyter nbconvert --to notebook --execute notebooks/09_production_features_pipeline.ipynb | |
| - name: Commit and push changes | |
| shell: bash | |
| run: | | |
| set -e | |
| git config --global user.name 'GitHub Action' | |
| git config --global user.email 'action@github.com' | |
| # Add only files that exist to avoid pathspec errors | |
| for f in \ | |
| data/games.csv \ | |
| data/games_engineered.csv \ | |
| data/games_dashboard.csv \ | |
| data/running_accuracy_metrics.csv \ | |
| data/season_summary_stats.csv; do | |
| [ -f "$f" ] && git add "$f" || true | |
| done | |
| # Commit only if something is staged | |
| if ! git diff --staged --quiet; then | |
| git commit -m "Update new data" | |
| else | |
| echo "No staged changes to commit" | |
| fi | |
| git push | |
| - name: Clone second repository | |
| run: | | |
| # Create a temp directory for the second repo | |
| mkdir -p /tmp/nba_dashboard | |
| # Clone the second repository using PAT | |
| git clone https://${{ secrets.PAT_TOKEN }}@github.com/${{ github.repository_owner }}/nba_dashboard.git /tmp/nba_dashboard | |
| # Ensure the data directory exists in the second repo | |
| mkdir -p /tmp/nba_dashboard/data | |
| - name: Copy specific files to second repository | |
| run: | | |
| # Copy the specific files | |
| cp data/games_dashboard.csv /tmp/nba_dashboard/data/ | |
| cp data/season_summary_stats.csv /tmp/nba_dashboard/data/ | |
| cp data/running_accuracy_metrics.csv /tmp/nba_dashboard/data/ | |
| - name: Commit and push to second repository | |
| run: | | |
| cd /tmp/nba_dashboard | |
| git config --global user.name 'GitHub Action' | |
| git config --global user.email 'action@github.com' | |
| git add data/games_dashboard.csv | |
| git add data/season_summary_stats.csv | |
| git add data/running_accuracy_metrics.csv | |
| # Only commit if there are changes | |
| git diff --staged --quiet || git commit -m "Update dashboard data" | |
| git push | |