Nightly Pipeline #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Pipeline | |
| on: | |
| schedule: | |
| - cron: '0 6 * * *' # 6am UTC daily | |
| workflow_dispatch: # Manual trigger | |
| jobs: | |
| pipeline: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Download previous DB | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: agentrank-db | |
| path: data/ | |
| continue-on-error: true # First run has no artifact | |
| - name: Determine crawl mode | |
| id: mode | |
| run: | | |
| DOW=$(date +%u) # 1=Monday, 7=Sunday | |
| if [ "$DOW" = "7" ]; then | |
| echo "mode=full" >> $GITHUB_OUTPUT | |
| else | |
| echo "mode=incremental" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Crawl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_PAT }} | |
| CRAWL_MODE: ${{ steps.mode.outputs.mode }} | |
| GLAMA_MAX_PAGES: '500' | |
| run: cd crawler && npx tsx src/index.ts | |
| timeout-minutes: 100 | |
| - name: Score | |
| run: cd scorer && npx tsx src/index.ts | |
| - name: Generate D1 seed | |
| run: npx tsx scripts/seed-d1.ts | |
| - name: Seed D1 | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }} | |
| run: | | |
| cd site | |
| npx wrangler d1 execute agentrank-db --remote --file=../scripts/d1-schema.sql | |
| npx wrangler d1 execute agentrank-db --remote --file=../data/d1-seed.sql | |
| - name: Build & Deploy | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }} | |
| run: | | |
| cd site | |
| npm run build | |
| npx wrangler pages deploy dist --commit-dirty=true | |
| - name: Upload DB artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: agentrank-db | |
| path: data/agentrank.db | |
| retention-days: 30 |