Settings, Config, and Minimal Analytics Hooks #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scrape Bookmarks | ||
| on: | ||
| schedule: | ||
| # Run twice daily at 9 AM and 9 PM UTC | ||
| - cron: '0 9,21 * * *' | ||
| workflow_dispatch: # Allow manual triggering | ||
| inputs: | ||
| test_mode: | ||
| description: 'Run in test mode with fixtures' | ||
| required: false | ||
| default: 'true' | ||
| type: boolean | ||
| jobs: | ||
| scrape-twitter-bookmarks: | ||
| name: Scrape Twitter Bookmarks | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Setup Node.js | ||
| uses: actions/setup-node@v4 | ||
| with: | ||
| node-version: '20' | ||
| cache: 'npm' | ||
| - name: Install dependencies | ||
| run: npm ci | ||
| - name: Create fixtures directory | ||
| run: mkdir -p fixtures | ||
| - name: Generate fixture filename | ||
| id: fixture | ||
| run: echo "filename=twitter-bookmarks-$(date +%Y%m%d-%H%M%S).json" >> $GITHUB_OUTPUT | ||
| - name: Scrape Twitter bookmarks (Mock Mode) | ||
| run: | | ||
| if [ "${{ github.event.inputs.test_mode }}" = "true" ] || [ "${{ github.event_name }}" = "schedule" ]; then | ||
| echo "🔍 Running Twitter scraper in mock mode..." | ||
| node automation/twitter_scraper_adapter.js --fixture "fixtures/${{ steps.fixture.outputs.filename }}" | ||
| else | ||
| echo "🔍 Running Twitter scraper in production mode..." | ||
| node automation/twitter_scraper_adapter.js | ||
| fi | ||
| - name: Verify output file exists | ||
| run: | | ||
| if [ -f "data/twitter-bookmarks.json" ]; then | ||
| echo "✅ Twitter bookmarks data generated successfully" | ||
| wc -l data/twitter-bookmarks.json | ||
| head -5 data/twitter-bookmarks.json | ||
| else | ||
| echo "❌ Twitter bookmarks data not found" | ||
| exit 1 | ||
| fi | ||
| - name: Check for changes | ||
| id: changes | ||
| run: | | ||
| if git diff --quiet data/twitter-bookmarks.json; then | ||
| echo "changed=false" >> $GITHUB_OUTPUT | ||
| echo "📄 No changes in Twitter bookmarks data" | ||
| else | ||
| echo "changed=true" >> $GITHUB_OUTPUT | ||
| echo "📄 Changes detected in Twitter bookmarks data" | ||
| git diff --stat data/twitter-bookmarks.json | ||
| fi | ||
| - name: Commit and push changes | ||
| if: steps.changes.outputs.changed == 'true' | ||
| run: | | ||
| git config --local user.email "action@github.com" | ||
| git config --local user.name "GitHub Action" | ||
| git add data/twitter-bookmarks.json | ||
| # Check if fixture file exists and add it too | ||
| if [ -f "fixtures/${{ steps.fixture.outputs.filename }}" ]; then | ||
| git add "fixtures/${{ steps.fixture.outputs.filename }}" | ||
| fi | ||
| git commit -m "chore: update Twitter bookmarks data | ||
| 🤖 Generated with [GitHub Actions](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) | ||
| - Updated: $(date -u +"%Y-%m-%d %H:%M:%S UTC") | ||
| - Source: Twitter Bookmarks Scraper | ||
| - Mode: Mock/TestFixture | ||
| - Items: $(jq '.bookmarks | length' data/twitter-bookmarks.json 2>/dev/null || echo 'unknown') | ||
| Co-Authored-By: GitHub Action <action@users.noreply.github.com>" | ||
| git push | ||
| - name: Upload fixture as artifact | ||
| if: github.event.inputs.test_mode == 'true' | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: twitter-bookmarks-fixture | ||
| path: fixtures/${{ steps.fixture.outputs.filename }} | ||
| retention-days: 7 | ||
| - name: Summary | ||
| run: | | ||
| echo "## 📊 Twitter Bookmarks Scraping Summary" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo "**Status:** ${{ steps.changes.outputs.changed == 'true' && '✅ Updated' || '✅ No changes' }}" >> $GITHUB_STEP_SUMMARY | ||
| echo "**Mode:** ${{ github.event.inputs.test_mode == 'true' && 'Mock/Test' || 'Production' }}" >> $GITHUB_STEP_SUMMARY | ||
| echo "**Timestamp:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| if [ -f "data/twitter-bookmarks.json" ]; then | ||
| echo "**Data File:** \`data/twitter-bookmarks.json\`" >> $GITHUB_STEP_SUMMARY | ||
| echo "**File Size:** $(du -h data/twitter-bookmarks.json | cut -f1)" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| if command -v jq >/dev/null 2>&1; then | ||
| local item_count=$(jq '.bookmarks | length' data/twitter-bookmarks.json 2>/dev/null || echo "N/A") | ||
| echo "**Bookmarks Count:** $item_count" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| fi | ||
| if [ "${{ steps.changes.outputs.changed }}" == "true" ]; then | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo "### 📝 Changes Committed" >> $GITHUB_STEP_SUMMARY | ||
| echo "- Data file updated with latest bookmarks" >> $GITHUB_STEP_SUMMARY | ||
| if [ -f "fixtures/${{ steps.fixture.outputs.filename }}" ]; then | ||
| echo "- Test fixture saved: \`${{ steps.fixture.outputs.filename }}\`" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| fi | ||
