check-links.yml

name: Check Links

on:
  # Run M/W/F mornings
  schedule:
    - cron: '0 10 * * 1,2,3,4,5'
  workflow_dispatch:

permissions:
  contents: write
  pull-requests: write

jobs:
  check-links:
    if: github.repository == 'fern-api/docs'
    name: Check links
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Create lychee config
        run: |
          cat > lychee.toml << 'EOF'
          # Lychee link checker configuration
          # https://lychee.cli.rs/

          # Accept these status codes as valid
          accept = [200, 204, 301, 302, 307, 308]

          # User agent to avoid being blocked
          user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"

          # Exclude patterns (regex)
          exclude = [
            # Malformed ToC anchor links generated by the docs platform
            # These contain internal routing params (domain/host/auth/audience/version/path)
            # and are not real broken links — this is a fern-platform rendering bug
            "buildwithfern\\.com/_/buildwithfern\\.com/",
            "app\\.buildwithfern\\.com/buildwithfern\\.com/",
            
            # Placeholder/example URLs
            "^https://example\\.com",
            "^https://github\\.com/owner/repo",
            
            # Analytics and tracking
            "^https://us\\.i\\.posthog\\.com",
            "^https://c\\.vialoops\\.com",
            "_vercel/(speed-)?insights",
            
            # Next.js image optimization URLs with content hashes that go stale after each deployment
            # These are generated by the platform (not source-level links) and resolve after redeployment
            "buildwithfern\\.com/_next/image",
            "app\\.buildwithfern\\.com/_next/static/chunks/",
            
            # Asset URLs that work on the live site but are not directly accessible.
            # The `../assets/pdf-ui.png` reference in dashboard/pages/pdf-export.mdx
            # resolves to a locale-specific URL when rendered in translated pages
            # (e.g. /learn/zh/dashboard/...), so exclude both the default-locale
            # and translated variants.
            "buildwithfern\\.com/learn/dashboard/assets/pdf-ui\\.png",
            "buildwithfern\\.com/learn/zh/dashboard/assets/pdf-ui\\.png",
            
            # Sites that block automated requests
            "^https://docs\\.stack-auth\\.com",
            "^https://www\\.linkedin\\.com",
            "^https://linkedin\\.com",
            "^https://twitter\\.com",
            "^https://x\\.com",
            "^https://www\\.npmjs\\.com",
            "^https://cdn\\.simpleicons\\.org",
            
            # All GitHub blob/tree/tag URLs - verified locally before lychee runs
            # This avoids 503 rate limiting from GitHub's automated request detection
            # Issues/PRs/compare links are still checked via HTTP (they rarely break)
            "^https://github\\.com/[^/]+/[^/]+/(blob|tree)/",
            "^https://github\\.com/[^/]+/[^/]+/releases/tag/",
            
            # Non-HTTP links
            "^mailto:",
            "^tel:",
            "^javascript:",
            
            # All GitHub URLs - checked separately in dedicated steps
            "^https://github\\.com/"
          ]
          EOF

      - name: Fetch sitemap and extract URLs
        run: |
          # The sitemap may be either a regular <urlset> or a <sitemapindex>
          # that points to per-language sub-sitemaps (e.g. sitemap-en.xml,
          # sitemap-zh.xml). Handle both shapes by recursively expanding any
          # sitemapindex into its child sitemaps before extracting page URLs.
          set -euo pipefail
          ROOT_SITEMAP="https://buildwithfern.com/learn/sitemap.xml"

          fetch_sitemap_urls() {
            local sitemap_url="$1"
            local body
            body=$(curl -fsSL "$sitemap_url") || {
              echo "Warning: failed to fetch $sitemap_url" >&2
              return 0
            }

            local locs
            locs=$(echo "$body" | grep -oP '(?<=<loc>)[^<]+' || true)

            if echo "$body" | grep -q '<sitemapindex'; then
              # Recursively expand each child sitemap.
              while IFS= read -r child; do
                if [ -n "$child" ]; then
                  fetch_sitemap_urls "$child"
                fi
              done <<< "$locs"
            else
              # Regular <urlset> — emit page URLs directly.
              if [ -n "$locs" ]; then
                echo "$locs"
              fi
            fi
          }

          fetch_sitemap_urls "$ROOT_SITEMAP" | sort -u > urls.txt

          total=$(wc -l < urls.txt | tr -d ' ')
          echo "Found $total URLs in sitemap"

          if [ "$total" -eq 0 ]; then
            echo "::error::No URLs were extracted from the sitemap. The link checker has nothing to scan."
            echo "Root sitemap response:"
            curl -fsSL "$ROOT_SITEMAP" || true
            exit 1
          fi

      - name: Extract and verify GitHub blob/tree/tag URLs locally
        id: verify_github
        run: |
          # Extract all GitHub blob/tree/tag URLs from the repo source files
          # This is much faster than fetching all published pages via HTTP (~11 min -> seconds)
          # These URLs are excluded from lychee and verified locally instead to avoid 503 errors
          
          echo "Scanning repo for GitHub blob/tree/tag URLs..."
          start_time=$(date +%s)
          > github-urls.txt
          
          # Search the content directories for GitHub blob/tree URLs
          # Include fern/ (main docs) and README.md (root)
          # Exclude .git and any cloned repos
          grep -RhoE 'https://github\.com/[^/]+/[^/]+/(blob|tree)/[^"'"'"')<>[:space:]]+' \
            fern/ \
            README.md \
            --exclude-dir=.git \
            --exclude-dir=.github-repos \
            >> github-urls.txt 2>/dev/null || true
          
          # Also search for releases/tag URLs
          grep -RhoE 'https://github\.com/[^/]+/[^/]+/releases/tag/[^"'"'"')<>[:space:]]+' \
            fern/ \
            README.md \
            --exclude-dir=.git \
            --exclude-dir=.github-repos \
            >> github-urls.txt 2>/dev/null || true
          
          # Deduplicate URLs
          sort -u github-urls.txt -o github-urls.txt
          
          # Remove example/placeholder URLs (e.g., github.com/your-org/...)
          # These are documentation examples, not real repos to verify
          if [ -s github-urls.txt ]; then
            grep -v 'github\.com/your-org/' github-urls.txt > github-urls.filtered || true
            mv github-urls.filtered github-urls.txt
          fi
          
          total_urls=$(wc -l < github-urls.txt | tr -d ' ')
          end_time=$(date +%s)
          echo "Found $total_urls unique GitHub URLs to verify locally (took $((end_time - start_time))s)"
          
          if [ "$total_urls" -eq 0 ]; then
            echo "No GitHub URLs to verify"
            echo "verified_count=0" >> $GITHUB_OUTPUT
            echo "missing_count=0" >> $GITHUB_OUTPUT
            echo "has_missing=false" >> $GITHUB_OUTPUT
            exit 0
          fi
          
          # Extract unique repos that need to be cloned (org/repo format)
          grep -oE 'https://github\.com/[^/]+/[^/]+' github-urls.txt | sort -u > github-repos.txt
          
          echo "Repos to clone:"
          cat github-repos.txt
          
          # Clone each repo (shallow clone for efficiency)
          mkdir -p .github-repos
          while IFS= read -r repo_url; do
            [ -z "$repo_url" ] && continue
            # Extract org/repo for directory structure
            org_repo=$(echo "$repo_url" | sed -E 's#https://github\.com/##')
            org=$(echo "$org_repo" | cut -d'/' -f1)
            repo_name=$(echo "$org_repo" | cut -d'/' -f2)
            
            echo "Cloning $org/$repo_name..."
            mkdir -p ".github-repos/$org"
            
            # Use sparse checkout for efficiency - we only need to check if paths exist
            git clone --depth 1 --filter=blob:none --sparse "$repo_url.git" ".github-repos/$org/$repo_name" 2>/dev/null || {
              echo "Warning: Failed to clone $repo_url, will mark URLs from this repo as unverifiable"
              continue
            }
            
            # Fetch all tags for tag verification (lightweight fetch)
            cd ".github-repos/$org/$repo_name"
            git fetch --tags --depth 1 2>/dev/null || true
            cd ../../..
          done < github-repos.txt
          
          # Verify each URL by checking if the path exists in the cloned repo
          verified_count=0
          missing_count=0
          > github-missing.txt
          > github-verified.txt
          
          while IFS= read -r url; do
            [ -z "$url" ] && continue
            
            # Remove any URL fragments or query strings
            clean_url="${url%%#*}"
            clean_url="${clean_url%%\?*}"
            
            # Extract org, repo name, and determine URL type
            org=$(echo "$clean_url" | sed -E 's#https://github\.com/([^/]+)/.*#\1#')
            repo_name=$(echo "$clean_url" | sed -E 's#https://github\.com/[^/]+/([^/]+)/.*#\1#')
            
            # Check if we have the repo cloned
            if [ ! -d ".github-repos/$org/$repo_name" ]; then
              echo "Repo not cloned, cannot verify: $url"
              echo "- [UNVERIFIABLE] $url (repo clone failed)" >> github-missing.txt
              missing_count=$((missing_count + 1))
              continue
            fi
            
            cd ".github-repos/$org/$repo_name"
            
            # Check if this is a releases/tag URL
            if echo "$clean_url" | grep -qE '/releases/tag/'; then
              # Extract tag name
              tag_name=$(echo "$clean_url" | sed -E 's#.*/releases/tag/(.*)#\1#')
              
              # Check if tag exists
              if git tag -l "$tag_name" | grep -q "^${tag_name}$"; then
                echo "Verified (tag): $url -> tag $tag_name"
                echo "$url" >> ../../../github-verified.txt
                verified_count=$((verified_count + 1))
              else
                echo "MISSING (tag): $url -> tag $tag_name"
                echo "- [LOCAL_MISSING] $url (tag: $tag_name in $org/$repo_name)" >> ../../../github-missing.txt
                missing_count=$((missing_count + 1))
              fi
            else
              # This is a blob/tree URL - extract ref and path
              # Format: https://github.com/ORG/REPO/(blob|tree)/REF/PATH
              ref=$(echo "$clean_url" | sed -E 's#https://github\.com/[^/]+/[^/]+/(blob|tree)/([^/]+)/.*#\2#')
              rel_path=$(echo "$clean_url" | sed -E 's#https://github\.com/[^/]+/[^/]+/(blob|tree)/[^/]+/(.*)#\2#')
              
              # URL-decode the relative path (handles %5B -> [, %5D -> ], spaces, etc.)
              # This is needed because GitHub URLs encode special characters like [ and ]
              rel_path=$(python3 -c "import sys, urllib.parse; print(urllib.parse.unquote(sys.argv[1]))" "$rel_path")
              
              # Handle HEAD ref specially
              if [ "$ref" = "HEAD" ]; then
                ref=$(git symbolic-ref refs/remotes/origin/HEAD 2>/dev/null | sed 's@^refs/remotes/origin/@@' || echo "main")
              fi
              
              # For specific commit SHAs, we need to fetch them
              if echo "$ref" | grep -qE '^[0-9a-f]{40}$'; then
                git fetch --depth 1 origin "$ref" 2>/dev/null || true
              fi
              
                # Check if path exists using git ls-tree (works with sparse checkout)
                # Use grep -qxF for literal string matching (paths may contain regex chars like [ and ])
                if git ls-tree -r --name-only "$ref" -- "$rel_path" 2>/dev/null | grep -qxF "$rel_path" || \
                   git ls-tree -d --name-only "$ref" -- "$rel_path" 2>/dev/null | grep -qxF "$rel_path"; then
                echo "Verified: $url -> $rel_path (ref: $ref)"
                echo "$url" >> ../../../github-verified.txt
                verified_count=$((verified_count + 1))
              else
                # Also check if it's a directory (tree link)
                if git ls-tree -d "$ref" -- "$rel_path" 2>/dev/null | grep -q .; then
                  echo "Verified (directory): $url -> $rel_path (ref: $ref)"
                  echo "$url" >> ../../../github-verified.txt
                  verified_count=$((verified_count + 1))
                else
                  echo "MISSING: $url -> $rel_path (ref: $ref)"
                  echo "- [LOCAL_MISSING] $url (path: $rel_path in $org/$repo_name, ref: $ref)" >> ../../../github-missing.txt
                  missing_count=$((missing_count + 1))
                fi
              fi
            fi
            
            cd ../../..
          done < github-urls.txt
          
          echo ""
          echo "=== GitHub URL Verification Summary ==="
          echo "Total URLs checked: $total_urls"
          echo "Verified locally: $verified_count"
          echo "Missing/unverifiable: $missing_count"
          
          echo "verified_count=$verified_count" >> $GITHUB_OUTPUT
          echo "missing_count=$missing_count" >> $GITHUB_OUTPUT
          
          if [ "$missing_count" -gt 0 ]; then
            echo "has_missing=true" >> $GITHUB_OUTPUT
            echo ""
            echo "Missing URLs:"
            cat github-missing.txt
          else
            echo "has_missing=false" >> $GITHUB_OUTPUT
          fi
          
          # Cleanup cloned repos
          rm -rf .github-repos

      - name: Extract GitHub URLs for HTTP checking from source files
        id: extract_github_http
        run: |
          # Extract GitHub URLs that need HTTP checking (issues, compare, commit, discussions, non-fern-api PRs)
          # These are extracted from source files and passed directly to lychee
          # This avoids the --include filter issue where lychee applies it to input pages, not discovered links
          
          echo "Scanning repo for GitHub URLs to check via HTTP..."
          > github-http-urls.txt
          
          # Search for issues URLs
          grep -RhoE 'https://github\.com/[^/]+/[^/]+/issues/[^"'"'"')<>[:space:]]+' \
            fern/ README.md \
            --exclude-dir=.git --exclude-dir=.github-repos \
            >> github-http-urls.txt 2>/dev/null || true
          
          # Search for compare URLs
          grep -RhoE 'https://github\.com/[^/]+/[^/]+/compare/[^"'"'"')<>[:space:]]+' \
            fern/ README.md \
            --exclude-dir=.git --exclude-dir=.github-repos \
            >> github-http-urls.txt 2>/dev/null || true
          
          # Search for commit/commits URLs
          grep -RhoE 'https://github\.com/[^/]+/[^/]+/commits?/[^"'"'"')<>[:space:]]+' \
            fern/ README.md \
            --exclude-dir=.git --exclude-dir=.github-repos \
            >> github-http-urls.txt 2>/dev/null || true
          
          # Search for discussions URLs
          grep -RhoE 'https://github\.com/[^/]+/[^/]+/discussions/[^"'"'"')<>[:space:]]+' \
            fern/ README.md \
            --exclude-dir=.git --exclude-dir=.github-repos \
            >> github-http-urls.txt 2>/dev/null || true
          
          # Search for pull request URLs (excluding fern-api org - those are too slow to check)
          grep -RhoE 'https://github\.com/[^/]+/[^/]+/pull/[^"'"'"')<>[:space:]]+' \
            fern/ README.md \
            --exclude-dir=.git --exclude-dir=.github-repos \
            2>/dev/null | grep -v 'github\.com/fern-api/' \
            >> github-http-urls.txt || true
          
          # Remove example/placeholder URLs (e.g., github.com/your-org/...)
          if [ -s github-http-urls.txt ]; then
            grep -v 'github\.com/your-org/' github-http-urls.txt > github-http-urls.filtered || true
            mv github-http-urls.filtered github-http-urls.txt
          fi
          
          # Deduplicate URLs
          sort -u github-http-urls.txt -o github-http-urls.txt
          
          total_urls=$(wc -l < github-http-urls.txt | tr -d ' ')
          echo "Found $total_urls unique GitHub URLs to check via HTTP"
          echo "github_http_count=$total_urls" >> $GITHUB_OUTPUT
          
          if [ "$total_urls" -gt 0 ]; then
            echo "URLs to check:"
            cat github-http-urls.txt
          fi


      - name: Upload URLs (early, for debugging)
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: urls
          path: |
            github-urls.txt
            github-http-urls.txt
            urls.txt
          if-no-files-found: ignore

      - name: Check GitHub links (very low concurrency to avoid 503 rate limiting)
        id: lychee_github
        if: steps.extract_github_http.outputs.github_http_count != '0'
        uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
        with:
          args: >-
            --no-progress
            --timeout 30
            --max-retries 5
            --retry-wait-time 20
            --max-concurrency 2
            github-http-urls.txt
          output: ./lychee-raw-github.md
          format: markdown
          fail: false
          jobSummary: false

      - name: Check non-GitHub links (high concurrency)
        id: lychee_main
        uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
        with:
          args: >-
            --config lychee.toml
            --no-progress
            --timeout 30
            --max-retries 3
            --retry-wait-time 10
            --max-concurrency 20
            --include-fragments
            --files-from urls.txt
          output: ./lychee-raw-main.md
          format: markdown
          fail: false
          jobSummary: false

      - name: Combine lychee outputs
        run: |
          # Combine both lychee runs into a single raw file for downstream processing
          : > lychee-raw.md
          [ -f lychee-raw-main.md ] && cat lychee-raw-main.md >> lychee-raw.md
          [ -f lychee-raw-github.md ] && cat lychee-raw-github.md >> lychee-raw.md
          echo "Combined lychee outputs into lychee-raw.md"

      - name: Extract 429 URLs and retry with exponential backoff
        id: retry429
        run: |
          # Extract all 429 URLs from the raw lychee report
          # Note: 403s are excluded from broken links report but NOT retried (bot-blocking sites won't change)
          grep '\[429\]' ./lychee-raw.md | sed -E 's/.*<([^>]+)>.*/\1/' | sort -u > urls-429-all.txt || true
          
          # Initialize counters for repo-internal GitHub URLs
          verified_locally=0
          missing_locally=0
          
          # Process repo-internal GitHub URLs by checking if files exist locally
          # This avoids false positives from GitHub rate limiting while still catching real broken links
          > urls-429.txt
          > urls-429-repo-missing.txt
          
          while IFS= read -r url; do
            [ -z "$url" ] && continue
            
            # Check if this is a repo-internal GitHub URL (blob/main pattern)
            if echo "$url" | grep -qE '^https://github\.com/fern-api/docs/blob/main/'; then
              # Extract relative path from URL (strip prefix and query string)
              rel_path="${url#https://github.com/fern-api/docs/blob/main/}"
              rel_path="${rel_path%%\?*}"
              
              if [ -f "$rel_path" ]; then
                echo "Verified locally (file exists): $url -> $rel_path"
                verified_locally=$((verified_locally + 1))
              else
                echo "MISSING locally: $url -> $rel_path"
                echo "- [LOCAL_MISSING] $url (expected path: $rel_path)" >> urls-429-repo-missing.txt
                missing_locally=$((missing_locally + 1))
              fi
            else
              # Non-repo URL, add to retry list
              echo "$url" >> urls-429.txt
            fi
          done < urls-429-all.txt
          
          echo "Repo-internal GitHub URLs verified locally: $verified_locally"
          echo "Repo-internal GitHub URLs missing locally: $missing_locally"
          echo "verified_locally=$verified_locally" >> $GITHUB_OUTPUT
          echo "missing_locally=$missing_locally" >> $GITHUB_OUTPUT
          
          count=$(wc -l < urls-429.txt | tr -d ' ')
          echo "Found $count other URLs with 429 status to retry"
          echo "rate_limited_count=$count" >> $GITHUB_OUTPUT
          
          # Initialize still-failing file
          > urls-429-still-failing.txt
          
          # Add any locally-missing repo URLs to the still-failing list
          if [ -s urls-429-repo-missing.txt ]; then
            cat urls-429-repo-missing.txt >> urls-429-still-failing.txt
          fi
          
          if [ "$count" -eq "0" ] || [ ! -s urls-429.txt ]; then
            echo "No other 429 URLs to retry"
          else
            echo "Retrying $count URLs with exponential backoff..."
            
            while IFS= read -r url; do
              [ -z "$url" ] && continue
              delay=15
              max_attempts=4
              success=false
              
              for attempt in $(seq 1 $max_attempts); do
                echo "[$attempt/$max_attempts] Checking: $url"
                status=$(curl -s -o /dev/null -w "%{http_code}" \
                  -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" \
                  --max-time 30 \
                  "$url" || echo "000")
                
                  if [ "$status" -lt 400 ]; then
                    echo "OK on retry (status $status): $url"
                    success=true
                    break
                  elif [ "$status" -eq 429 ]; then
                    echo "Still 429, sleeping ${delay}s before next attempt..."
                    sleep "$delay"
                    delay=$((delay * 2))
                  else
                    echo "Non-429 failure (status $status): $url"
                    break
                  fi
              done
              
              if [ "$success" = false ]; then
                echo "- [${status}] $url" >> urls-429-still-failing.txt
              fi
            done < urls-429.txt
          fi
          
          still_failing=$(wc -l < urls-429-still-failing.txt | tr -d ' ')
          echo "URLs still failing after retry: $still_failing"
          echo "still_failing_429=$still_failing" >> $GITHUB_OUTPUT
          
          if [ "$still_failing" -gt 0 ]; then
            echo "has_429_failures=true" >> $GITHUB_OUTPUT
          else
            echo "has_429_failures=false" >> $GITHUB_OUTPUT
          fi

      - name: Build errors-only report and summarize
        id: check_failures
        run: |
          # Only 4xx responses are actionable as source-level broken links.
          # 403 and 429 are handled separately. 5xx responses are reported as
          # informational because changing docs source cannot fix server errors.
          FAILURE_PATTERN='\[400\]|\[401\]|\[402\]|\[40[4-9]\]|\[41[0-9]\]|\[42[0-8]\]|\[4[3-9][0-9]\]'
          SERVER_ERROR_PATTERN='\[5[0-9]{2}\]'
          
          # Extract actionable broken links and informational server errors
          grep -E "$FAILURE_PATTERN" ./lychee-raw.md > broken-links-raw.txt 2>/dev/null || true
          grep -E "$SERVER_ERROR_PATTERN" ./lychee-raw.md > server-errors.txt 2>/dev/null || true
          
          # Filter out repo-internal GitHub URLs that exist locally
          > broken-links.txt
          while IFS= read -r line; do
            [ -z "$line" ] && continue
            
            # Extract URL and status code from the line (format: ... [STATUS] <URL> ...)
            url=$(echo "$line" | sed -E 's/.*<([^>]+)>.*/\1/')
            status=$(echo "$line" | sed -E 's/.*\[([0-9]+)\].*/\1/')
            
            # Skip malformed ToC anchor links (fern-platform rendering bug)
            if echo "$url" | grep -qE 'buildwithfern\.com/_/buildwithfern\.com/|app\.buildwithfern\.com/buildwithfern\.com/'; then
              continue
            fi
            
            # Check if this is a repo-internal GitHub URL (blob/main pattern)
            if echo "$url" | grep -qE '^https://github\.com/fern-api/docs/blob/main/'; then
              # Extract relative path from URL (strip prefix and query string)
              rel_path="${url#https://github.com/fern-api/docs/blob/main/}"
              rel_path="${rel_path%%\?*}"
              
              if [ -f "$rel_path" ]; then
                echo "Verified locally (file exists): $url -> $rel_path"
              else
                # File doesn't exist locally, this is a real broken link
                echo "$line" >> broken-links.txt
              fi
            else
              # Not a repo-internal URL, keep it in the broken links list
              echo "$line" >> broken-links.txt
            fi
          done < broken-links-raw.txt
          
          # Deduplicate error files (same URL may appear multiple times if linked from multiple pages)
          sort -u broken-links.txt -o broken-links.txt
          sort -u server-errors.txt -o server-errors.txt
          
          broken_count=$(wc -l < broken-links.txt | tr -d ' ')
          server_error_count=$(wc -l < server-errors.txt | tr -d ' ')
          
          # Get rate limit stats
          rate_limited="${STEPS_RETRY429_OUTPUTS_RATE_LIMITED_COUNT}"
          rate_limited=${rate_limited:-0}
          still_failing_429="${STEPS_RETRY429_OUTPUTS_STILL_FAILING_429}"
          still_failing_429=${still_failing_429:-0}
          
          # Get GitHub local verification stats
          github_verified="${STEPS_VERIFY_GITHUB_OUTPUTS_VERIFIED_COUNT}"
          github_verified=${github_verified:-0}
          github_missing="${STEPS_VERIFY_GITHUB_OUTPUTS_MISSING_COUNT}"
          github_missing=${github_missing:-0}
          
          # Build clean errors-only report
          cat > lychee-report.md << 'HEADER'
          # Link Check Report
          
          This report only shows broken links (errors). Redirects and successful links are not included.
          
          HEADER
          
          if [ "$broken_count" -gt 0 ]; then
            echo "## Broken Links ($broken_count)" >> lychee-report.md
            echo "" >> lychee-report.md
            # Format: - [STATUS] URL
            sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/- [\1] \2/' broken-links.txt >> lychee-report.md
            echo "" >> lychee-report.md
          fi
          
          if [ "$still_failing_429" -gt 0 ]; then
            echo "## Rate-Limited URLs (429) - Still Failing After Retry" >> lychee-report.md
            echo "" >> lychee-report.md
            cat urls-429-still-failing.txt >> lychee-report.md
            echo "" >> lychee-report.md
          fi
          
          if [ "$github_missing" -gt 0 ]; then
            echo "## GitHub URLs - Missing Locally ($github_missing)" >> lychee-report.md
            echo "" >> lychee-report.md
            cat github-missing.txt >> lychee-report.md
            echo "" >> lychee-report.md
          fi
          
          if [ "$server_error_count" -gt 0 ]; then
            echo "## Server Errors (5xx) - Informational" >> lychee-report.md
            echo "" >> lychee-report.md
            echo "_These URLs returned server errors. They do not open a broken-link PR because docs source changes cannot fix them._" >> lychee-report.md
            echo "" >> lychee-report.md
            sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/- [\1] \2/' server-errors.txt >> lychee-report.md
            echo "" >> lychee-report.md
          fi
          
          if [ "$broken_count" -eq 0 ] && [ "$still_failing_429" -eq 0 ] && [ "$github_missing" -eq 0 ]; then
            echo "No broken links found!" >> lychee-report.md
          fi
          
          echo "" >> lychee-report.md
          echo "---" >> lychee-report.md
          echo "" >> lychee-report.md
          echo "Rate-limited URLs (429) that recovered after retry: $rate_limited" >> lychee-report.md
          
          # Calculate recovered 429s (rate_limited - still_failing_429)
          recovered_429=0
          if [ "$rate_limited" -gt 0 ]; then
            recovered_429=$((rate_limited - still_failing_429))
          fi
          
          # Set output for failure detection
          if [ "$broken_count" -gt 0 ]; then
            echo "has_other_failures=true" >> $GITHUB_OUTPUT
          else
            echo "has_other_failures=false" >> $GITHUB_OUTPUT
          fi
          
          # Extract lychee's summary table only (stop at first ## heading)
          sed -n '/^# Summary$/,/^## /p' ./lychee-raw.md | head -n -1 > lychee-summary-table.md
          
          # Print summary to logs
          echo ""
          cat lychee-summary-table.md
          echo ""
          # Get repo-internal GitHub URL counts
          verified_locally="${STEPS_RETRY429_OUTPUTS_VERIFIED_LOCALLY}"
          verified_locally=${verified_locally:-0}
          missing_locally="${STEPS_RETRY429_OUTPUTS_MISSING_LOCALLY}"
          missing_locally=${missing_locally:-0}
          
          echo "Recovery Info:"
          echo "  Repo-internal GitHub URLs (429 verified locally): $verified_locally"
          echo "  Repo-internal GitHub URLs (429 missing locally): $missing_locally"
          echo "  Rate-limited (429) - recovered after retry: $recovered_429"
          echo "  Rate-limited (429) - still failing: $still_failing_429"
          echo "  GitHub URLs verified locally: $github_verified"
          echo "  GitHub URLs missing locally: $github_missing"
          echo "  Server errors (5xx, informational): $server_error_count"
          
          if [ "$broken_count" -gt 0 ]; then
            echo ""
            echo "BROKEN LINKS (non-429):"
            cat broken-links.txt | sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/  - [\1] \2/'
          fi
          
          if [ "$still_failing_429" -gt 0 ]; then
            echo ""
            echo "RATE-LIMITED URLs STILL FAILING (429):"
            cat urls-429-still-failing.txt
          fi
          
          if [ "$github_missing" -gt 0 ]; then
            echo ""
            echo "GITHUB URLs MISSING LOCALLY:"
            cat github-missing.txt
          fi
          
          # Build GitHub Step Summary using lychee's table + 429 info + persistent errors only
          {
            # Copy lychee's summary table
            cat lychee-summary-table.md
            
            # Add recovery info
            echo ""
            echo "### Local Verification & Rate Limit Recovery"
            echo ""
            echo "| Status | Count |"
            echo "|--------|------:|"
            echo "| Repo-internal GitHub URLs (429 verified locally) | $verified_locally |"
            echo "| Repo-internal GitHub URLs (429 missing locally) | $missing_locally |"
            echo "| Rate-limited (429) recovered after retry | $recovered_429 |"
            echo "| Rate-limited (429) still failing | $still_failing_429 |"
            echo "| GitHub URLs verified locally | $github_verified |"
            echo "| GitHub URLs missing locally | $github_missing |"
            echo "| Server errors (5xx, informational) | $server_error_count |"
            echo ""
            
            # Show persistent errors only (non-429 broken links + 429s that didn't recover)
            echo "## Errors (after retry)"
            echo ""
            
            has_persistent_errors=false
            
            if [ "$broken_count" -gt 0 ]; then
              has_persistent_errors=true
              sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/- [\1] \2/' broken-links.txt
            fi
            
            if [ "$still_failing_429" -gt 0 ]; then
              has_persistent_errors=true
              cat urls-429-still-failing.txt
            fi
            
            if [ "$github_missing" -gt 0 ]; then
              has_persistent_errors=true
              cat github-missing.txt
            fi
            
            if [ "$has_persistent_errors" = false ]; then
              echo "No errors remained after retry."
            fi
            
            echo ""
          } >> "$GITHUB_STEP_SUMMARY"
        env:
          STEPS_RETRY429_OUTPUTS_RATE_LIMITED_COUNT: ${{ steps.retry429.outputs.rate_limited_count }}
          STEPS_RETRY429_OUTPUTS_STILL_FAILING_429: ${{ steps.retry429.outputs.still_failing_429 }}
          STEPS_VERIFY_GITHUB_OUTPUTS_VERIFIED_COUNT: ${{ steps.verify_github.outputs.verified_count }}
          STEPS_VERIFY_GITHUB_OUTPUTS_MISSING_COUNT: ${{ steps.verify_github.outputs.missing_count }}
          STEPS_RETRY429_OUTPUTS_VERIFIED_LOCALLY: ${{ steps.retry429.outputs.verified_locally }}
          STEPS_RETRY429_OUTPUTS_MISSING_LOCALLY: ${{ steps.retry429.outputs.missing_locally }}

      - name: Upload errors-only report
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: lychee-report
          path: ./lychee-report.md
          if-no-files-found: ignore

      - name: Upload lychee outputs and verification results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: lychee-outputs
          path: |
            github-verified.txt
            github-missing.txt
            lychee-raw-main.md
            lychee-raw-github.md
          if-no-files-found: ignore

      - name: Fix broken links with Claude
        id: fix-links
        if: steps.check_failures.outputs.has_other_failures == 'true' || steps.retry429.outputs.has_429_failures == 'true' || steps.verify_github.outputs.has_missing == 'true'
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: | # zizmor: ignore[adhoc-packages] claude-code CLI is only needed in this ephemeral CI job
          set -euo pipefail

          # Assemble the list of broken links to hand to Claude.
          {
            echo "## Non-429 broken links"
            if [ -s broken-links.txt ]; then
              sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/- [\1] \2/' broken-links.txt
            else
              echo "(none)"
            fi
            echo ""
            echo "## Rate-limited URLs (429) still failing after retry"
            if [ -s urls-429-still-failing.txt ]; then
              cat urls-429-still-failing.txt
            else
              echo "(none)"
            fi
            echo ""
            echo "## GitHub URLs missing locally"
            if [ -s github-missing.txt ]; then
              cat github-missing.txt
            else
              echo "(none)"
            fi
          } > broken-links-list.md

          # Build the prompt.
          cat > claude-prompt.md << 'PROMPT'
          You are fixing broken links in the fern-api/docs documentation repository.
          A scheduled link checker found the broken links listed at the end of this prompt.

          Rules:
          1. For each broken URL, search the source under `fern/` and `README.md` to find where
             the link is defined (grep for the URL, its path, or a distinctive slug), and fix it
             at the source.
          2. Internal links between docs pages must be published `/learn/...` URL paths built from
             the YAML config, never file paths or relative paths. Read the "Link checking" section
             of `AGENTS.md` and the relevant `docs.yml` files to construct the correct path.
          3. Common fixes: correct a moved or renamed slug, add a missing path segment (e.g.
             `/learn`), update an outdated commit SHA to a branch name, or fix URL-encoded paths.
          4. If you cannot confidently determine the correct replacement (e.g. the only signal is a
             5xx/connection error or rate limiting, or you cannot find where the link is defined),
             DO NOT guess or delete the link. Leave it unchanged and record it in
             `broken-links-unfixed.md` with a one-line reason.
          5. Only fix the specific failures listed below. Do not touch other links that merely look
             similar.
          6. Do not edit anything under `fern/translations/` — those files are generated.
          7. When done, write a short bullet-list summary of what you changed to
             `broken-links-summary.md`.

          The broken links to fix:
          PROMPT
          cat broken-links-list.md >> claude-prompt.md

          # Install and run Claude Code headlessly.
          npm install -g @anthropic-ai/claude-code
          claude -p "$(cat claude-prompt.md)" \
            --dangerously-skip-permissions \
            --max-turns 80 \
            --model sonnet \
            --output-format text || echo "Claude exited non-zero; will still check for changes"

          # Remove scratch files so they are never committed.
          rm -f claude-prompt.md broken-links-list.md

          if [ -n "$(git status --porcelain -- fern README.md)" ]; then
            echo "changes_made=true" >> "$GITHUB_OUTPUT"
            echo "Source changes made by Claude:"
            git status --porcelain -- fern README.md
          else
            echo "changes_made=false" >> "$GITHUB_OUTPUT"
            echo "Claude made no source changes."
          fi

      - name: Create or update PR with fixes
        if: steps.fix-links.outputs.changes_made == 'true'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          BRANCH="chore/fix-broken-links"

          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

          # Build the PR body from Claude's summary plus the full detected report.
          {
            echo "Automated fixes for broken links found by the scheduled link checker."
            echo ""
            echo "[View workflow run](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})"
            echo ""
            if [ -f broken-links-summary.md ]; then
              echo "## Summary of changes"
              echo ""
              cat broken-links-summary.md
              echo ""
            fi
            if [ -f broken-links-unfixed.md ]; then
              echo "## Links needing manual review"
              echo ""
              cat broken-links-unfixed.md
              echo ""
            fi
            if [ -f lychee-report.md ]; then
              echo "## All detected broken links"
              echo ""
              cat lychee-report.md
            fi
          } > pr-body.md

          # Stage only source fixes (never the report/scratch artifacts).
          git add fern README.md
          git checkout -b "$BRANCH"
          git commit -m "fix: repair broken documentation links

          Automated fix generated by the scheduled link checker using Claude.
          Detected by workflow run ${GITHUB_RUN_ID}."

          git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
          git push --force origin "$BRANCH"

          existing=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number' 2>/dev/null || echo "")
          if [ -n "$existing" ]; then
            echo "Updating existing PR #$existing"
            gh pr edit "$existing" --body-file pr-body.md || true
          else
            gh pr create --draft --base main --head "$BRANCH" \
              --title "Fix broken links" \
              --body-file pr-body.md
          fi

      - name: Fail if there are broken links
        if: steps.check_failures.outputs.has_other_failures == 'true' || steps.retry429.outputs.has_429_failures == 'true' || steps.verify_github.outputs.has_missing == 'true'
        run: |
          echo "Link check failed!"
          if [ "${STEPS_CHECK_FAILURES_OUTPUTS_HAS_OTHER_FAILURES}" == "true" ]; then
            echo "There are broken links (non-429 failures) in the report."
          fi
          if [ "${STEPS_RETRY429_OUTPUTS_HAS_429_FAILURES}" == "true" ]; then
            echo "Some URLs still returned 429 after exponential backoff retry."
            echo "These URLs may need to be excluded or the rate limit needs more time to reset."
          fi
          if [ "${STEPS_VERIFY_GITHUB_OUTPUTS_HAS_MISSING}" == "true" ]; then
            echo "Some GitHub URLs point to paths that don't exist in the repos."
          fi
          exit 1
        env:
          STEPS_CHECK_FAILURES_OUTPUTS_HAS_OTHER_FAILURES: ${{ steps.check_failures.outputs.has_other_failures }}
          STEPS_RETRY429_OUTPUTS_HAS_429_FAILURES: ${{ steps.retry429.outputs.has_429_failures }}
          STEPS_VERIFY_GITHUB_OUTPUTS_HAS_MISSING: ${{ steps.verify_github.outputs.has_missing }}