Check Links #283
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check Links | |
| on: | |
| # Run M/W/F mornings | |
| schedule: | |
| - cron: '0 10 * * 1,2,3,4,5' | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| check-links: | |
| if: github.repository == 'fern-api/docs' | |
| name: Check links | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - name: Create lychee config | |
| run: | | |
| cat > lychee.toml << 'EOF' | |
| # Lychee link checker configuration | |
| # https://lychee.cli.rs/ | |
| # Accept these status codes as valid | |
| accept = [200, 204, 301, 302, 307, 308] | |
| # User agent to avoid being blocked | |
| user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" | |
| # Exclude patterns (regex) | |
| exclude = [ | |
| # Malformed ToC anchor links generated by the docs platform | |
| # These contain internal routing params (domain/host/auth/audience/version/path) | |
| # and are not real broken links — this is a fern-platform rendering bug | |
| "buildwithfern\\.com/_/buildwithfern\\.com/", | |
| "app\\.buildwithfern\\.com/buildwithfern\\.com/", | |
| # Placeholder/example URLs | |
| "^https://example\\.com", | |
| "^https://github\\.com/owner/repo", | |
| # Analytics and tracking | |
| "^https://us\\.i\\.posthog\\.com", | |
| "^https://c\\.vialoops\\.com", | |
| "_vercel/(speed-)?insights", | |
| # Next.js image optimization URLs with content hashes that go stale after each deployment | |
| # These are generated by the platform (not source-level links) and resolve after redeployment | |
| "buildwithfern\\.com/_next/image", | |
| "app\\.buildwithfern\\.com/_next/static/chunks/", | |
| # Asset URLs that work on the live site but are not directly accessible. | |
| # The `../assets/pdf-ui.png` reference in dashboard/pages/pdf-export.mdx | |
| # resolves to a locale-specific URL when rendered in translated pages | |
| # (e.g. /learn/zh/dashboard/...), so exclude both the default-locale | |
| # and translated variants. | |
| "buildwithfern\\.com/learn/dashboard/assets/pdf-ui\\.png", | |
| "buildwithfern\\.com/learn/zh/dashboard/assets/pdf-ui\\.png", | |
| # Sites that block automated requests | |
| "^https://docs\\.stack-auth\\.com", | |
| "^https://www\\.linkedin\\.com", | |
| "^https://linkedin\\.com", | |
| "^https://twitter\\.com", | |
| "^https://x\\.com", | |
| "^https://www\\.npmjs\\.com", | |
| "^https://cdn\\.simpleicons\\.org", | |
| # All GitHub blob/tree/tag URLs - verified locally before lychee runs | |
| # This avoids 503 rate limiting from GitHub's automated request detection | |
| # Issues/PRs/compare links are still checked via HTTP (they rarely break) | |
| "^https://github\\.com/[^/]+/[^/]+/(blob|tree)/", | |
| "^https://github\\.com/[^/]+/[^/]+/releases/tag/", | |
| # Non-HTTP links | |
| "^mailto:", | |
| "^tel:", | |
| "^javascript:", | |
| # All GitHub URLs - checked separately in dedicated steps | |
| "^https://github\\.com/" | |
| ] | |
| EOF | |
| - name: Fetch sitemap and extract URLs | |
| run: | | |
| # The sitemap may be either a regular <urlset> or a <sitemapindex> | |
| # that points to per-language sub-sitemaps (e.g. sitemap-en.xml, | |
| # sitemap-zh.xml). Handle both shapes by recursively expanding any | |
| # sitemapindex into its child sitemaps before extracting page URLs. | |
| set -euo pipefail | |
| ROOT_SITEMAP="https://buildwithfern.com/learn/sitemap.xml" | |
| fetch_sitemap_urls() { | |
| local sitemap_url="$1" | |
| local body | |
| body=$(curl -fsSL "$sitemap_url") || { | |
| echo "Warning: failed to fetch $sitemap_url" >&2 | |
| return 0 | |
| } | |
| local locs | |
| locs=$(echo "$body" | grep -oP '(?<=<loc>)[^<]+' || true) | |
| if echo "$body" | grep -q '<sitemapindex'; then | |
| # Recursively expand each child sitemap. | |
| while IFS= read -r child; do | |
| if [ -n "$child" ]; then | |
| fetch_sitemap_urls "$child" | |
| fi | |
| done <<< "$locs" | |
| else | |
| # Regular <urlset> — emit page URLs directly. | |
| if [ -n "$locs" ]; then | |
| echo "$locs" | |
| fi | |
| fi | |
| } | |
| fetch_sitemap_urls "$ROOT_SITEMAP" | sort -u > urls.txt | |
| total=$(wc -l < urls.txt | tr -d ' ') | |
| echo "Found $total URLs in sitemap" | |
| if [ "$total" -eq 0 ]; then | |
| echo "::error::No URLs were extracted from the sitemap. The link checker has nothing to scan." | |
| echo "Root sitemap response:" | |
| curl -fsSL "$ROOT_SITEMAP" || true | |
| exit 1 | |
| fi | |
| - name: Extract and verify GitHub blob/tree/tag URLs locally | |
| id: verify_github | |
| run: | | |
| # Extract all GitHub blob/tree/tag URLs from the repo source files | |
| # This is much faster than fetching all published pages via HTTP (~11 min -> seconds) | |
| # These URLs are excluded from lychee and verified locally instead to avoid 503 errors | |
| echo "Scanning repo for GitHub blob/tree/tag URLs..." | |
| start_time=$(date +%s) | |
| > github-urls.txt | |
| # Search the content directories for GitHub blob/tree URLs | |
| # Include fern/ (main docs) and README.md (root) | |
| # Exclude .git and any cloned repos | |
| grep -RhoE 'https://github\.com/[^/]+/[^/]+/(blob|tree)/[^"'"'"')<>[:space:]]+' \ | |
| fern/ \ | |
| README.md \ | |
| --exclude-dir=.git \ | |
| --exclude-dir=.github-repos \ | |
| >> github-urls.txt 2>/dev/null || true | |
| # Also search for releases/tag URLs | |
| grep -RhoE 'https://github\.com/[^/]+/[^/]+/releases/tag/[^"'"'"')<>[:space:]]+' \ | |
| fern/ \ | |
| README.md \ | |
| --exclude-dir=.git \ | |
| --exclude-dir=.github-repos \ | |
| >> github-urls.txt 2>/dev/null || true | |
| # Deduplicate URLs | |
| sort -u github-urls.txt -o github-urls.txt | |
| # Remove example/placeholder URLs (e.g., github.com/your-org/...) | |
| # These are documentation examples, not real repos to verify | |
| if [ -s github-urls.txt ]; then | |
| grep -v 'github\.com/your-org/' github-urls.txt > github-urls.filtered || true | |
| mv github-urls.filtered github-urls.txt | |
| fi | |
| total_urls=$(wc -l < github-urls.txt | tr -d ' ') | |
| end_time=$(date +%s) | |
| echo "Found $total_urls unique GitHub URLs to verify locally (took $((end_time - start_time))s)" | |
| if [ "$total_urls" -eq 0 ]; then | |
| echo "No GitHub URLs to verify" | |
| echo "verified_count=0" >> $GITHUB_OUTPUT | |
| echo "missing_count=0" >> $GITHUB_OUTPUT | |
| echo "has_missing=false" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| # Extract unique repos that need to be cloned (org/repo format) | |
| grep -oE 'https://github\.com/[^/]+/[^/]+' github-urls.txt | sort -u > github-repos.txt | |
| echo "Repos to clone:" | |
| cat github-repos.txt | |
| # Clone each repo (shallow clone for efficiency) | |
| mkdir -p .github-repos | |
| while IFS= read -r repo_url; do | |
| [ -z "$repo_url" ] && continue | |
| # Extract org/repo for directory structure | |
| org_repo=$(echo "$repo_url" | sed -E 's#https://github\.com/##') | |
| org=$(echo "$org_repo" | cut -d'/' -f1) | |
| repo_name=$(echo "$org_repo" | cut -d'/' -f2) | |
| echo "Cloning $org/$repo_name..." | |
| mkdir -p ".github-repos/$org" | |
| # Use sparse checkout for efficiency - we only need to check if paths exist | |
| git clone --depth 1 --filter=blob:none --sparse "$repo_url.git" ".github-repos/$org/$repo_name" 2>/dev/null || { | |
| echo "Warning: Failed to clone $repo_url, will mark URLs from this repo as unverifiable" | |
| continue | |
| } | |
| # Fetch all tags for tag verification (lightweight fetch) | |
| cd ".github-repos/$org/$repo_name" | |
| git fetch --tags --depth 1 2>/dev/null || true | |
| cd ../../.. | |
| done < github-repos.txt | |
| # Verify each URL by checking if the path exists in the cloned repo | |
| verified_count=0 | |
| missing_count=0 | |
| > github-missing.txt | |
| > github-verified.txt | |
| while IFS= read -r url; do | |
| [ -z "$url" ] && continue | |
| # Remove any URL fragments or query strings | |
| clean_url="${url%%#*}" | |
| clean_url="${clean_url%%\?*}" | |
| # Extract org, repo name, and determine URL type | |
| org=$(echo "$clean_url" | sed -E 's#https://github\.com/([^/]+)/.*#\1#') | |
| repo_name=$(echo "$clean_url" | sed -E 's#https://github\.com/[^/]+/([^/]+)/.*#\1#') | |
| # Check if we have the repo cloned | |
| if [ ! -d ".github-repos/$org/$repo_name" ]; then | |
| echo "Repo not cloned, cannot verify: $url" | |
| echo "- [UNVERIFIABLE] $url (repo clone failed)" >> github-missing.txt | |
| missing_count=$((missing_count + 1)) | |
| continue | |
| fi | |
| cd ".github-repos/$org/$repo_name" | |
| # Check if this is a releases/tag URL | |
| if echo "$clean_url" | grep -qE '/releases/tag/'; then | |
| # Extract tag name | |
| tag_name=$(echo "$clean_url" | sed -E 's#.*/releases/tag/(.*)#\1#') | |
| # Check if tag exists | |
| if git tag -l "$tag_name" | grep -q "^${tag_name}$"; then | |
| echo "Verified (tag): $url -> tag $tag_name" | |
| echo "$url" >> ../../../github-verified.txt | |
| verified_count=$((verified_count + 1)) | |
| else | |
| echo "MISSING (tag): $url -> tag $tag_name" | |
| echo "- [LOCAL_MISSING] $url (tag: $tag_name in $org/$repo_name)" >> ../../../github-missing.txt | |
| missing_count=$((missing_count + 1)) | |
| fi | |
| else | |
| # This is a blob/tree URL - extract ref and path | |
| # Format: https://github.com/ORG/REPO/(blob|tree)/REF/PATH | |
| ref=$(echo "$clean_url" | sed -E 's#https://github\.com/[^/]+/[^/]+/(blob|tree)/([^/]+)/.*#\2#') | |
| rel_path=$(echo "$clean_url" | sed -E 's#https://github\.com/[^/]+/[^/]+/(blob|tree)/[^/]+/(.*)#\2#') | |
| # URL-decode the relative path (handles %5B -> [, %5D -> ], spaces, etc.) | |
| # This is needed because GitHub URLs encode special characters like [ and ] | |
| rel_path=$(python3 -c "import sys, urllib.parse; print(urllib.parse.unquote(sys.argv[1]))" "$rel_path") | |
| # Handle HEAD ref specially | |
| if [ "$ref" = "HEAD" ]; then | |
| ref=$(git symbolic-ref refs/remotes/origin/HEAD 2>/dev/null | sed 's@^refs/remotes/origin/@@' || echo "main") | |
| fi | |
| # For specific commit SHAs, we need to fetch them | |
| if echo "$ref" | grep -qE '^[0-9a-f]{40}$'; then | |
| git fetch --depth 1 origin "$ref" 2>/dev/null || true | |
| fi | |
| # Check if path exists using git ls-tree (works with sparse checkout) | |
| # Use grep -qxF for literal string matching (paths may contain regex chars like [ and ]) | |
| if git ls-tree -r --name-only "$ref" -- "$rel_path" 2>/dev/null | grep -qxF "$rel_path" || \ | |
| git ls-tree -d --name-only "$ref" -- "$rel_path" 2>/dev/null | grep -qxF "$rel_path"; then | |
| echo "Verified: $url -> $rel_path (ref: $ref)" | |
| echo "$url" >> ../../../github-verified.txt | |
| verified_count=$((verified_count + 1)) | |
| else | |
| # Also check if it's a directory (tree link) | |
| if git ls-tree -d "$ref" -- "$rel_path" 2>/dev/null | grep -q .; then | |
| echo "Verified (directory): $url -> $rel_path (ref: $ref)" | |
| echo "$url" >> ../../../github-verified.txt | |
| verified_count=$((verified_count + 1)) | |
| else | |
| echo "MISSING: $url -> $rel_path (ref: $ref)" | |
| echo "- [LOCAL_MISSING] $url (path: $rel_path in $org/$repo_name, ref: $ref)" >> ../../../github-missing.txt | |
| missing_count=$((missing_count + 1)) | |
| fi | |
| fi | |
| fi | |
| cd ../../.. | |
| done < github-urls.txt | |
| echo "" | |
| echo "=== GitHub URL Verification Summary ===" | |
| echo "Total URLs checked: $total_urls" | |
| echo "Verified locally: $verified_count" | |
| echo "Missing/unverifiable: $missing_count" | |
| echo "verified_count=$verified_count" >> $GITHUB_OUTPUT | |
| echo "missing_count=$missing_count" >> $GITHUB_OUTPUT | |
| if [ "$missing_count" -gt 0 ]; then | |
| echo "has_missing=true" >> $GITHUB_OUTPUT | |
| echo "" | |
| echo "Missing URLs:" | |
| cat github-missing.txt | |
| else | |
| echo "has_missing=false" >> $GITHUB_OUTPUT | |
| fi | |
| # Cleanup cloned repos | |
| rm -rf .github-repos | |
| - name: Extract GitHub URLs for HTTP checking from source files | |
| id: extract_github_http | |
| run: | | |
| # Extract GitHub URLs that need HTTP checking (issues, compare, commit, discussions, non-fern-api PRs) | |
| # These are extracted from source files and passed directly to lychee | |
| # This avoids the --include filter issue where lychee applies it to input pages, not discovered links | |
| echo "Scanning repo for GitHub URLs to check via HTTP..." | |
| > github-http-urls.txt | |
| # Search for issues URLs | |
| grep -RhoE 'https://github\.com/[^/]+/[^/]+/issues/[^"'"'"')<>[:space:]]+' \ | |
| fern/ README.md \ | |
| --exclude-dir=.git --exclude-dir=.github-repos \ | |
| >> github-http-urls.txt 2>/dev/null || true | |
| # Search for compare URLs | |
| grep -RhoE 'https://github\.com/[^/]+/[^/]+/compare/[^"'"'"')<>[:space:]]+' \ | |
| fern/ README.md \ | |
| --exclude-dir=.git --exclude-dir=.github-repos \ | |
| >> github-http-urls.txt 2>/dev/null || true | |
| # Search for commit/commits URLs | |
| grep -RhoE 'https://github\.com/[^/]+/[^/]+/commits?/[^"'"'"')<>[:space:]]+' \ | |
| fern/ README.md \ | |
| --exclude-dir=.git --exclude-dir=.github-repos \ | |
| >> github-http-urls.txt 2>/dev/null || true | |
| # Search for discussions URLs | |
| grep -RhoE 'https://github\.com/[^/]+/[^/]+/discussions/[^"'"'"')<>[:space:]]+' \ | |
| fern/ README.md \ | |
| --exclude-dir=.git --exclude-dir=.github-repos \ | |
| >> github-http-urls.txt 2>/dev/null || true | |
| # Search for pull request URLs (excluding fern-api org - those are too slow to check) | |
| grep -RhoE 'https://github\.com/[^/]+/[^/]+/pull/[^"'"'"')<>[:space:]]+' \ | |
| fern/ README.md \ | |
| --exclude-dir=.git --exclude-dir=.github-repos \ | |
| 2>/dev/null | grep -v 'github\.com/fern-api/' \ | |
| >> github-http-urls.txt || true | |
| # Remove example/placeholder URLs (e.g., github.com/your-org/...) | |
| if [ -s github-http-urls.txt ]; then | |
| grep -v 'github\.com/your-org/' github-http-urls.txt > github-http-urls.filtered || true | |
| mv github-http-urls.filtered github-http-urls.txt | |
| fi | |
| # Deduplicate URLs | |
| sort -u github-http-urls.txt -o github-http-urls.txt | |
| total_urls=$(wc -l < github-http-urls.txt | tr -d ' ') | |
| echo "Found $total_urls unique GitHub URLs to check via HTTP" | |
| echo "github_http_count=$total_urls" >> $GITHUB_OUTPUT | |
| if [ "$total_urls" -gt 0 ]; then | |
| echo "URLs to check:" | |
| cat github-http-urls.txt | |
| fi | |
| - name: Upload URLs (early, for debugging) | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: urls | |
| path: | | |
| github-urls.txt | |
| github-http-urls.txt | |
| urls.txt | |
| if-no-files-found: ignore | |
| - name: Check GitHub links (very low concurrency to avoid 503 rate limiting) | |
| id: lychee_github | |
| if: steps.extract_github_http.outputs.github_http_count != '0' | |
| uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2 | |
| with: | |
| args: >- | |
| --no-progress | |
| --timeout 30 | |
| --max-retries 5 | |
| --retry-wait-time 20 | |
| --max-concurrency 2 | |
| github-http-urls.txt | |
| output: ./lychee-raw-github.md | |
| format: markdown | |
| fail: false | |
| jobSummary: false | |
| - name: Check non-GitHub links (high concurrency) | |
| id: lychee_main | |
| uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2 | |
| with: | |
| args: >- | |
| --config lychee.toml | |
| --no-progress | |
| --timeout 30 | |
| --max-retries 3 | |
| --retry-wait-time 10 | |
| --max-concurrency 20 | |
| --include-fragments | |
| --files-from urls.txt | |
| output: ./lychee-raw-main.md | |
| format: markdown | |
| fail: false | |
| jobSummary: false | |
| - name: Combine lychee outputs | |
| run: | | |
| # Combine both lychee runs into a single raw file for downstream processing | |
| : > lychee-raw.md | |
| [ -f lychee-raw-main.md ] && cat lychee-raw-main.md >> lychee-raw.md | |
| [ -f lychee-raw-github.md ] && cat lychee-raw-github.md >> lychee-raw.md | |
| echo "Combined lychee outputs into lychee-raw.md" | |
| - name: Extract 429 URLs and retry with exponential backoff | |
| id: retry429 | |
| run: | | |
| # Extract all 429 URLs from the raw lychee report | |
| # Note: 403s are excluded from broken links report but NOT retried (bot-blocking sites won't change) | |
| grep '\[429\]' ./lychee-raw.md | sed -E 's/.*<([^>]+)>.*/\1/' | sort -u > urls-429-all.txt || true | |
| # Initialize counters for repo-internal GitHub URLs | |
| verified_locally=0 | |
| missing_locally=0 | |
| # Process repo-internal GitHub URLs by checking if files exist locally | |
| # This avoids false positives from GitHub rate limiting while still catching real broken links | |
| > urls-429.txt | |
| > urls-429-repo-missing.txt | |
| while IFS= read -r url; do | |
| [ -z "$url" ] && continue | |
| # Check if this is a repo-internal GitHub URL (blob/main pattern) | |
| if echo "$url" | grep -qE '^https://github\.com/fern-api/docs/blob/main/'; then | |
| # Extract relative path from URL (strip prefix and query string) | |
| rel_path="${url#https://github.com/fern-api/docs/blob/main/}" | |
| rel_path="${rel_path%%\?*}" | |
| if [ -f "$rel_path" ]; then | |
| echo "Verified locally (file exists): $url -> $rel_path" | |
| verified_locally=$((verified_locally + 1)) | |
| else | |
| echo "MISSING locally: $url -> $rel_path" | |
| echo "- [LOCAL_MISSING] $url (expected path: $rel_path)" >> urls-429-repo-missing.txt | |
| missing_locally=$((missing_locally + 1)) | |
| fi | |
| else | |
| # Non-repo URL, add to retry list | |
| echo "$url" >> urls-429.txt | |
| fi | |
| done < urls-429-all.txt | |
| echo "Repo-internal GitHub URLs verified locally: $verified_locally" | |
| echo "Repo-internal GitHub URLs missing locally: $missing_locally" | |
| echo "verified_locally=$verified_locally" >> $GITHUB_OUTPUT | |
| echo "missing_locally=$missing_locally" >> $GITHUB_OUTPUT | |
| count=$(wc -l < urls-429.txt | tr -d ' ') | |
| echo "Found $count other URLs with 429 status to retry" | |
| echo "rate_limited_count=$count" >> $GITHUB_OUTPUT | |
| # Initialize still-failing file | |
| > urls-429-still-failing.txt | |
| # Add any locally-missing repo URLs to the still-failing list | |
| if [ -s urls-429-repo-missing.txt ]; then | |
| cat urls-429-repo-missing.txt >> urls-429-still-failing.txt | |
| fi | |
| if [ "$count" -eq "0" ] || [ ! -s urls-429.txt ]; then | |
| echo "No other 429 URLs to retry" | |
| else | |
| echo "Retrying $count URLs with exponential backoff..." | |
| while IFS= read -r url; do | |
| [ -z "$url" ] && continue | |
| delay=15 | |
| max_attempts=4 | |
| success=false | |
| for attempt in $(seq 1 $max_attempts); do | |
| echo "[$attempt/$max_attempts] Checking: $url" | |
| status=$(curl -s -o /dev/null -w "%{http_code}" \ | |
| -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" \ | |
| --max-time 30 \ | |
| "$url" || echo "000") | |
| if [ "$status" -lt 400 ]; then | |
| echo "OK on retry (status $status): $url" | |
| success=true | |
| break | |
| elif [ "$status" -eq 429 ]; then | |
| echo "Still 429, sleeping ${delay}s before next attempt..." | |
| sleep "$delay" | |
| delay=$((delay * 2)) | |
| else | |
| echo "Non-429 failure (status $status): $url" | |
| break | |
| fi | |
| done | |
| if [ "$success" = false ]; then | |
| echo "- [${status}] $url" >> urls-429-still-failing.txt | |
| fi | |
| done < urls-429.txt | |
| fi | |
| still_failing=$(wc -l < urls-429-still-failing.txt | tr -d ' ') | |
| echo "URLs still failing after retry: $still_failing" | |
| echo "still_failing_429=$still_failing" >> $GITHUB_OUTPUT | |
| if [ "$still_failing" -gt 0 ]; then | |
| echo "has_429_failures=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_429_failures=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Build errors-only report and summarize | |
| id: check_failures | |
| run: | | |
| # Only 4xx responses are actionable as source-level broken links. | |
| # 403 and 429 are handled separately. 5xx responses are reported as | |
| # informational because changing docs source cannot fix server errors. | |
| FAILURE_PATTERN='\[400\]|\[401\]|\[402\]|\[40[4-9]\]|\[41[0-9]\]|\[42[0-8]\]|\[4[3-9][0-9]\]' | |
| SERVER_ERROR_PATTERN='\[5[0-9]{2}\]' | |
| # Extract actionable broken links and informational server errors | |
| grep -E "$FAILURE_PATTERN" ./lychee-raw.md > broken-links-raw.txt 2>/dev/null || true | |
| grep -E "$SERVER_ERROR_PATTERN" ./lychee-raw.md > server-errors.txt 2>/dev/null || true | |
| # Filter out repo-internal GitHub URLs that exist locally | |
| > broken-links.txt | |
| while IFS= read -r line; do | |
| [ -z "$line" ] && continue | |
| # Extract URL and status code from the line (format: ... [STATUS] <URL> ...) | |
| url=$(echo "$line" | sed -E 's/.*<([^>]+)>.*/\1/') | |
| status=$(echo "$line" | sed -E 's/.*\[([0-9]+)\].*/\1/') | |
| # Skip malformed ToC anchor links (fern-platform rendering bug) | |
| if echo "$url" | grep -qE 'buildwithfern\.com/_/buildwithfern\.com/|app\.buildwithfern\.com/buildwithfern\.com/'; then | |
| continue | |
| fi | |
| # Check if this is a repo-internal GitHub URL (blob/main pattern) | |
| if echo "$url" | grep -qE '^https://github\.com/fern-api/docs/blob/main/'; then | |
| # Extract relative path from URL (strip prefix and query string) | |
| rel_path="${url#https://github.com/fern-api/docs/blob/main/}" | |
| rel_path="${rel_path%%\?*}" | |
| if [ -f "$rel_path" ]; then | |
| echo "Verified locally (file exists): $url -> $rel_path" | |
| else | |
| # File doesn't exist locally, this is a real broken link | |
| echo "$line" >> broken-links.txt | |
| fi | |
| else | |
| # Not a repo-internal URL, keep it in the broken links list | |
| echo "$line" >> broken-links.txt | |
| fi | |
| done < broken-links-raw.txt | |
| # Deduplicate error files (same URL may appear multiple times if linked from multiple pages) | |
| sort -u broken-links.txt -o broken-links.txt | |
| sort -u server-errors.txt -o server-errors.txt | |
| broken_count=$(wc -l < broken-links.txt | tr -d ' ') | |
| server_error_count=$(wc -l < server-errors.txt | tr -d ' ') | |
| # Get rate limit stats | |
| rate_limited="${STEPS_RETRY429_OUTPUTS_RATE_LIMITED_COUNT}" | |
| rate_limited=${rate_limited:-0} | |
| still_failing_429="${STEPS_RETRY429_OUTPUTS_STILL_FAILING_429}" | |
| still_failing_429=${still_failing_429:-0} | |
| # Get GitHub local verification stats | |
| github_verified="${STEPS_VERIFY_GITHUB_OUTPUTS_VERIFIED_COUNT}" | |
| github_verified=${github_verified:-0} | |
| github_missing="${STEPS_VERIFY_GITHUB_OUTPUTS_MISSING_COUNT}" | |
| github_missing=${github_missing:-0} | |
| # Build clean errors-only report | |
| cat > lychee-report.md << 'HEADER' | |
| # Link Check Report | |
| This report only shows broken links (errors). Redirects and successful links are not included. | |
| HEADER | |
| if [ "$broken_count" -gt 0 ]; then | |
| echo "## Broken Links ($broken_count)" >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| # Format: - [STATUS] URL | |
| sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/- [\1] \2/' broken-links.txt >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| fi | |
| if [ "$still_failing_429" -gt 0 ]; then | |
| echo "## Rate-Limited URLs (429) - Still Failing After Retry" >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| cat urls-429-still-failing.txt >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| fi | |
| if [ "$github_missing" -gt 0 ]; then | |
| echo "## GitHub URLs - Missing Locally ($github_missing)" >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| cat github-missing.txt >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| fi | |
| if [ "$server_error_count" -gt 0 ]; then | |
| echo "## Server Errors (5xx) - Informational" >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| echo "_These URLs returned server errors. They do not open a broken-link PR because docs source changes cannot fix them._" >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/- [\1] \2/' server-errors.txt >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| fi | |
| if [ "$broken_count" -eq 0 ] && [ "$still_failing_429" -eq 0 ] && [ "$github_missing" -eq 0 ]; then | |
| echo "No broken links found!" >> lychee-report.md | |
| fi | |
| echo "" >> lychee-report.md | |
| echo "---" >> lychee-report.md | |
| echo "" >> lychee-report.md | |
| echo "Rate-limited URLs (429) that recovered after retry: $rate_limited" >> lychee-report.md | |
| # Calculate recovered 429s (rate_limited - still_failing_429) | |
| recovered_429=0 | |
| if [ "$rate_limited" -gt 0 ]; then | |
| recovered_429=$((rate_limited - still_failing_429)) | |
| fi | |
| # Set output for failure detection | |
| if [ "$broken_count" -gt 0 ]; then | |
| echo "has_other_failures=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_other_failures=false" >> $GITHUB_OUTPUT | |
| fi | |
| # Extract lychee's summary table only (stop at first ## heading) | |
| sed -n '/^# Summary$/,/^## /p' ./lychee-raw.md | head -n -1 > lychee-summary-table.md | |
| # Print summary to logs | |
| echo "" | |
| cat lychee-summary-table.md | |
| echo "" | |
| # Get repo-internal GitHub URL counts | |
| verified_locally="${STEPS_RETRY429_OUTPUTS_VERIFIED_LOCALLY}" | |
| verified_locally=${verified_locally:-0} | |
| missing_locally="${STEPS_RETRY429_OUTPUTS_MISSING_LOCALLY}" | |
| missing_locally=${missing_locally:-0} | |
| echo "Recovery Info:" | |
| echo " Repo-internal GitHub URLs (429 verified locally): $verified_locally" | |
| echo " Repo-internal GitHub URLs (429 missing locally): $missing_locally" | |
| echo " Rate-limited (429) - recovered after retry: $recovered_429" | |
| echo " Rate-limited (429) - still failing: $still_failing_429" | |
| echo " GitHub URLs verified locally: $github_verified" | |
| echo " GitHub URLs missing locally: $github_missing" | |
| echo " Server errors (5xx, informational): $server_error_count" | |
| if [ "$broken_count" -gt 0 ]; then | |
| echo "" | |
| echo "BROKEN LINKS (non-429):" | |
| cat broken-links.txt | sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/ - [\1] \2/' | |
| fi | |
| if [ "$still_failing_429" -gt 0 ]; then | |
| echo "" | |
| echo "RATE-LIMITED URLs STILL FAILING (429):" | |
| cat urls-429-still-failing.txt | |
| fi | |
| if [ "$github_missing" -gt 0 ]; then | |
| echo "" | |
| echo "GITHUB URLs MISSING LOCALLY:" | |
| cat github-missing.txt | |
| fi | |
| # Build GitHub Step Summary using lychee's table + 429 info + persistent errors only | |
| { | |
| # Copy lychee's summary table | |
| cat lychee-summary-table.md | |
| # Add recovery info | |
| echo "" | |
| echo "### Local Verification & Rate Limit Recovery" | |
| echo "" | |
| echo "| Status | Count |" | |
| echo "|--------|------:|" | |
| echo "| Repo-internal GitHub URLs (429 verified locally) | $verified_locally |" | |
| echo "| Repo-internal GitHub URLs (429 missing locally) | $missing_locally |" | |
| echo "| Rate-limited (429) recovered after retry | $recovered_429 |" | |
| echo "| Rate-limited (429) still failing | $still_failing_429 |" | |
| echo "| GitHub URLs verified locally | $github_verified |" | |
| echo "| GitHub URLs missing locally | $github_missing |" | |
| echo "| Server errors (5xx, informational) | $server_error_count |" | |
| echo "" | |
| # Show persistent errors only (non-429 broken links + 429s that didn't recover) | |
| echo "## Errors (after retry)" | |
| echo "" | |
| has_persistent_errors=false | |
| if [ "$broken_count" -gt 0 ]; then | |
| has_persistent_errors=true | |
| sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/- [\1] \2/' broken-links.txt | |
| fi | |
| if [ "$still_failing_429" -gt 0 ]; then | |
| has_persistent_errors=true | |
| cat urls-429-still-failing.txt | |
| fi | |
| if [ "$github_missing" -gt 0 ]; then | |
| has_persistent_errors=true | |
| cat github-missing.txt | |
| fi | |
| if [ "$has_persistent_errors" = false ]; then | |
| echo "No errors remained after retry." | |
| fi | |
| echo "" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| env: | |
| STEPS_RETRY429_OUTPUTS_RATE_LIMITED_COUNT: ${{ steps.retry429.outputs.rate_limited_count }} | |
| STEPS_RETRY429_OUTPUTS_STILL_FAILING_429: ${{ steps.retry429.outputs.still_failing_429 }} | |
| STEPS_VERIFY_GITHUB_OUTPUTS_VERIFIED_COUNT: ${{ steps.verify_github.outputs.verified_count }} | |
| STEPS_VERIFY_GITHUB_OUTPUTS_MISSING_COUNT: ${{ steps.verify_github.outputs.missing_count }} | |
| STEPS_RETRY429_OUTPUTS_VERIFIED_LOCALLY: ${{ steps.retry429.outputs.verified_locally }} | |
| STEPS_RETRY429_OUTPUTS_MISSING_LOCALLY: ${{ steps.retry429.outputs.missing_locally }} | |
| - name: Upload errors-only report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: lychee-report | |
| path: ./lychee-report.md | |
| if-no-files-found: ignore | |
| - name: Upload lychee outputs and verification results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: lychee-outputs | |
| path: | | |
| github-verified.txt | |
| github-missing.txt | |
| lychee-raw-main.md | |
| lychee-raw-github.md | |
| if-no-files-found: ignore | |
| - name: Fix broken links with Claude | |
| id: fix-links | |
| if: steps.check_failures.outputs.has_other_failures == 'true' || steps.retry429.outputs.has_429_failures == 'true' || steps.verify_github.outputs.has_missing == 'true' | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: | # zizmor: ignore[adhoc-packages] claude-code CLI is only needed in this ephemeral CI job | |
| set -euo pipefail | |
| # Assemble the list of broken links to hand to Claude. | |
| { | |
| echo "## Non-429 broken links" | |
| if [ -s broken-links.txt ]; then | |
| sed -E 's/.*\[([0-9]+)\].*<([^>]+)>.*/- [\1] \2/' broken-links.txt | |
| else | |
| echo "(none)" | |
| fi | |
| echo "" | |
| echo "## Rate-limited URLs (429) still failing after retry" | |
| if [ -s urls-429-still-failing.txt ]; then | |
| cat urls-429-still-failing.txt | |
| else | |
| echo "(none)" | |
| fi | |
| echo "" | |
| echo "## GitHub URLs missing locally" | |
| if [ -s github-missing.txt ]; then | |
| cat github-missing.txt | |
| else | |
| echo "(none)" | |
| fi | |
| } > broken-links-list.md | |
| # Build the prompt. | |
| cat > claude-prompt.md << 'PROMPT' | |
| You are fixing broken links in the fern-api/docs documentation repository. | |
| A scheduled link checker found the broken links listed at the end of this prompt. | |
| Rules: | |
| 1. For each broken URL, search the source under `fern/` and `README.md` to find where | |
| the link is defined (grep for the URL, its path, or a distinctive slug), and fix it | |
| at the source. | |
| 2. Internal links between docs pages must be published `/learn/...` URL paths built from | |
| the YAML config, never file paths or relative paths. Read the "Link checking" section | |
| of `AGENTS.md` and the relevant `docs.yml` files to construct the correct path. | |
| 3. Common fixes: correct a moved or renamed slug, add a missing path segment (e.g. | |
| `/learn`), update an outdated commit SHA to a branch name, or fix URL-encoded paths. | |
| 4. If you cannot confidently determine the correct replacement (e.g. the only signal is a | |
| 5xx/connection error or rate limiting, or you cannot find where the link is defined), | |
| DO NOT guess or delete the link. Leave it unchanged and record it in | |
| `broken-links-unfixed.md` with a one-line reason. | |
| 5. Only fix the specific failures listed below. Do not touch other links that merely look | |
| similar. | |
| 6. Do not edit anything under `fern/translations/` — those files are generated. | |
| 7. When done, write a short bullet-list summary of what you changed to | |
| `broken-links-summary.md`. | |
| The broken links to fix: | |
| PROMPT | |
| cat broken-links-list.md >> claude-prompt.md | |
| # Install and run Claude Code headlessly. | |
| npm install -g @anthropic-ai/claude-code | |
| claude -p "$(cat claude-prompt.md)" \ | |
| --dangerously-skip-permissions \ | |
| --max-turns 80 \ | |
| --model sonnet \ | |
| --output-format text || echo "Claude exited non-zero; will still check for changes" | |
| # Remove scratch files so they are never committed. | |
| rm -f claude-prompt.md broken-links-list.md | |
| if [ -n "$(git status --porcelain -- fern README.md)" ]; then | |
| echo "changes_made=true" >> "$GITHUB_OUTPUT" | |
| echo "Source changes made by Claude:" | |
| git status --porcelain -- fern README.md | |
| else | |
| echo "changes_made=false" >> "$GITHUB_OUTPUT" | |
| echo "Claude made no source changes." | |
| fi | |
| - name: Create or update PR with fixes | |
| if: steps.fix-links.outputs.changes_made == 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| BRANCH="chore/fix-broken-links" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| # Build the PR body from Claude's summary plus the full detected report. | |
| { | |
| echo "Automated fixes for broken links found by the scheduled link checker." | |
| echo "" | |
| echo "[View workflow run](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" | |
| echo "" | |
| if [ -f broken-links-summary.md ]; then | |
| echo "## Summary of changes" | |
| echo "" | |
| cat broken-links-summary.md | |
| echo "" | |
| fi | |
| if [ -f broken-links-unfixed.md ]; then | |
| echo "## Links needing manual review" | |
| echo "" | |
| cat broken-links-unfixed.md | |
| echo "" | |
| fi | |
| if [ -f lychee-report.md ]; then | |
| echo "## All detected broken links" | |
| echo "" | |
| cat lychee-report.md | |
| fi | |
| } > pr-body.md | |
| # Stage only source fixes (never the report/scratch artifacts). | |
| git add fern README.md | |
| git checkout -b "$BRANCH" | |
| git commit -m "fix: repair broken documentation links | |
| Automated fix generated by the scheduled link checker using Claude. | |
| Detected by workflow run ${GITHUB_RUN_ID}." | |
| git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| git push --force origin "$BRANCH" | |
| existing=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number' 2>/dev/null || echo "") | |
| if [ -n "$existing" ]; then | |
| echo "Updating existing PR #$existing" | |
| gh pr edit "$existing" --body-file pr-body.md || true | |
| else | |
| gh pr create --draft --base main --head "$BRANCH" \ | |
| --title "Fix broken links" \ | |
| --body-file pr-body.md | |
| fi | |
| - name: Fail if there are broken links | |
| if: steps.check_failures.outputs.has_other_failures == 'true' || steps.retry429.outputs.has_429_failures == 'true' || steps.verify_github.outputs.has_missing == 'true' | |
| run: | | |
| echo "Link check failed!" | |
| if [ "${STEPS_CHECK_FAILURES_OUTPUTS_HAS_OTHER_FAILURES}" == "true" ]; then | |
| echo "There are broken links (non-429 failures) in the report." | |
| fi | |
| if [ "${STEPS_RETRY429_OUTPUTS_HAS_429_FAILURES}" == "true" ]; then | |
| echo "Some URLs still returned 429 after exponential backoff retry." | |
| echo "These URLs may need to be excluded or the rate limit needs more time to reset." | |
| fi | |
| if [ "${STEPS_VERIFY_GITHUB_OUTPUTS_HAS_MISSING}" == "true" ]; then | |
| echo "Some GitHub URLs point to paths that don't exist in the repos." | |
| fi | |
| exit 1 | |
| env: | |
| STEPS_CHECK_FAILURES_OUTPUTS_HAS_OTHER_FAILURES: ${{ steps.check_failures.outputs.has_other_failures }} | |
| STEPS_RETRY429_OUTPUTS_HAS_429_FAILURES: ${{ steps.retry429.outputs.has_429_failures }} | |
| STEPS_VERIFY_GITHUB_OUTPUTS_HAS_MISSING: ${{ steps.verify_github.outputs.has_missing }} |