Automation Health Monitor #1581
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Automation Health Monitor | |
| # Runs every 4 hours to check for stuck/broken automation state. | |
| # Posts a rolling report to a pinned "Automation Health" issue. | |
| # Also dispatches auto-fixes where possible (re-dispatch stuck reviews, etc.). | |
| on: | |
| schedule: | |
| - cron: '0 */4 * * *' # Every 4 hours | |
| workflow_dispatch: # Manual trigger for testing | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| actions: write | |
| jobs: | |
| health-check: | |
| name: Check automation health | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPO: ${{ github.repository }} | |
| steps: | |
| # Needed by Check 6, which reads .github/workflows/*.yml to discover | |
| # which workflows have a `schedule:` trigger. | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run health checks | |
| id: health | |
| run: | | |
| set -e | |
| TIMESTAMP=$(date -u '+%Y-%m-%d %H:%M UTC') | |
| echo "=== Automation Health Check — $TIMESTAMP ===" | |
| # Portable ISO-8601 → epoch helper (avoids GNU vs BSD date differences) | |
| iso_to_epoch() { | |
| python3 -c "import sys, datetime; ts = datetime.datetime.fromisoformat(sys.argv[1].replace('Z', '+00:00')); print(int(ts.timestamp()))" "$1" 2>/dev/null || echo "0" | |
| } | |
| # ── Check 1: Stuck AI reviews ───────────────────────────────────────── | |
| # PRs with ai-reviewing label where the latest ai-review.yml run is >2h old | |
| echo "" | |
| echo "--- Check 1: Stuck AI reviews ---" | |
| stuck_reviews="" | |
| stuck_review_count=0 | |
| reviewing_prs=$(gh api "repos/$REPO/pulls?state=open&per_page=100" \ | |
| --jq '[.[] | select(.labels[].name == "ai-reviewing")] | .[].number' \ | |
| 2>/dev/null || echo "") | |
| for pr_num in $reviewing_prs; do | |
| # Find the latest ai-review.yml run | |
| latest_run=$(gh run list --workflow=ai-review.yml --repo "$REPO" --limit 20 \ | |
| --json databaseId,createdAt,status,conclusion \ | |
| --jq "sort_by(.createdAt) | reverse | first" \ | |
| 2>/dev/null || echo "") | |
| if [ -z "$latest_run" ]; then | |
| continue | |
| fi | |
| run_status=$(echo "$latest_run" | jq -r '.status') | |
| created_at=$(echo "$latest_run" | jq -r '.createdAt') | |
| age_seconds=$(( $(date -u +%s) - $(iso_to_epoch "$created_at") )) | |
| age_hours=$(( age_seconds / 3600 )) | |
| if [ "$run_status" != "completed" ] && [ "$age_hours" -ge 2 ]; then | |
| # AI review disabled — Copilot handles reviews. Just clear stale label. | |
| echo " PR #$pr_num: stuck in ai-reviewing for ${age_hours}h — clearing stale label" | |
| gh pr edit "$pr_num" --repo "$REPO" --remove-label "ai-reviewing" 2>/dev/null || true | |
| stuck_reviews="$stuck_reviews PR #$pr_num (${age_hours}h, label-cleared)" | |
| stuck_review_count=$((stuck_review_count + 1)) | |
| elif [ "$run_status" = "completed" ] && [ "$age_hours" -ge 1 ]; then | |
| # Review completed but label not cleared — remove stale label | |
| echo " PR #$pr_num: ai-review completed but label stale — clearing" | |
| gh pr edit "$pr_num" --repo "$REPO" --remove-label "ai-reviewing" 2>/dev/null || true | |
| fi | |
| done | |
| if [ "$stuck_review_count" -eq 0 ]; then | |
| check1_status="✅ None" | |
| check1_detail="" | |
| else | |
| check1_status="⚠️ $stuck_review_count found" | |
| check1_detail="$stuck_reviews" | |
| fi | |
| # ── Check 2: Stuck AI fixers ────────────────────────────────────────── | |
| # claude-code.yml runs with fix_ai_review action queued/in_progress for >1h | |
| echo "" | |
| echo "--- Check 2: Stuck AI fixers ---" | |
| stuck_fixers=$(gh run list --workflow=claude-code.yml --repo "$REPO" --limit 30 \ | |
| --json databaseId,status,createdAt,displayTitle \ | |
| --jq '[.[] | select(.status == "queued" or .status == "in_progress")]' \ | |
| 2>/dev/null || echo "[]") | |
| stuck_fixer_count=0 | |
| stuck_fixer_detail="" | |
| while IFS= read -r run; do | |
| run_id=$(echo "$run" | jq -r '.databaseId') | |
| created_at=$(echo "$run" | jq -r '.createdAt') | |
| title=$(echo "$run" | jq -r '.displayTitle') | |
| age_seconds=$(( $(date -u +%s) - $(iso_to_epoch "$created_at") )) | |
| age_minutes=$(( age_seconds / 60 )) | |
| if [ "$age_minutes" -ge 60 ]; then | |
| echo " Run #$run_id stuck for ${age_minutes}min: $title" | |
| stuck_fixer_count=$((stuck_fixer_count + 1)) | |
| stuck_fixer_detail="$stuck_fixer_detail Run #$run_id (${age_minutes}min)" | |
| fi | |
| done < <(echo "$stuck_fixers" | jq -c '.[]' 2>/dev/null || true) | |
| if [ "$stuck_fixer_count" -eq 0 ]; then | |
| check2_status="✅ None" | |
| check2_detail="" | |
| else | |
| check2_status="⚠️ $stuck_fixer_count found" | |
| check2_detail="$stuck_fixer_detail" | |
| fi | |
| # ── Check 3: Unresolved rebase conflicts ────────────────────────────── | |
| # PRs with bot "Rebase conflict detected" comment but no push in last 6h | |
| echo "" | |
| echo "--- Check 3: Unresolved rebase conflicts ---" | |
| conflict_count=0 | |
| conflict_detail="" | |
| open_prs=$(gh api "repos/$REPO/pulls?state=open&per_page=100" \ | |
| --jq '.[].number' 2>/dev/null || echo "") | |
| for pr_num in $open_prs; do | |
| # Check for unresolved conflict comment in last 6 hours | |
| conflict_comment=$(gh api "repos/$REPO/issues/${pr_num}/comments" \ | |
| --jq '[.[] | select(.body | test("Rebase conflict detected";"")) | select(.user.login == "github-actions[bot]")] | sort_by(.created_at) | last' \ | |
| 2>/dev/null || echo "") | |
| if [ -z "$conflict_comment" ] || [ "$conflict_comment" = "null" ]; then | |
| continue | |
| fi | |
| conflict_at=$(echo "$conflict_comment" | jq -r '.created_at // empty') | |
| if [ -z "$conflict_at" ]; then | |
| continue | |
| fi | |
| age_seconds=$(( $(date -u +%s) - $(date -u -d "$conflict_at" +%s 2>/dev/null || date -u -j -f "%Y-%m-%dT%H:%M:%SZ" "$conflict_at" +%s 2>/dev/null || echo "0") )) | |
| age_hours=$(( age_seconds / 3600 )) | |
| if [ "$age_hours" -ge 6 ]; then | |
| echo " PR #$pr_num has unresolved conflict for ${age_hours}h" | |
| conflict_count=$((conflict_count + 1)) | |
| conflict_detail="$conflict_detail PR #$pr_num (${age_hours}h)" | |
| # Nudge with a comment if last nudge was >24h ago | |
| last_nudge=$(gh api "repos/$REPO/issues/${pr_num}/comments" \ | |
| --jq '[.[] | select(.body | test("health-nudge";""))] | last | .created_at // empty' \ | |
| 2>/dev/null || echo "") | |
| nudge_age=999999 | |
| if [ -n "$last_nudge" ]; then | |
| nudge_age=$(( ($(date -u +%s) - $(iso_to_epoch "$last_nudge")) / 3600 )) | |
| fi | |
| if [ "$nudge_age" -ge 24 ]; then | |
| gh pr comment "$pr_num" --repo "$REPO" \ | |
| --body "<!-- health-nudge --> | |
| ⏰ **Health check reminder**: This PR has an unresolved rebase conflict for ${age_hours}h. Comment \`/rebase\` to retry automatic resolution." \ | |
| 2>/dev/null || true | |
| fi | |
| fi | |
| done | |
| if [ "$conflict_count" -eq 0 ]; then | |
| check3_status="✅ None" | |
| check3_detail="" | |
| else | |
| check3_status="⚠️ $conflict_count found" | |
| check3_detail="$conflict_detail" | |
| fi | |
| # ── Check 4: Orphaned agent-work issues ─────────────────────────────── | |
| # Issues with agent-work label but no open PR for >24h | |
| echo "" | |
| echo "--- Check 4: Orphaned agent-work issues ---" | |
| orphan_count=0 | |
| orphan_detail="" | |
| agent_issues=$(gh api "repos/$REPO/issues?labels=agent-work&state=open&per_page=100" \ | |
| --jq '.[] | select(.pull_request == null) | {number: .number, title: .title, created_at: .created_at}' \ | |
| 2>/dev/null || echo "") | |
| while IFS= read -r issue; do | |
| issue_num=$(echo "$issue" | jq -r '.number') | |
| issue_title=$(echo "$issue" | jq -r '.title') | |
| created_at=$(echo "$issue" | jq -r '.created_at') | |
| age_seconds=$(( $(date -u +%s) - $(iso_to_epoch "$created_at") )) | |
| age_hours=$(( age_seconds / 3600 )) | |
| if [ "$age_hours" -ge 24 ]; then | |
| # Check if there's actually a PR for this issue (linked or branch named agent/issue-N) | |
| linked_pr=$(gh pr list --repo "$REPO" --search "agent/issue-${issue_num}" --state open --json number --jq '.[0].number' 2>/dev/null || echo "") | |
| if [ -z "$linked_pr" ]; then | |
| echo " Issue #$issue_num orphaned for ${age_hours}h: $issue_title" | |
| orphan_count=$((orphan_count + 1)) | |
| orphan_detail="$orphan_detail #$issue_num (${age_hours}h)" | |
| fi | |
| fi | |
| done < <(echo "$agent_issues" | jq -c '.' 2>/dev/null | grep -v '^$' || true) | |
| if [ "$orphan_count" -eq 0 ]; then | |
| check4_status="✅ None" | |
| check4_detail="" | |
| else | |
| check4_status="⚠️ $orphan_count found" | |
| check4_detail="$orphan_detail" | |
| fi | |
| # ── Check 5: Workflow failure rate (last 24h) ───────────────────────── | |
| # Catches workflows that fail *often*. By construction it cannot catch | |
| # a workflow that fails rarely-but-always — a weekly cron produces at | |
| # most one failure per 24h window and can never cross the threshold | |
| # below. That blind spot is Check 6's job. | |
| echo "" | |
| echo "--- Check 5: Workflow failure rate ---" | |
| check5_status="✅ OK" | |
| check5_detail="" | |
| noisy_count=0 | |
| # Discover workflows instead of hardcoding a watch list, so anything | |
| # added later is covered automatically. Previously this iterated four | |
| # named files and every other workflow in the repo was unmonitored. | |
| all_workflows=$(gh api "repos/$REPO/actions/workflows" --paginate \ | |
| --jq '.workflows[] | select(.state == "active") | .path' \ | |
| 2>/dev/null | sed 's|.*/||' | sort -u) | |
| if [ -z "$all_workflows" ]; then | |
| check5_status="⚠️ Could not enumerate workflows" | |
| fi | |
| for workflow in $all_workflows; do | |
| # --status failure filters server-side, so `--limit` counts failures | |
| # rather than a mix of runs, and in-progress runs (whose conclusion | |
| # is null) can never be miscounted as failures. | |
| failures=$(gh run list --workflow="$workflow" --repo "$REPO" \ | |
| --status failure --limit 100 \ | |
| --json createdAt \ | |
| --jq "now as \$now | [.[] | select((\$now - (.createdAt | fromdateiso8601)) < 86400)] | length" \ | |
| 2>/dev/null || echo "0") | |
| echo " $workflow: ${failures:-0} failure(s) in 24h" | |
| if [ "${failures:-0}" -gt 3 ]; then | |
| check5_status="⚠️ High failure rate" | |
| check5_detail="$check5_detail $workflow: ${failures}/24h" | |
| noisy_count=$((noisy_count + 1)) | |
| fi | |
| done | |
| # Only offenders go in the detail column — listing every workflow in | |
| # the repo would swamp the report table. | |
| workflow_total=$(echo "$all_workflows" | grep -c . || echo "0") | |
| if [ "$noisy_count" -eq 0 ] && [ -n "$all_workflows" ]; then | |
| check5_detail="${workflow_total} workflows checked, none over 3 failures/24h" | |
| fi | |
| # ── Write outputs for report step ───────────────────────────────────── | |
| { | |
| echo "timestamp=$TIMESTAMP" | |
| echo "check1_status=$check1_status" | |
| echo "check1_detail=$check1_detail" | |
| echo "check2_status=$check2_status" | |
| echo "check2_detail=$check2_detail" | |
| echo "check3_status=$check3_status" | |
| echo "check3_detail=$check3_detail" | |
| echo "check4_status=$check4_status" | |
| echo "check4_detail=$check4_detail" | |
| echo "check5_status=$check5_status" | |
| echo "check5_detail=$check5_detail" | |
| } >> "$GITHUB_OUTPUT" | |
| # ── Check 6: Stale scheduled workflows ────────────────────────────────── | |
| # Orthogonal to Check 5's failure *rate*: this asks when each scheduled | |
| # workflow's schedule last actually succeeded, and alerts when that is | |
| # older than a multiple of its own cron interval. This is the check that | |
| # catches a low-frequency job failing 100% of the time — the failure mode | |
| # that let update-catalogs.yml rot through 23 consecutive scheduled runs | |
| # over five months without a single alert. | |
| # The staleness scanner parses workflow YAML by hand, so a regression in | |
| # it fails *silently* — it just stops finding crons and the report goes | |
| # green. No other workflow runs these tests, so they run here, next to | |
| # the only consumer. | |
| # | |
| # Deliberately NOT continue-on-error: a scanner that fails its own tests | |
| # is not a monitor, and the job should go red. Check 6 below is gated on | |
| # this passing, so the report renders "check did not report" for | |
| # staleness rather than a verdict from an unverified scanner. | |
| - name: Check 6a — self-test the staleness scanner | |
| id: staleness_selftest | |
| if: always() | |
| run: python3 .github/scripts/test_scheduled_workflow_staleness.py | |
| - name: Check 6 — stale scheduled workflows | |
| id: staleness | |
| # Independent of checks 1-5: those run under `set -e`, and without | |
| # always() a failure there would skip this step entirely, dropping | |
| # staleness from a report that is otherwise built to survive a | |
| # partial run. The one thing it *does* depend on is its own self-test | |
| # — skipping here writes no output, which the report step renders as | |
| # "check did not report" instead of trusting a broken scanner. | |
| if: always() && steps.staleness_selftest.outcome == 'success' | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| echo "--- Check 6: Stale scheduled workflows ---" | |
| python3 .github/scripts/scheduled-workflow-staleness.py \ | |
| --repo "$REPO" \ | |
| --workflows-dir .github/workflows \ | |
| --format github | |
| - name: Post health report to pinned issue | |
| # Runs even if an earlier check step failed, so a partial report still | |
| # reaches the issue instead of the whole report vanishing. | |
| if: always() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TIMESTAMP: ${{ steps.health.outputs.timestamp }} | |
| CHECK1_STATUS: ${{ steps.health.outputs.check1_status }} | |
| CHECK1_DETAIL: ${{ steps.health.outputs.check1_detail }} | |
| CHECK2_STATUS: ${{ steps.health.outputs.check2_status }} | |
| CHECK2_DETAIL: ${{ steps.health.outputs.check2_detail }} | |
| CHECK3_STATUS: ${{ steps.health.outputs.check3_status }} | |
| CHECK3_DETAIL: ${{ steps.health.outputs.check3_detail }} | |
| CHECK4_STATUS: ${{ steps.health.outputs.check4_status }} | |
| CHECK4_DETAIL: ${{ steps.health.outputs.check4_detail }} | |
| CHECK5_STATUS: ${{ steps.health.outputs.check5_status }} | |
| CHECK5_DETAIL: ${{ steps.health.outputs.check5_detail }} | |
| CHECK6_STATUS: ${{ steps.staleness.outputs.check6_status }} | |
| CHECK6_DETAIL: ${{ steps.staleness.outputs.check6_detail }} | |
| run: | | |
| # A check step that died leaves its output empty. Render that as an | |
| # explicit "did not run" rather than a blank cell, which reads as OK. | |
| for n in 1 2 3 4 5 6; do | |
| var="CHECK${n}_STATUS" | |
| if [ -z "${!var}" ]; then | |
| printf -v "$var" '%s' '❓ check did not report' | |
| fi | |
| done | |
| TIMESTAMP="${TIMESTAMP:-$(date -u '+%Y-%m-%d %H:%M UTC')}" | |
| # Build report body | |
| { | |
| echo '<!-- automation-health-report -->' | |
| echo "## Automation Health Report — ${TIMESTAMP}" | |
| echo '' | |
| echo '| Check | Status | Details |' | |
| echo '|-------|--------|---------|' | |
| echo "| Stuck AI reviews | ${CHECK1_STATUS} | ${CHECK1_DETAIL} |" | |
| echo "| Stuck fixers | ${CHECK2_STATUS} | ${CHECK2_DETAIL} |" | |
| echo "| Rebase conflicts | ${CHECK3_STATUS} | ${CHECK3_DETAIL} |" | |
| echo "| Orphaned agent-work | ${CHECK4_STATUS} | ${CHECK4_DETAIL} |" | |
| echo "| Workflow failures (24h) | ${CHECK5_STATUS} | ${CHECK5_DETAIL} |" | |
| echo "| Stale scheduled workflows | ${CHECK6_STATUS} | ${CHECK6_DETAIL} |" | |
| echo '' | |
| echo '---' | |
| echo '*Auto-generated every 4h by [repo-health.yml](../actions/workflows/repo-health.yml). Trigger manually via [Actions tab](../actions/workflows/repo-health.yml).*' | |
| } > /tmp/health_report.md | |
| # Find or create the pinned Automation Health issue. | |
| # | |
| # This block used to swallow every failure with `2>/dev/null || echo ""` | |
| # and then print "posted to issue #unknown" while exiting green — so | |
| # the report was never actually posted anywhere, and the monitor | |
| # reported SUCCESS every 4h for five months while findings piled up | |
| # in a log nobody reads. Failures here are now fatal and loud: if the | |
| # report cannot reach the issue, this workflow is not monitoring | |
| # anything and must go red. | |
| HEALTH_ISSUE=$(gh issue list --repo "$REPO" \ | |
| --search "Automation Health Monitor in:title" \ | |
| --state open --limit 1 \ | |
| --json number --jq '.[0].number // empty') | |
| if [ -z "$HEALTH_ISSUE" ]; then | |
| echo "No open Automation Health issue found — creating one." | |
| # `gh issue create` prints the issue URL on stdout; it has no --jq | |
| # flag (passing one made the whole command fail). Take the trailing | |
| # path component as the issue number. | |
| if ! ISSUE_URL=$(gh issue create \ | |
| --repo "$REPO" \ | |
| --title "Automation Health Monitor" \ | |
| --body-file /tmp/health_report.md); then | |
| echo "::error::Failed to create the Automation Health issue — health report has nowhere to go." | |
| exit 1 | |
| fi | |
| HEALTH_ISSUE="${ISSUE_URL##*/}" | |
| echo "Created issue #$HEALTH_ISSUE ($ISSUE_URL)" | |
| # Labels are applied AFTER creation, one at a time, and are never | |
| # fatal. `--label` on a label that does not exist in the repo fails | |
| # the create outright; issue existence must not depend on repo | |
| # label configuration. | |
| for wanted in automation monitoring; do | |
| gh issue edit "$HEALTH_ISSUE" --repo "$REPO" --add-label "$wanted" >/dev/null 2>&1 \ | |
| || echo "::notice::Label '$wanted' not applied (create it in the repo to categorise this issue)." | |
| done | |
| else | |
| echo "Updating issue #$HEALTH_ISSUE..." | |
| if ! gh issue edit "$HEALTH_ISSUE" --repo "$REPO" \ | |
| --body-file /tmp/health_report.md; then | |
| echo "::error::Failed to update issue #$HEALTH_ISSUE with the health report." | |
| exit 1 | |
| fi | |
| fi | |
| # Guard against a parse surprise leaving a non-numeric value behind. | |
| case "$HEALTH_ISSUE" in | |
| ''|*[!0-9]*) | |
| echo "::error::Could not determine the health issue number (got '${HEALTH_ISSUE}')." | |
| exit 1 | |
| ;; | |
| esac | |
| echo "Health report posted to issue #${HEALTH_ISSUE}" |