Repository navigation
Flaky Test Tracker #240
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Flaky Test Tracker | |
| on: | |
| schedule: | |
| # Run daily at 6:00 UTC | |
| - cron: '0 6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| num_runs: | |
| description: 'Number of recent CI runs to analyze per workflow' | |
| required: false | |
| default: '30' | |
| type: string | |
| permissions: | |
| contents: write | |
| actions: read | |
| issues: write | |
| env: | |
| FLAKY_RUNS_TO_ANALYZE: ${{ inputs.num_runs || '30' }} | |
| FLAKY_WORKFLOWS: 'test-linux.yml' | |
| jobs: | |
| analyze-flaky-tests: | |
| runs-on: ubuntu-latest | |
| # One report at a time, since each one appends to flaky/data.json. With | |
| # `queue: max`, later ones wait instead of being cancelled. Other jobs that | |
| # push to gh-pages write other folders, and the push below rebases onto | |
| # them. | |
| concurrency: | |
| group: gh-pages-flaky | |
| queue: max | |
| outputs: | |
| flaky_count: ${{ steps.analyze.outputs.flaky_count }} | |
| new_flaky_count: ${{ steps.analyze.outputs.new_flaky_count }} | |
| resolved_count: ${{ steps.analyze.outputs.resolved_count }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Checkout gh-pages branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| path: gh-pages | |
| fetch-depth: 1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Analyze flaky tests | |
| id: analyze | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| run: | | |
| python .github/scripts/analyze_flaky_tests.py \ | |
| --runs "$FLAKY_RUNS_TO_ANALYZE" \ | |
| --workflows "$FLAKY_WORKFLOWS" \ | |
| --previous-report gh-pages/flaky/flaky-tests.json \ | |
| --output-dir flaky-reports | |
| - name: Generate dashboard | |
| run: | | |
| python .github/scripts/generate_dashboard.py \ | |
| --input-dir flaky-reports \ | |
| --output-dir flaky-reports | |
| - name: Upload reports as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: flaky-test-reports | |
| path: flaky-reports/ | |
| retention-days: 90 | |
| - name: Deploy to GitHub Pages | |
| run: | | |
| # Create flaky directory if it doesn't exist | |
| mkdir -p gh-pages/flaky | |
| # Copy reports | |
| cp flaky-reports/flaky-tests.json gh-pages/flaky/ | |
| cp flaky-reports/flaky-tests.md gh-pages/flaky/ | |
| cp flaky-reports/badge.json gh-pages/flaky/ | |
| cp flaky-reports/index.html gh-pages/flaky/ | |
| # Update historical data | |
| if [ -f gh-pages/flaky/data.json ]; then | |
| # Merge new data point with existing history | |
| python -c " | |
| import json | |
| from datetime import datetime | |
| with open('gh-pages/flaky/data.json') as f: | |
| history = json.load(f) | |
| with open('flaky-reports/flaky-tests.json') as f: | |
| new_report = json.load(f) | |
| new_point = { | |
| 'date': datetime.utcnow().strftime('%Y-%m-%d'), | |
| 'flaky_count': new_report['summary']['flaky_count'], | |
| 'new_flaky_count': new_report['summary'].get('new_flaky_count', 0), | |
| 'resolved_count': new_report['summary'].get('resolved_count', 0), | |
| 'total_tests': new_report['summary']['total_tests'], | |
| } | |
| history['history'].append(new_point) | |
| # Keep last 90 days | |
| history['history'] = history['history'][-90:] | |
| with open('gh-pages/flaky/data.json', 'w') as f: | |
| json.dump(history, f, indent=2) | |
| " | |
| else | |
| # Create initial data file | |
| python -c " | |
| import json | |
| from datetime import datetime | |
| with open('flaky-reports/flaky-tests.json') as f: | |
| report = json.load(f) | |
| history = { | |
| 'history': [{ | |
| 'date': datetime.utcnow().strftime('%Y-%m-%d'), | |
| 'flaky_count': report['summary']['flaky_count'], | |
| 'new_flaky_count': report['summary'].get('new_flaky_count', 0), | |
| 'resolved_count': report['summary'].get('resolved_count', 0), | |
| 'total_tests': report['summary']['total_tests'], | |
| }] | |
| } | |
| with open('gh-pages/flaky/data.json', 'w') as f: | |
| json.dump(history, f, indent=2) | |
| " | |
| fi | |
| # Commit and push | |
| cd gh-pages | |
| git config user.name 'github-actions[bot]' | |
| git config user.email 'github-actions[bot]@users.noreply.github.com' | |
| git add flaky/ | |
| git diff --staged --quiet || git commit -m "Update flaky test report - $(date -u +%Y-%m-%d)" | |
| for attempt in 1 2 3 4 5; do | |
| if git push; then | |
| break | |
| elif [[ "${attempt}" == 5 ]]; then | |
| echo "::error::Could not push to gh-pages after ${attempt} attempts." | |
| exit 1 | |
| fi | |
| sleep $((attempt * 10)) | |
| git pull --rebase origin gh-pages | |
| done | |
| notify: | |
| needs: analyze-flaky-tests | |
| runs-on: ubuntu-latest | |
| # Always run so we can close issues when flaky tests are resolved | |
| if: always() && needs.analyze-flaky-tests.result == 'success' | |
| steps: | |
| - name: Download reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: flaky-test-reports | |
| path: flaky-reports/ | |
| - name: Create, update, or close flaky test tracking issue | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const report = fs.readFileSync('flaky-reports/flaky-tests.md', 'utf8'); | |
| const jsonReport = JSON.parse(fs.readFileSync('flaky-reports/flaky-tests.json', 'utf8')); | |
| const flakyCount = jsonReport.summary.flaky_count; | |
| const flakyCaseCount = jsonReport.summary.flaky_case_count || 0; | |
| const newFlakyCount = jsonReport.summary.new_flaky_count || 0; | |
| const resolvedCount = jsonReport.summary.resolved_count || 0; | |
| // Look for existing open tracking issue | |
| const issues = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| labels: 'flaky-test-tracker', | |
| state: 'open', | |
| per_page: 1 | |
| }); | |
| if (flakyCount === 0) { | |
| // No flaky tests: close existing issue if any | |
| if (issues.data.length > 0) { | |
| await github.rest.issues.update({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issues.data[0].number, | |
| state: 'closed', | |
| state_reason: 'completed', | |
| body: `## Flaky Test Report - ${new Date().toISOString().split('T')[0]}\n\nNo tests have recent fail/pass evidence on the same commit and CI environment.\n\n*This issue was automatically closed.*` | |
| }); | |
| console.log(`Closed issue #${issues.data[0].number} - no flaky tests`); | |
| } else { | |
| console.log('No flaky tests and no open tracking issue. Nothing to do.'); | |
| } | |
| return; | |
| } | |
| // There are flaky tests: create or update issue | |
| const familyLabel = flakyCount === 1 ? 'family' : 'families'; | |
| const title = `[Flaky Tests] ${flakyCount} flaky test ${familyLabel} detected`; | |
| const body = [ | |
| `## Flaky Test Report - ${new Date().toISOString().split('T')[0]}`, | |
| '', | |
| '**Summary:**', | |
| `- Confirmed flaky test families: **${flakyCount}**`, | |
| `- Affected parameterized cases: **${flakyCaseCount}**`, | |
| `- Newly confirmed: **${newFlakyCount}**`, | |
| `- Resolved: **${resolvedCount}**`, | |
| '', | |
| '---', | |
| '', | |
| report, | |
| '', | |
| '---', | |
| '', | |
| '*This issue is automatically updated daily. See the [full dashboard](https://pytorch.github.io/tensordict/flaky/) for more details.*', | |
| ].join('\n'); | |
| if (issues.data.length > 0) { | |
| await github.rest.issues.update({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issues.data[0].number, | |
| title: title, | |
| body: body | |
| }); | |
| console.log(`Updated issue #${issues.data[0].number}`); | |
| } else { | |
| const newIssue = await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: title, | |
| body: body, | |
| labels: ['flaky-test-tracker', 'CI'] | |
| }); | |
| console.log(`Created issue #${newIssue.data.number}`); | |
| } |