Outerloop Tests #2330
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Executes outerloop tests | |
| # | |
| # COPILOT INSTRUCTIONS: | |
| # - Keep the shared 'paths:' entries (specialized-test-runner.yml, | |
| # run-tests.yml, build-cli-e2e-image.yml) in sync across | |
| # tests-outerloop.yml and tests-quarantine.yml. Each workflow also lists itself. | |
| # - Validate that each path exists in the repository before adding or | |
| # updating the list | |
| # - No external YAML file is used—only the workflow YAMLs themselves | |
| # hold the list | |
| # | |
| name: Outerloop Tests | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 2 * * *' # Daily at 02:00 UTC | |
| # Re-enabled with narrow paths filter scoped to key workflow files that | |
| # orchestrate this pipeline. Changes to downstream workflows (e.g. | |
| # build-packages.yml) are validated by the regular CI and don't need to | |
| # re-trigger the full quarantine/outerloop run. | |
| # Previously disabled (#12143) due to broad paths filter causing disk | |
| # space issues on every CI/eng change. | |
| pull_request: | |
| paths: | |
| - '.github/workflows/tests-outerloop.yml' | |
| - '.github/workflows/specialized-test-runner.yml' | |
| - '.github/workflows/run-tests.yml' | |
| - '.github/workflows/build-cli-e2e-image.yml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| outerloop_tests: | |
| uses: ./.github/workflows/specialized-test-runner.yml | |
| with: | |
| testRunnerName: "OuterloopTestRunsheetBuilder" | |
| attributeName: "OuterloopTest" | |
| extraRunSheetBuilderArgs: "-p:RunOuterloopTests=true" | |
| extraTestArgs: "--filter-trait outerloop=true" | |
| enablePlaywrightInstall: true | |
| # On a scheduled failure, file/append a single deduplicated issue: a | |
| # 'failing-test' issue listing the failed outerloop tests, or an | |
| # 'automation-broken' infra issue when the run broke before producing test | |
| # results. PR-triggered runs (paths filter) are excluded so workflow edits | |
| # don't file issues. See docs/ci/specialized-test-failure-issues.md. | |
| report_failures: | |
| name: Report failures | |
| needs: [outerloop_tests] | |
| if: ${{ failure() && github.event_name == 'schedule' && github.repository_owner == 'microsoft' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # checkout to require the local .js modules | |
| actions: read | |
| issues: write | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: main | |
| persist-credentials: false | |
| # The runner uploads per-job test logs (incl. .trx) as 'logs-*' artifacts. | |
| - name: Download test result artifacts | |
| id: download | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| continue-on-error: true | |
| with: | |
| pattern: 'logs-*' | |
| path: ${{ github.workspace }}/all-logs | |
| - name: Extract failed test names | |
| id: failed | |
| continue-on-error: true | |
| env: | |
| DOWNLOAD_OUTCOME: ${{ steps.download.outcome }} | |
| run: | | |
| mkdir -p "$RUNNER_TEMP/out" | |
| OUT="$RUNNER_TEMP/out/failed-tests.json" | |
| # extractionFailed distinguishes "we could not read the results" from | |
| # "the run produced zero failed tests". The reporter classifies a | |
| # genuinely-red run with unknown results as a test failure (not infra), | |
| # so a transient download/tool flake doesn't silently misfile the issue | |
| # or drop the failing-test list. | |
| if [ -d "$GITHUB_WORKSPACE/all-logs" ]; then | |
| if ./dotnet.sh run --project tools/GenerateTestSummary/GenerateTestSummary.csproj -- \ | |
| "$GITHUB_WORKSPACE/all-logs" --failed-tests-json "$OUT"; then | |
| : # tool wrote $OUT (with or without failures) | |
| else | |
| echo '{"failedTests":[],"count":0,"extractionFailed":true}' > "$OUT" | |
| fi | |
| elif [ "$DOWNLOAD_OUTCOME" = "failure" ]; then | |
| # No artifacts directory AND the download step errored — a download | |
| # flake, not a clean "no results". A real build break still uploads | |
| # logs-runsheet, so a present-but-empty result set stays infra below. | |
| echo '{"failedTests":[],"count":0,"extractionFailed":true}' > "$OUT" | |
| else | |
| echo '{"failedTests":[],"count":0}' > "$OUT" | |
| fi | |
| echo "path=$OUT" >> "$GITHUB_OUTPUT" | |
| - name: File or update failure issue | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| WORKFLOW_FILE: tests-outerloop.yml | |
| DISPLAY_NAME: Outerloop Tests | |
| IGNORE_TEST_FAILURES: 'false' | |
| FAILED_TESTS_PATH: ${{ steps.failed.outputs.path }} | |
| with: | |
| script: | | |
| const reporter = require('./.github/workflows/report-specialized-test-failures.js'); | |
| await require('./.github/workflows/specialized-test-failure-runner.js')({ github, context, core, reporter }); | |