nightly-build #301
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: nightly-build | |
| on: | |
| schedule: | |
| - cron: '35 8 * * *' # 08:35 UTC (01:35 PST / 04:35 EST) — original nightly | |
| - cron: '0 0 * * *' # 00:00 UTC (~5:00 PM PT during DST) — tests-only preflight | |
| workflow_dispatch: | |
| inputs: | |
| skip_buildkite: | |
| description: 'Skip Buildkite tests (smoke, backward-compat, etc.)' | |
| required: false | |
| default: false | |
| type: boolean | |
| skip_test_pypi: | |
| description: 'Skip publishing to Test PyPI' | |
| required: false | |
| default: false | |
| type: boolean | |
| jobs: | |
| check-date: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_run: ${{ steps.compute_should_run.outputs.should_run }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: print latest_commit | |
| run: echo ${{ github.sha }} | |
| - id: compute_should_run | |
| name: determine whether nightly should run | |
| shell: bash | |
| run: | | |
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| if git rev-list --after="24 hours" ${{ github.sha }} | grep -q .; then | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "should_run=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| # For manual dispatches, always run | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| gate-tests: | |
| runs-on: ubuntu-latest | |
| needs: [check-date] | |
| outputs: | |
| run_tests: ${{ steps.compute_gate.outputs.run_tests }} | |
| publish_without_tests: ${{ steps.compute_gate.outputs.publish_without_tests }} | |
| steps: | |
| - id: compute_gate | |
| name: compute run_tests gate for smoke tests | |
| shell: bash | |
| env: | |
| EVENT: ${{ github.event_name }} | |
| SKIP_BUILDKITE: ${{ inputs.skip_buildkite }} | |
| SHOULD_RUN: ${{ needs.check-date.outputs.should_run }} | |
| run: | | |
| if [[ "$EVENT" == "workflow_dispatch" ]]; then | |
| if [[ "$SKIP_BUILDKITE" == "true" ]]; then | |
| echo "run_tests=false" >> "$GITHUB_OUTPUT" | |
| echo "publish_without_tests=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | |
| echo "publish_without_tests=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| # For scheduled runs, rely on check-date | |
| if [[ "$SHOULD_RUN" == "true" ]]; then | |
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | |
| echo "publish_without_tests=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_tests=false" >> "$GITHUB_OUTPUT" | |
| echo "publish_without_tests=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| nightly-build-pypi: | |
| runs-on: ubuntu-latest | |
| needs: check-date | |
| if: ${{ needs.check-date.outputs.should_run != 'false' }} | |
| outputs: | |
| version: ${{ steps.set-release-version.outputs.version }} | |
| steps: | |
| - name: Clone repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '23' | |
| cache: 'npm' | |
| cache-dependency-path: 'sky/dashboard/package-lock.json' | |
| - name: Install dashboard dependencies | |
| run: | | |
| cd sky/dashboard | |
| npm ci | |
| - name: Build dashboard | |
| run: | | |
| cd sky/dashboard | |
| npm run build | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' # caching pip dependencies | |
| - name: Install pypa/build | |
| run: >- | |
| python -m | |
| pip install | |
| build | |
| --user | |
| - name: Set release version | |
| id: set-release-version | |
| run: | | |
| RELEASE_VERSION=$(date +%Y%m%d) | |
| sed -i "s/{{SKYPILOT_COMMIT_SHA}}/${{ github.sha }}/g" sky/__init__.py | |
| sed -i "s/__version__ = '.*'/__version__ = '1.0.0.dev${RELEASE_VERSION}'/g" sky/__init__.py | |
| sed -i "s/name='skypilot',/name='skypilot-nightly',/g" sky/setup_files/setup.py | |
| echo "version=1.0.0.dev${RELEASE_VERSION}" >> $GITHUB_OUTPUT | |
| echo "Version: 1.0.0.dev${RELEASE_VERSION}" | |
| - name: Build a binary wheel and a source tarball | |
| run: >- | |
| python -m | |
| build | |
| --sdist | |
| --wheel | |
| --outdir dist/ | |
| . | |
| - name: Upload distribution artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: skypilot-artifacts-skypilot-nightly | |
| path: dist/ | |
| smoke-tests-aws: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi --aws" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "--aws"}' | |
| timeout_minutes: 60 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| backward-compat-test-nightly: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi backward compatibility test --base-branch pypi/skypilot-nightly" | |
| pipeline: "quicktest-core" | |
| build_env_vars: '{"ARGS": "--base-branch pypi/skypilot-nightly"}' | |
| timeout_minutes: 60 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| backward-compat-test-stable: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi backward compatibility test --base-branch pypi/skypilot" | |
| pipeline: "quicktest-core" | |
| build_env_vars: '{"ARGS": "--base-branch pypi/skypilot"}' | |
| timeout_minutes: 60 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| smoke-tests-kubernetes-resource-heavy: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi --kubernetes --resource-heavy" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "--kubernetes --resource-heavy"}' | |
| timeout_minutes: 100 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| smoke-tests-kubernetes-no-resource-heavy: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi --kubernetes --no-resource-heavy" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "--kubernetes --no-resource-heavy"}' | |
| timeout_minutes: 80 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| smoke-tests-kubernetes-no-resource-heavy-limit-deps: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi --kubernetes --no-resource-heavy --dependency" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "--kubernetes --no-resource-heavy --dependency"}' | |
| timeout_minutes: 80 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| smoke-tests-remote-server-kubernetes: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi --remote-server --kubernetes" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "--remote-server --kubernetes"}' | |
| timeout_minutes: 80 | |
| sleep_seconds: 600 # 10 minute delay to reduce buildkite resource usage | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| smoke-tests-shared-gke-api-server: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi shared-gke-api-server with postgres tests" | |
| pipeline: "nightly-build-shared-gke-api-server" | |
| timeout_minutes: 120 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| smoke-tests-lambda-job-queue: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi test_lambda_job_queue --lambda" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "-k test_lambda_job_queue --lambda", "FILE_PATTERN": "test_cluster_job.py"}' | |
| timeout_minutes: 60 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| # Disable runpod until #8293 is fixed. | |
| # smoke-tests-runpod-minimal: | |
| # needs: [gate-tests, nightly-build-pypi] | |
| # if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| # uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| # with: | |
| # commit: ${{ github.sha }} | |
| # branch: ${{ github.ref_name }} | |
| # message: "nightly-build-pypi test_minimal --runpod" | |
| # pipeline: "smoke-tests" | |
| # build_env_vars: '{"ARGS": "-k test_minimal --runpod", "FILE_PATTERN": "test_basic.py"}' | |
| # timeout_minutes: 60 | |
| # secrets: | |
| # BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| publish-and-validate-both: | |
| # needs: [gate-tests, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-kubernetes-no-resource-heavy-limit-deps, smoke-tests-remote-server-kubernetes, smoke-tests-shared-gke-api-server, smoke-tests-lambda-job-queue, smoke-tests-runpod-minimal, backward-compat-test-nightly, backward-compat-test-stable] | |
| needs: [gate-tests, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-kubernetes-no-resource-heavy-limit-deps, smoke-tests-remote-server-kubernetes, smoke-tests-shared-gke-api-server, smoke-tests-lambda-job-queue, backward-compat-test-nightly, backward-compat-test-stable] | |
| # Allow publish/validate for manual dispatch or the original nightly cron; skip for the 5PM PT preflight | |
| # Use always() so this job evaluates even if some test jobs were skipped when skip_buildkite is selected | |
| # if: ${{ always() && (github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.event.schedule == '35 8 * * *')) && needs.nightly-build-pypi.result == 'success' && (needs.gate-tests.outputs.publish_without_tests == 'true' || (needs.gate-tests.outputs.run_tests == 'true' && needs.smoke-tests-aws.result == 'success' && needs.smoke-tests-kubernetes-resource-heavy.result == 'success' && needs.smoke-tests-kubernetes-no-resource-heavy.result == 'success' && needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result == 'success' && needs.smoke-tests-remote-server-kubernetes.result == 'success' && needs.smoke-tests-shared-gke-api-server.result == 'success' && needs.smoke-tests-lambda-job-queue.result == 'success' && needs.smoke-tests-runpod-minimal.result == 'success' && needs.backward-compat-test-nightly.result == 'success' && needs.backward-compat-test-stable.result == 'success')) }} | |
| if: ${{ always() && (github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.event.schedule == '35 8 * * *')) && needs.nightly-build-pypi.result == 'success' && (needs.gate-tests.outputs.publish_without_tests == 'true' || (needs.gate-tests.outputs.run_tests == 'true' && needs.smoke-tests-aws.result == 'success' && needs.smoke-tests-kubernetes-resource-heavy.result == 'success' && needs.smoke-tests-kubernetes-no-resource-heavy.result == 'success' && needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result == 'success' && needs.smoke-tests-remote-server-kubernetes.result == 'success' && needs.smoke-tests-shared-gke-api-server.result == 'success' && needs.smoke-tests-lambda-job-queue.result == 'success' && needs.backward-compat-test-nightly.result == 'success' && needs.backward-compat-test-stable.result == 'success')) }} | |
| uses: ./.github/workflows/publish-and-validate-both.yml | |
| with: | |
| package_name: skypilot-nightly | |
| expected_version: ${{ needs.nightly-build-pypi.outputs.version }} | |
| skip_test_pypi: ${{ github.event_name == 'workflow_dispatch' && inputs.skip_test_pypi || false }} | |
| secrets: inherit | |
| trigger-helm-release: | |
| needs: [publish-and-validate-both] | |
| # Allow helm release for manual dispatch or the original nightly cron; skip for the 5PM PT preflight | |
| if: ${{ always() && (github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.event.schedule == '35 8 * * *')) && needs.publish-and-validate-both.result == 'success' }} | |
| uses: ./.github/workflows/helm-docker-release.yaml | |
| with: | |
| package_name: skypilot-nightly | |
| secrets: inherit | |
| summary: | |
| runs-on: ubuntu-latest | |
| needs: [check-date, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-kubernetes-no-resource-heavy-limit-deps, smoke-tests-remote-server-kubernetes, smoke-tests-shared-gke-api-server, smoke-tests-lambda-job-queue, backward-compat-test-nightly, backward-compat-test-stable] | |
| # needs: [check-date, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-kubernetes-no-resource-heavy-limit-deps, smoke-tests-remote-server-kubernetes, smoke-tests-shared-gke-api-server, smoke-tests-lambda-job-queue, smoke-tests-runpod-minimal, backward-compat-test-nightly, backward-compat-test-stable] | |
| if: always() | |
| steps: | |
| - name: Summary | |
| run: | | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| # Nightly Build Summary | |
| ## Buildkite Test Links | |
| EOF | |
| if [ "${{ needs.smoke-tests-aws.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-aws.outputs.build_number }}" ]; then | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| - [Smoke Tests AWS](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-aws.outputs.build_number }}) - $([ "${{ needs.smoke-tests-aws.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| EOF | |
| fi | |
| if [ "${{ needs.smoke-tests-kubernetes-resource-heavy.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-kubernetes-resource-heavy.outputs.build_number }}" ]; then | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| - [Smoke Tests Kubernetes (Resource Heavy)](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-resource-heavy.outputs.build_number }}) - $([ "${{ needs.smoke-tests-kubernetes-resource-heavy.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| EOF | |
| fi | |
| if [ "${{ needs.smoke-tests-kubernetes-no-resource-heavy.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-kubernetes-no-resource-heavy.outputs.build_number }}" ]; then | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| - [Smoke Tests Kubernetes (No Resource Heavy)](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-no-resource-heavy.outputs.build_number }}) - $([ "${{ needs.smoke-tests-kubernetes-no-resource-heavy.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| EOF | |
| fi | |
| if [ "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.outputs.build_number }}" ]; then | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| - [Smoke Tests Kubernetes (No Resource Heavy, Limit Deps)](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.outputs.build_number }}) - $([ "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| EOF | |
| fi | |
| if [ "${{ needs.smoke-tests-remote-server-kubernetes.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-remote-server-kubernetes.outputs.build_number }}" ]; then | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| - [Smoke Tests Remote Server Kubernetes](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-remote-server-kubernetes.outputs.build_number }}) - $([ "${{ needs.smoke-tests-remote-server-kubernetes.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| EOF | |
| fi | |
| if [ "${{ needs.smoke-tests-shared-gke-api-server.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-shared-gke-api-server.outputs.build_number }}" ]; then | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| - [Smoke Tests Shared GKE API Server](https://buildkite.com/skypilot-1/nightly-build-shared-gke-api-server/builds/${{ needs.smoke-tests-shared-gke-api-server.outputs.build_number }}) - $([ "${{ needs.smoke-tests-shared-gke-api-server.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| EOF | |
| fi | |
| if [ "${{ needs.smoke-tests-lambda-job-queue.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-lambda-job-queue.outputs.build_number }}" ]; then | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| - [Smoke Tests Lambda Job Queue](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-lambda-job-queue.outputs.build_number }}) - $([ "${{ needs.smoke-tests-lambda-job-queue.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| EOF | |
| fi | |
| # if [ "${{ needs.smoke-tests-runpod-minimal.result }}" != "skipped" ] && [ -n "${{ needs.smoke-tests-runpod-minimal.outputs.build_number }}" ]; then | |
| # cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| # - [Smoke Tests RunPod Minimal](https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-runpod-minimal.outputs.build_number }}) - $([ "${{ needs.smoke-tests-runpod-minimal.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| # EOF | |
| # fi | |
| if [ "${{ needs.backward-compat-test-nightly.result }}" != "skipped" ] && [ -n "${{ needs.backward-compat-test-nightly.outputs.build_number }}" ]; then | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| - [Backward Compat Test (Nightly)](https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-nightly.outputs.build_number }}) - $([ "${{ needs.backward-compat-test-nightly.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| EOF | |
| fi | |
| if [ "${{ needs.backward-compat-test-stable.result }}" != "skipped" ] && [ -n "${{ needs.backward-compat-test-stable.outputs.build_number }}" ]; then | |
| cat <<EOF >> "$GITHUB_STEP_SUMMARY" | |
| - [Backward Compat Test (Stable)](https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-stable.outputs.build_number }}) - $([ "${{ needs.backward-compat-test-stable.outputs.build_status }}" == "success" ] && echo "✅ Success" || echo "❌ Failed") | |
| EOF | |
| fi | |
| notify-slack-failure: | |
| runs-on: ubuntu-latest | |
| needs: [check-date, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-kubernetes-no-resource-heavy-limit-deps, smoke-tests-remote-server-kubernetes, smoke-tests-shared-gke-api-server, smoke-tests-lambda-job-queue, backward-compat-test-nightly, backward-compat-test-stable, publish-and-validate-both, trigger-helm-release] | |
| # needs: [check-date, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-kubernetes-no-resource-heavy-limit-deps, smoke-tests-remote-server-kubernetes, smoke-tests-shared-gke-api-server, smoke-tests-lambda-job-queue, smoke-tests-runpod-minimal, backward-compat-test-nightly, backward-compat-test-stable, publish-and-validate-both, trigger-helm-release] | |
| # Only run this job if any of the previous jobs failed | |
| if: failure() | |
| steps: | |
| - name: Prepare failure message | |
| id: message_content | |
| run: | | |
| TITLE_TEXT="Workflow ${{ github.workflow }} failed." | |
| COMMIT_SHA="${{ github.sha }}" | |
| COMMIT_URL="${{ github.server_url }}/${{ github.repository }}/commit/${COMMIT_SHA}" | |
| SHORT_SHA=$(echo "$COMMIT_SHA" | cut -c1-7) | |
| BUILDKITE_MSG="" | |
| if [[ "${{ needs.smoke-tests-aws.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-resource-heavy.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-no-resource-heavy.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result }}" == "failure" || "${{ needs.smoke-tests-remote-server-kubernetes.result }}" == "failure" || "${{ needs.smoke-tests-shared-gke-api-server.result }}" == "failure" || "${{ needs.smoke-tests-lambda-job-queue.result }}" == "failure" || "${{ needs.backward-compat-test-nightly.result }}" == "failure" || "${{ needs.backward-compat-test-stable.result }}" == "failure" ]]; then | |
| # if [[ "${{ needs.smoke-tests-aws.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-resource-heavy.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-no-resource-heavy.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result }}" == "failure" || "${{ needs.smoke-tests-remote-server-kubernetes.result }}" == "failure" || "${{ needs.smoke-tests-shared-gke-api-server.result }}" == "failure" || "${{ needs.smoke-tests-lambda-job-queue.result }}" == "failure" || "${{ needs.smoke-tests-runpod-minimal.result }}" == "failure" || "${{ needs.backward-compat-test-nightly.result }}" == "failure" || "${{ needs.backward-compat-test-stable.result }}" == "failure" ]]; then | |
| if [[ "${{ needs.smoke-tests-aws.result }}" == "failure" ]]; then | |
| BUILDKITE_MSG="<https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-aws.outputs.build_number }}|Buildkite Log(--aws)>" | |
| fi | |
| if [[ "${{ needs.smoke-tests-kubernetes-resource-heavy.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-resource-heavy.outputs.build_number }}|Buildkite Log(--kubernetes --resource-heavy)>" | |
| fi | |
| if [[ "${{ needs.smoke-tests-kubernetes-no-resource-heavy.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-no-resource-heavy.outputs.build_number }}|Buildkite Log(--kubernetes --no-resource-heavy)>" | |
| fi | |
| if [[ "${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-no-resource-heavy-limit-deps.outputs.build_number }}|Buildkite Log(--kubernetes --no-resource-heavy --dependency kubernetes)>" | |
| fi | |
| if [[ "${{ needs.smoke-tests-remote-server-kubernetes.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-remote-server-kubernetes.outputs.build_number }}|Buildkite Log(--remote-server --kubernetes)>" | |
| fi | |
| if [[ "${{ needs.smoke-tests-shared-gke-api-server.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/nightly-build-shared-gke-api-server/builds/${{ needs.smoke-tests-shared-gke-api-server.outputs.build_number }}|Buildkite Log(shared-gke-api-server)>" | |
| fi | |
| if [[ "${{ needs.smoke-tests-lambda-job-queue.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-lambda-job-queue.outputs.build_number }}|Buildkite Log(test_lambda_job_queue --lambda)>" | |
| fi | |
| # if [[ "${{ needs.smoke-tests-runpod-minimal.result }}" == "failure" ]]; then | |
| # if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| # BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| # fi | |
| # BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-runpod-minimal.outputs.build_number }}|Buildkite Log(test_minimal --runpod)>" | |
| # fi | |
| if [[ "${{ needs.backward-compat-test-nightly.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-nightly.outputs.build_number }}|Buildkite Log(backward-compat-nightly)>" | |
| fi | |
| if [[ "${{ needs.backward-compat-test-stable.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-stable.outputs.build_number }}|Buildkite Log(backward-compat-stable)>" | |
| fi | |
| MARKDOWN_TEXT="🚨 Workflow <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|*${{ github.workflow }}*> failed at Buildkite step for commit <${COMMIT_URL}|${SHORT_SHA}>. ${BUILDKITE_MSG}" | |
| else | |
| MARKDOWN_TEXT="🚨 Workflow <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|*${{ github.workflow }}*> failed for commit <${COMMIT_URL}|${SHORT_SHA}>." | |
| fi | |
| echo "message_text=${TITLE_TEXT}" >> $GITHUB_OUTPUT | |
| echo "message_block=${MARKDOWN_TEXT}" >> $GITHUB_OUTPUT | |
| - name: Notify Slack channel on failure | |
| uses: slackapi/[email protected] | |
| with: | |
| method: chat.postMessage | |
| token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| payload: | | |
| { | |
| "channel": "${{ secrets.SLACK_CHANNEL_ID }}", | |
| "text": "${{ steps.message_content.outputs.message_text }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "${{ steps.message_content.outputs.message_block }}" | |
| } | |
| } | |
| ] | |
| } |