nightly-build #129
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: nightly-build | |
| on: | |
| schedule: | |
| - cron: '35 8 * * *' # 8:35am UTC, 1:35am PST, 4:35am EST | |
| workflow_dispatch: | |
| inputs: | |
| skip_buildkite: | |
| description: 'Skip Buildkite tests (smoke, backward-compat, etc.)' | |
| required: false | |
| default: false | |
| type: boolean | |
| skip_test_pypi: | |
| description: 'Skip publishing to Test PyPI' | |
| required: false | |
| default: false | |
| type: boolean | |
| jobs: | |
| check-date: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_run: ${{ steps.compute_should_run.outputs.should_run }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: print latest_commit | |
| run: echo ${{ github.sha }} | |
| - id: compute_should_run | |
| name: determine whether nightly should run | |
| shell: bash | |
| run: | | |
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| if git rev-list --after="24 hours" ${{ github.sha }} | grep -q .; then | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "should_run=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| # For manual dispatches, always run | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| gate-tests: | |
| runs-on: ubuntu-latest | |
| needs: [check-date] | |
| outputs: | |
| run_tests: ${{ steps.compute_gate.outputs.run_tests }} | |
| publish_without_tests: ${{ steps.compute_gate.outputs.publish_without_tests }} | |
| steps: | |
| - id: compute_gate | |
| name: compute run_tests gate for smoke tests | |
| shell: bash | |
| env: | |
| EVENT: ${{ github.event_name }} | |
| SKIP_BUILDKITE: ${{ inputs.skip_buildkite }} | |
| SHOULD_RUN: ${{ needs.check-date.outputs.should_run }} | |
| run: | | |
| if [[ "$EVENT" == "workflow_dispatch" ]]; then | |
| if [[ "$SKIP_BUILDKITE" == "true" ]]; then | |
| echo "run_tests=false" >> "$GITHUB_OUTPUT" | |
| echo "publish_without_tests=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | |
| echo "publish_without_tests=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| # For scheduled runs, rely on check-date | |
| if [[ "$SHOULD_RUN" == "true" ]]; then | |
| echo "run_tests=true" >> "$GITHUB_OUTPUT" | |
| echo "publish_without_tests=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_tests=false" >> "$GITHUB_OUTPUT" | |
| echo "publish_without_tests=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| nightly-build-pypi: | |
| runs-on: ubuntu-latest | |
| needs: check-date | |
| if: ${{ needs.check-date.outputs.should_run != 'false' }} | |
| outputs: | |
| version: ${{ steps.set-release-version.outputs.version }} | |
| steps: | |
| - name: Clone repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '23' | |
| cache: 'npm' | |
| cache-dependency-path: 'sky/dashboard/package-lock.json' | |
| - name: Install dashboard dependencies | |
| run: | | |
| cd sky/dashboard | |
| npm ci | |
| - name: Build dashboard | |
| run: | | |
| cd sky/dashboard | |
| npm run build | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' # caching pip dependencies | |
| - name: Install pypa/build | |
| run: >- | |
| python -m | |
| pip install | |
| build | |
| --user | |
| - name: Set release version | |
| id: set-release-version | |
| run: | | |
| RELEASE_VERSION=$(date +%Y%m%d) | |
| sed -i "s/{{SKYPILOT_COMMIT_SHA}}/${{ github.sha }}/g" sky/__init__.py | |
| sed -i "s/__version__ = '.*'/__version__ = '1.0.0.dev${RELEASE_VERSION}'/g" sky/__init__.py | |
| sed -i "s/name='skypilot',/name='skypilot-nightly',/g" sky/setup_files/setup.py | |
| echo "version=1.0.0.dev${RELEASE_VERSION}" >> $GITHUB_OUTPUT | |
| echo "Version: 1.0.0.dev${RELEASE_VERSION}" | |
| - name: Build a binary wheel and a source tarball | |
| run: >- | |
| python -m | |
| build | |
| --sdist | |
| --wheel | |
| --outdir dist/ | |
| . | |
| - name: Upload distribution artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: skypilot-artifacts-skypilot-nightly | |
| path: dist/ | |
| smoke-tests-aws: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi --aws" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "--aws"}' | |
| timeout_minutes: 60 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| backward-compat-test-nightly: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi backward compatibility test --base-branch pypi/skypilot-nightly" | |
| pipeline: "quicktest-core" | |
| build_env_vars: '{"ARGS": "--base-branch pypi/skypilot-nightly"}' | |
| timeout_minutes: 60 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| backward-compat-test-stable: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi backward compatibility test --base-branch pypi/skypilot" | |
| pipeline: "quicktest-core" | |
| build_env_vars: '{"ARGS": "--base-branch pypi/skypilot"}' | |
| timeout_minutes: 60 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| smoke-tests-kubernetes-resource-heavy: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi --kubernetes --resource-heavy" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "--kubernetes --resource-heavy"}' | |
| timeout_minutes: 100 | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| smoke-tests-kubernetes-no-resource-heavy: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi --kubernetes --no-resource-heavy" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "--kubernetes --no-resource-heavy"}' | |
| timeout_minutes: 60 | |
| sleep_seconds: 1200 # 20-minute delay to reduce buildkite resource usage | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| smoke-tests-remote-server-kubernetes: | |
| needs: [gate-tests, nightly-build-pypi] | |
| if: ${{ needs.gate-tests.outputs.run_tests == 'true' }} | |
| uses: ./.github/workflows/buildkite-trigger-wait.yml | |
| with: | |
| commit: ${{ github.sha }} | |
| branch: ${{ github.ref_name }} | |
| message: "nightly-build-pypi --remote-server --kubernetes" | |
| pipeline: "smoke-tests" | |
| build_env_vars: '{"ARGS": "--remote-server --kubernetes"}' | |
| timeout_minutes: 60 | |
| sleep_seconds: 1500 # 25-minute delay to reduce buildkite resource usage | |
| secrets: | |
| BUILDKITE_TOKEN: ${{ secrets.BUILDKITE_TOKEN }} | |
| publish-and-validate-both: | |
| needs: [gate-tests, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-remote-server-kubernetes, backward-compat-test-nightly, backward-compat-test-stable] | |
| if: ${{ always() && needs.nightly-build-pypi.result == 'success' && (needs.gate-tests.outputs.publish_without_tests == 'true' || (needs.gate-tests.outputs.run_tests == 'true' && needs.smoke-tests-aws.result == 'success' && needs.smoke-tests-kubernetes-resource-heavy.result == 'success' && needs.smoke-tests-kubernetes-no-resource-heavy.result == 'success' && needs.smoke-tests-remote-server-kubernetes.result == 'success' && needs.backward-compat-test-nightly.result == 'success' && needs.backward-compat-test-stable.result == 'success')) }} | |
| uses: ./.github/workflows/publish-and-validate-both.yml | |
| with: | |
| package_name: skypilot-nightly | |
| expected_version: ${{ needs.nightly-build-pypi.outputs.version }} | |
| skip_test_pypi: ${{ github.event_name == 'workflow_dispatch' && inputs.skip_test_pypi || false }} | |
| secrets: inherit | |
| trigger-helm-release: | |
| needs: [publish-and-validate-both] | |
| if: ${{ always() && needs.publish-and-validate-both.result == 'success' }} | |
| uses: ./.github/workflows/helm-docker-release.yaml | |
| with: | |
| package_name: skypilot-nightly | |
| secrets: inherit | |
| notify-slack-failure: | |
| runs-on: ubuntu-latest | |
| needs: [check-date, nightly-build-pypi, smoke-tests-aws, smoke-tests-kubernetes-resource-heavy, smoke-tests-kubernetes-no-resource-heavy, smoke-tests-remote-server-kubernetes, backward-compat-test-nightly, backward-compat-test-stable, publish-and-validate-both, trigger-helm-release] | |
| if: failure() # Only run this job if any of the previous jobs failed | |
| steps: | |
| - name: Prepare failure message | |
| id: message_content | |
| run: | | |
| TITLE_TEXT="Workflow ${{ github.workflow }} failed." | |
| COMMIT_SHA="${{ github.sha }}" | |
| COMMIT_URL="${{ github.server_url }}/${{ github.repository }}/commit/${COMMIT_SHA}" | |
| SHORT_SHA=$(echo "$COMMIT_SHA" | cut -c1-7) | |
| BUILDKITE_MSG="" | |
| if [[ "${{ needs.smoke-tests-aws.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-resource-heavy.result }}" == "failure" || "${{ needs.smoke-tests-kubernetes-no-resource-heavy.result }}" == "failure" || "${{ needs.smoke-tests-remote-server-kubernetes.result }}" == "failure" || "${{ needs.backward-compat-test-nightly.result }}" == "failure" || "${{ needs.backward-compat-test-stable.result }}" == "failure" ]]; then | |
| if [[ "${{ needs.smoke-tests-aws.result }}" == "failure" ]]; then | |
| BUILDKITE_MSG="<https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-aws.outputs.build_number }}|Buildkite Log(--aws)>" | |
| fi | |
| if [[ "${{ needs.smoke-tests-kubernetes-resource-heavy.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-resource-heavy.outputs.build_number }}|Buildkite Log(--kubernetes --resource-heavy)>" | |
| fi | |
| if [[ "${{ needs.smoke-tests-kubernetes-no-resource-heavy.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-kubernetes-no-resource-heavy.outputs.build_number }}|Buildkite Log(--kubernetes --no-resource-heavy)>" | |
| fi | |
| if [[ "${{ needs.smoke-tests-remote-server-kubernetes.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/smoke-tests/builds/${{ needs.smoke-tests-remote-server-kubernetes.outputs.build_number }}|Buildkite Log(--remote-server --kubernetes)>" | |
| fi | |
| if [[ "${{ needs.backward-compat-test-nightly.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-nightly.outputs.build_number }}|Buildkite Log(backward-compat-nightly)>" | |
| fi | |
| if [[ "${{ needs.backward-compat-test-stable.result }}" == "failure" ]]; then | |
| if [[ ! -z "$BUILDKITE_MSG" ]]; then | |
| BUILDKITE_MSG="${BUILDKITE_MSG} and" | |
| fi | |
| BUILDKITE_MSG="${BUILDKITE_MSG} <https://buildkite.com/skypilot-1/quicktest-core/builds/${{ needs.backward-compat-test-stable.outputs.build_number }}|Buildkite Log(backward-compat-stable)>" | |
| fi | |
| MARKDOWN_TEXT="🚨 Workflow <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|*${{ github.workflow }}*> failed at Buildkite step for commit <${COMMIT_URL}|${SHORT_SHA}>. ${BUILDKITE_MSG}" | |
| else | |
| MARKDOWN_TEXT="🚨 Workflow <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|*${{ github.workflow }}*> failed for commit <${COMMIT_URL}|${SHORT_SHA}>." | |
| fi | |
| echo "message_text=${TITLE_TEXT}" >> $GITHUB_OUTPUT | |
| echo "message_block=${MARKDOWN_TEXT}" >> $GITHUB_OUTPUT | |
| - name: Notify Slack channel on failure | |
| uses: slackapi/slack-github-action@v2.0.0 | |
| with: | |
| method: chat.postMessage | |
| token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| payload: | | |
| { | |
| "channel": "${{ secrets.SLACK_CHANNEL_ID }}", | |
| "text": "${{ steps.message_content.outputs.message_text }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "${{ steps.message_content.outputs.message_block }}" | |
| } | |
| } | |
| ] | |
| } |