[DX-4553] Optimize Integration Tests Workflow #21173
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CRE Local Environment Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| chainlink_image_tag: | |
| required: true | |
| type: string | |
| description: "The tag of the Chainlink image to use for the tests." | |
| chainlink_version: | |
| required: false # defaulted | |
| type: string | |
| description: "The version of Chainlink repository to use for the tests." | |
| default: "develop" | |
| workflow_call: | |
| inputs: | |
| chainlink_image_tag: | |
| required: true | |
| type: string | |
| description: "The tag of the Chainlink image to use for the tests." | |
| chainlink_version: | |
| required: false # defaulted | |
| type: string | |
| description: "The version of Chainlink repository to use for the tests." | |
| default: "develop" | |
| pull_request: | |
| types: [labeled, synchronize] | |
| jobs: | |
| changes: | |
| runs-on: runs-on=${{ github.run_id }}/cpu=16/ram=64/family=m6i/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs | |
| environment: | |
| # http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments | |
| name: integration | |
| deployment: false | |
| permissions: | |
| contents: read | |
| outputs: | |
| should-run-tests: >- | |
| ${{ | |
| github.event_name == 'workflow_dispatch' || | |
| github.event_name == 'workflow_call' || | |
| github.event_name == 'schedule' || | |
| github.event_name == 'pull_request' && contains(join(github.event.pull_request.labels.*.name, ' '), 'run-local-cre-env-tests') || | |
| steps.changes.outputs.ci_changes == 'true' | |
| }} | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.sha || inputs.chainlink_version }} | |
| - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: changes | |
| with: | |
| filters: | | |
| ci_changes: | |
| - '.github/workflows/cre-local-env-tests.yaml' | |
| test-cli: | |
| runs-on: runs-on=${{ github.run_id }}/cpu=16/ram=64/family=m6i/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs | |
| environment: | |
| # http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments | |
| name: integration | |
| deployment: false | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| needs: changes | |
| if: needs.changes.outputs.should-run-tests == 'true' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.sha || inputs.chainlink_version }} | |
| - name: Enable S3 Cache for Self-Hosted Runners | |
| uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0 | |
| - name: Set up Go | |
| id: setup-go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: core/scripts/go.mod | |
| cache: true | |
| - name: Download Go dependencies | |
| id: go-deps | |
| working-directory: core/scripts/cre/environment | |
| env: | |
| SETUP_GO_CACHE_HIT: ${{ steps.setup-go.outputs.cache-hit }} | |
| run: | | |
| if [[ "${SETUP_GO_CACHE_HIT}" == "true" ]]; then | |
| echo "::debug::Cache hit - skipping dependency download" | |
| echo "deps_download_seconds=0" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "::debug::Cache miss - downloading dependencies" | |
| { time go mod tidy && go mod download; } 2> >(tee /tmp/go_deps_time >&2) | |
| deps_time=$(cat /tmp/go_deps_time | grep real | awk '{print $2}') | |
| minutes=$(echo "$deps_time" | awk -F'm' '{print $1}') | |
| seconds=$(echo "$deps_time" | awk -F'm' '{print $2}' | sed 's/s//') | |
| total_seconds=$(echo "$minutes * 60 + $seconds" | bc) | |
| echo "deps_download_seconds=$total_seconds" >> "${GITHUB_OUTPUT}" | |
| fi | |
| # We need to login to ECR to allow the test to pull the Job Distributor and Chainlink images | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 | |
| with: | |
| aws-region: ${{ secrets.QA_AWS_REGION }} | |
| role-to-assume: ${{ secrets.AWS_CTF_READ_ACCESS_ROLE_ARN }} | |
| role-duration-seconds: 1800 | |
| mask-aws-account-id: true | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6 | |
| with: | |
| registries: ${{ format('{0},{1}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.AWS_ACCOUNT_ID_PROD) }} | |
| env: | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| - name: Set date variable | |
| id: set-date | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: echo "date=$(date +'%Y%m%d')" >> "${GITHUB_OUTPUT}" | |
| - name: Setup GitHub token using GATI | |
| id: github-token | |
| uses: smartcontractkit/.github/actions/setup-github-token@setup-github-token/v1 | |
| with: | |
| aws-role-arn: ${{ secrets.AWS_OIDC_CHAINLINK_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }} | |
| aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }} | |
| aws-region: us-west-2 | |
| aws-role-duration-seconds: "1800" | |
| set-git-config: "true" | |
| - name: Install bun | |
| uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 | |
| with: | |
| bun-version: latest | |
| - name: Setup the environment | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| env: | |
| DISABLE_DX_TRACKING: true | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| MAIN_AWS_ECR: ${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.us-west-2.amazonaws.com | |
| SDLC_AWS_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.us-west-2.amazonaws.com | |
| run: | | |
| # Remove chip_ingress/chip_config sections since CI role lacks ECR permissions for the Atlas repo | |
| awk '/^\[chip_ingress\.build_config\]/,/^$/{next} /^\[chip_ingress\.pull_config\]/,/^$/{next} /^\[chip_config\.build_config\]/,/^$/{next} /^\[chip_config\.pull_config\]/,/^$/{next} {print}' configs/setup.toml > configs/setup.toml.tmp && mv configs/setup.toml.tmp configs/setup.toml | |
| go run . env setup --no-prompt | |
| - name: Start the environment | |
| id: start-cli | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| env: | |
| CTF_CONFIGS: "./configs/workflow-gateway-don.toml" | |
| CTF_JD_IMAGE: "${{ secrets.AWS_ACCOUNT_ID_PROD }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/job-distributor:0.28.0" | |
| CTF_CHAINLINK_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.event_name == 'pull_request' && format('nightly-{0}-plugins', steps.set-date.outputs.date) || inputs.chainlink_image_tag }}" | |
| CTF_CHIP_ROUTER_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1" | |
| DISABLE_DX_TRACKING: "true" | |
| CI: "true" | |
| run: | | |
| { time go run . env start; } 2> >(tee /tmp/time_output >&2) | |
| # Convert time to total seconds | |
| time_str=$(cat /tmp/time_output | grep real | awk '{print $2}') | |
| minutes=$(echo "${time_str}" | awk -F'm' '{print $1}') | |
| seconds=$(echo "${time_str}" | awk -F'm' '{print $2}' | sed 's/s//') | |
| total_seconds=$(echo "$minutes * 60 + $seconds" | bc) | |
| echo "execution_time=$total_seconds" >> "${GITHUB_OUTPUT}" | |
| - name: Deploy Cron workflow (Go) | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| run: | | |
| go run . env workflow deploy --compile --workflow-file-path ./examples/workflows/cron/main.go --name cronworkflow | |
| - name: Delete Cron workflow (Go) | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| run: | | |
| go run . env workflow delete --name cronworkflow | |
| - name: Deploy Cron workflow (TypeScript) | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| run: | | |
| # Create package.json | |
| cat > package.json << 'EOF' | |
| { | |
| "name": "typescript-por-template", | |
| "version": "1.0.0", | |
| "main": "dist/main.js", | |
| "private": true, | |
| "scripts": { | |
| "postinstall": "bunx cre-setup" | |
| }, | |
| "license": "UNLICENSED", | |
| "dependencies": { | |
| "@chainlink/cre-sdk": "^1.0.0", | |
| "viem": "2.34.0", | |
| "zod": "3.25.76" | |
| }, | |
| "devDependencies": { | |
| "@types/bun": "1.2.21" | |
| } | |
| } | |
| EOF | |
| # Install Bun packages | |
| bun install | |
| # Compile and deploy TS workflow | |
| go run . env workflow deploy --compile -n tsworkflow -w examples/workflows/ts/cron/index.ts -c examples/workflows/ts/cron/config.json | |
| - name: Build capability binaries for swap | |
| shell: bash | |
| env: | |
| GOPRIVATE: "github.com/smartcontractkit/*" | |
| run: | | |
| mkdir -p core/scripts/cre/environment/binaries | |
| GOBIN=$(pwd)/core/scripts/cre/environment/binaries go tool loopinstall plugins/plugins.private.yaml | |
| - name: Swap cron capability | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| run: | | |
| go run . env swap capability -n cron -b ./binaries/cron | |
| - name: Swap CL node image | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| env: | |
| CTF_CHAINLINK_IMAGE: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.event_name == 'pull_request' && format('nightly-{0}-plugins', steps.set-date.outputs.date) || inputs.chainlink_image_tag }}" | |
| run: | | |
| go run . env swap nodes | |
| - name: Execute example PoR workflow | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| run: | | |
| go run . env workflow run-por-example | |
| # no need to delete it as that command does it automatically | |
| - name: List environment state files | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| run: | | |
| go run . env state list | |
| - name: Purge environment state files | |
| shell: bash | |
| working-directory: core/scripts/cre/environment | |
| run: | | |
| go run . env state purge | |
| - name: Track local env startup success rate | |
| # track only scheduled runs to exclude failures in PRs or manual runs | |
| if: always() && github.event_name == 'schedule' | |
| shell: bash | |
| env: | |
| START_CLI_OUTCOME: ${{ steps.start-cli.outcome }} | |
| GETDX_SECRET_KEY: ${{ secrets.GETDX_SECRET_KEY }} | |
| DX_ACTOR: ${{ contains(github.actor, 'github-merge-queue') && 'Tofel' || github.actor }} | |
| run: | | |
| success=false | |
| if [[ "${START_CLI_OUTCOME}" == "success" ]]; then success=true; fi | |
| echo "::debug::success: $success" | |
| printf -v data '{"name": "cre.local.startup.result", "github_username": "%s", "timestamp": "%s", "metadata": {"success": "%s", "is_ci": true, "error": "N/A", "panicked": "N/A", "runner": "self-hosted-ubuntu-24", "infra": "docker"}}' "$DX_ACTOR" "$EPOCHSECONDS" "$success" | |
| curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer ${GETDX_SECRET_KEY}" -d "$data" https://api.getdx.com/events.track | |
| - name: Track local env startup duration | |
| # track only scheduled runs to exclude failures in PRs or manual runs | |
| if: always() && github.event_name == 'schedule' | |
| shell: bash | |
| env: | |
| GETDX_SECRET_KEY: ${{ secrets.GETDX_SECRET_KEY }} | |
| EXECUTION_TIME: ${{ steps.start-cli.outputs.execution_time }} | |
| CACHE_HIT: ${{ steps.setup-go.outputs.cache-hit }} | |
| DEPS_DOWNLOAD_SECONDS: ${{ steps.go-deps.outputs.deps_download_seconds }} | |
| DX_ACTOR: ${{ contains(github.actor, 'github-merge-queue') && 'Tofel' || github.actor }} | |
| run: | | |
| echo "::debug::execution_time: ${EXECUTION_TIME}" | |
| echo "::debug::cache_hit: ${CACHE_HIT}" | |
| echo "::debug::deps_download_seconds: ${DEPS_DOWNLOAD_SECONDS}" | |
| printf -v data '{"name": "cre.local.startup.time", "github_username": "%s", "timestamp": "%s", "metadata": {"startup_seconds": %s, "is_ci": true, "runner": "self-hosted-ubuntu-24", "infra": "docker", "go_cache_hit": %s, "deps_download_seconds": %s}}' "$DX_ACTOR" "$EPOCHSECONDS" "${EXECUTION_TIME}" "${CACHE_HIT}" "${DEPS_DOWNLOAD_SECONDS}" | |
| curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer ${GETDX_SECRET_KEY}" -d "$data" https://api.getdx.com/events.track | |
| - name: Upload all artifacts as single package | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: local-cre-docker-logs | |
| path: ./core/scripts/cre/environment/logs/ | |
| notify-test-failure: | |
| name: Notify about local env startup failure | |
| if: always() && github.event_name != 'pull_request' && (needs.test-cli.result == 'failure' || needs.test-cli.result == 'cancelled') | |
| needs: [test-cli] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check if job timed out or failed (not manually cancelled) | |
| id: check-timeout | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| TEST_CLI_RESULT: ${{ needs.test-cli.result }} | |
| run: | | |
| # Fast path: hard failures always notify; no API call needed. | |
| # We only call GH API for cancellations to estimate timeout-like cancels | |
| # (using duration) vs likely manual cancels (to avoid noisy alerts). | |
| conclusion="${TEST_CLI_RESULT}" | |
| duration_minutes=0 | |
| should_notify="false" | |
| reason="failure" | |
| if [[ "$TEST_CLI_RESULT" == "failure" ]]; then | |
| should_notify="true" | |
| elif [[ "$TEST_CLI_RESULT" == "cancelled" ]]; then | |
| # Reusable workflows prefix job names, so match both "test-cli" and "... / test-cli". | |
| job_data=$(gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs --paginate \ | |
| --jq '[.jobs[] | select(.name == "test-cli" or (.name | endswith("/ test-cli")))] | first') | |
| api_conclusion=$(echo "$job_data" | jq -r '.conclusion // empty') | |
| started_at=$(echo "$job_data" | jq -r '.started_at // empty') | |
| completed_at=$(echo "$job_data" | jq -r '.completed_at // empty') | |
| if [[ -n "$api_conclusion" ]]; then | |
| conclusion="$api_conclusion" | |
| fi | |
| # Calculate duration in minutes when timestamps are available. | |
| if [[ -n "$started_at" && -n "$completed_at" ]]; then | |
| start_epoch=$(date -d "$started_at" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$started_at" +%s) | |
| end_epoch=$(date -d "$completed_at" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$completed_at" +%s) | |
| duration_minutes=$(( (end_epoch - start_epoch) / 60 )) | |
| fi | |
| # Notify for explicit failure-like conclusions. | |
| if [[ "$conclusion" == "failure" || "$conclusion" == "timed_out" || "$conclusion" == "startup_failure" || "$conclusion" == "action_required" ]]; then | |
| should_notify="true" | |
| reason="failure" | |
| elif [[ "$conclusion" == "cancelled" && $duration_minutes -ge 14 ]]; then | |
| should_notify="true" | |
| reason="timeout" | |
| fi | |
| fi | |
| echo "duration_minutes=$duration_minutes" >> "${GITHUB_OUTPUT}" | |
| echo "conclusion=$conclusion" >> "${GITHUB_OUTPUT}" | |
| echo "reason=$reason" >> "${GITHUB_OUTPUT}" | |
| echo "should_notify=$should_notify" >> "${GITHUB_OUTPUT}" | |
| - name: Send slack notification | |
| if: steps.check-timeout.outputs.should_notify == 'true' | |
| uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 | |
| with: | |
| errors: "true" | |
| method: chat.postMessage | |
| token: ${{ secrets.QA_SLACK_API_KEY }} | |
| payload: | | |
| { | |
| "channel": "C0948PT3X38", | |
| "text": "At least one local CRE test failed", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "*:rotating_light: Local CRE test ${{ steps.check-timeout.outputs.reason == 'timeout' && 'timed out' || 'failed' }} :rotating_light:*" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Alerting <@U060CGGPY8H> local CRE test ${{ steps.check-timeout.outputs.reason == 'timeout' && 'timed out' || 'failed' }} (duration: ${{ steps.check-timeout.outputs.duration_minutes }}min) for commit <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> on run ID <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>. This commit was already merged to develop." | |
| } | |
| } | |
| ] | |
| } |