feat(appsec): upgrade to go-libddwaf v5 with subcontext-based ephemer… #14424
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Smoke Tests | |
| on: | |
| workflow_call: # allows to reuse this workflow | |
| inputs: | |
| ref: | |
| description: The branch or tag to run the workflow on | |
| required: true | |
| type: string | |
| go-libddwaf-ref: | |
| description: A git ref to update github.com/DataDog/go-libddwaf/v4 to. No-op if empty. | |
| required: false | |
| type: string | |
| push: | |
| branches: | |
| - release-v* | |
| - main | |
| tags-ignore: | |
| - 'contrib/**' | |
| - 'instrumentation/**' | |
| - 'internal/**' | |
| - 'orchestrion/**' | |
| - 'scripts/**' | |
| schedule: # nightly | |
| - cron: "0 0 * * *" | |
| workflow_dispatch: { } # manually | |
| env: | |
| TEST_RESULTS: /tmp/test-results # path to where test results will be saved | |
| GOPROXY: "proxy.golang.org|direct" # fall back to `direct` in case of proxy issues | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| setup-env: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| env: | |
| PACKAGES: ./internal/... ./ddtrace/... ./profiler/... ./appsec/... ./instrumentation/... | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ inputs.ref || github.ref }} | |
| repository: DataDog/dd-trace-go | |
| - name: Setup Go and development tools | |
| uses: ./.github/actions/setup-go | |
| with: | |
| go-version: stable | |
| tools-dir: ${{ github.workspace }}/_tools | |
| tools-bin: ${{ github.workspace }}/bin | |
| - name: Compile dd-trace-go with updated dependencies (sandboxed) | |
| # go get -u executes dependency code from the network. Keep those | |
| # changes inside the gVisor sandbox; this setup job only needs to prove | |
| # the updated dependency graph still compiles. | |
| uses: geomys/sandboxed-step@7d75eb49d17fdeeb3656b3a57d35932d205bcfb9 # v1.2.1 | |
| with: | |
| env: | | |
| GOPROXY=${{ env.GOPROXY }} | |
| GO_LIBDDWAF_REF=${{ inputs.go-libddwaf-ref }} | |
| PACKAGES=${{ env.PACKAGES }} | |
| TEST_RESULTS=${{ env.TEST_RESULTS }} | |
| run: |- | |
| mkdir -p "$TEST_RESULTS" | |
| # shellcheck disable=SC2086 | |
| go get -u -t $PACKAGES | |
| go mod tidy | |
| find . -iname go.mod -exec dirname {} \; | while read -r d; do pushd "$d" && go mod tidy && popd; done | |
| if [ -n "${GO_LIBDDWAF_REF:-}" ]; then | |
| go get -u -t "github.com/DataDog/go-libddwaf/v4@${GO_LIBDDWAF_REF}" | |
| go mod tidy | |
| fi | |
| # shellcheck disable=SC2086 | |
| go build $PACKAGES | |
| - name: Compute Matrix | |
| id: matrix | |
| run: |- | |
| echo -n "matrix=" >> "${GITHUB_OUTPUT}" | |
| go run ./scripts/ci_contrib_matrix.go >> "${GITHUB_OUTPUT}" | |
| go-get-u: | |
| # Run go get -u to upgrade dd-trace-go dependencies to their | |
| # latest minor version and see if dd-trace-go still compiles. | |
| # Related to issue https://github.com/DataDog/dd-trace-go/issues/1607 | |
| name: 'go get -u smoke test' | |
| needs: setup-env | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| chunk: ${{ fromJson(needs.setup-env.outputs.matrix) }} | |
| runs-on: ubuntu-latest | |
| if: github.repository_owner == 'DataDog' # only run on DataDog's repository, not in forks | |
| env: | |
| TEST_RESULTS: ${{ github.workspace }}/test-results | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ inputs.ref || github.ref }} | |
| # Manually specify the repository, which is necessary in the workflow_call situation, as the default is | |
| # otherwise the repository where the caller workflow started from. In this case, we need to check out the | |
| # repository where the called workflow is (i.e, this repository); but I don't know of a more elegant way to | |
| # obtain its name than hard-coding it. | |
| repository: DataDog/dd-trace-go | |
| - name: Setup Go and development tools | |
| uses: ./.github/actions/setup-go | |
| with: | |
| go-version: stable | |
| tools-dir: ${{ github.workspace }}/_tools | |
| tools-bin: ${{ github.workspace }}/bin | |
| - name: Capture trusted commit SHA | |
| # Pinned before any sandboxed step so we can re-checkout this exact | |
| # commit after the sandbox runs. Anything that escapes the sandbox | |
| # could otherwise rewrite .git/HEAD or refs and trick the host into | |
| # running attacker-controlled code via local actions. | |
| id: trusted | |
| run: echo "sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" | |
| - name: Test contribs (sandboxed) | |
| # It needs to run before "Test dd-trace-go" to avoid TestTelemetryEnabled tests to fail. | |
| # The smoke script runs go get -u inside each contrib module, so run it | |
| # in gVisor and persist only the generated reports/coverage for uploads. | |
| uses: geomys/sandboxed-step@7d75eb49d17fdeeb3656b3a57d35932d205bcfb9 # v1.2.1 | |
| with: | |
| persist-workspace-changes: 'true' | |
| env: | | |
| GOPROXY=${{ env.GOPROXY }} | |
| TEST_RESULTS=${{ env.TEST_RESULTS }} | |
| run: |- | |
| export PATH="${GITHUB_WORKSPACE}/bin:${PATH}" | |
| ./scripts/ci_test_contrib.sh smoke ${{ toJson(matrix.chunk) }} | |
| - name: Stage sandbox artifacts outside workspace | |
| # Move the test results and coverage files to RUNNER_TEMP so the | |
| # next checkout (which wipes the workspace) does not delete them. | |
| # Only regular files are staged, to avoid following any symlinks | |
| # that may have been planted inside the sandbox. | |
| id: stage | |
| if: always() | |
| run: |- | |
| set -euo pipefail | |
| SAFE="${RUNNER_TEMP}/sandbox-artifacts" | |
| rm -rf "${SAFE}" | |
| mkdir -p "${SAFE}" | |
| if [ -d "${TEST_RESULTS}" ]; then | |
| ( cd "${TEST_RESULTS}" && find . -type f -print0 \ | |
| | tar --null --no-recursion -T - -cf "${SAFE}/test-results.tar" ) | |
| fi | |
| ( cd "${GITHUB_WORKSPACE}" \ | |
| && find . -type f \( -name 'coverage*.txt' -o -name 'profile.cov' \) -print0 \ | |
| | tar --null --no-recursion -T - -cf "${SAFE}/coverage.tar" ) || true | |
| - name: Re-checkout trusted commit after sandbox | |
| # Wipe the (potentially tampered) workspace and check out the SHA we | |
| # captured before the sandbox ran. This restores .git, tracked files, | |
| # and any local actions to a known-good state before any credentialed | |
| # local action is invoked. | |
| id: recheckout | |
| if: always() | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ steps.trusted.outputs.sha }} | |
| repository: DataDog/dd-trace-go | |
| clean: true | |
| - name: Restore sandbox artifacts | |
| id: restore | |
| if: always() && steps.recheckout.outcome == 'success' | |
| run: |- | |
| set -euo pipefail | |
| SAFE="${RUNNER_TEMP}/sandbox-artifacts" | |
| mkdir -p "${TEST_RESULTS}" | |
| if [ -f "${SAFE}/test-results.tar" ]; then | |
| tar -C "${TEST_RESULTS}" -xf "${SAFE}/test-results.tar" | |
| fi | |
| if [ -f "${SAFE}/coverage.tar" ]; then | |
| tar -C "${GITHUB_WORKSPACE}" -xf "${SAFE}/coverage.tar" | |
| fi | |
| - name: Get Datadog credentials | |
| id: dd-sts | |
| if: always() && steps.restore.outcome == 'success' | |
| continue-on-error: true | |
| uses: DataDog/dd-sts-action@7d2d231c02fd54a3da912e582ff87cb995d1fd30 | |
| with: | |
| policy: dd-trace-go | |
| - name: Upload the results to Datadog CI App | |
| if: always() && steps.restore.outcome == 'success' | |
| continue-on-error: true | |
| uses: ./.github/actions/dd-ci-upload | |
| with: | |
| dd-api-key: ${{ steps.dd-sts.outputs.api_key }} | |
| path: ${{ env.TEST_RESULTS }} | |
| tags: go:stable,arch:${{ runner.arch }},os:${{ runner.os }} | |
| - name: Merge contrib coverage files | |
| if: always() && steps.restore.outcome == 'success' | |
| run: | | |
| echo "mode: atomic" > coverage-contrib-merged.txt | |
| find . \( -path "*/contrib/*" -o -path "*/instrumentation/*" \) \ | |
| -name "coverage-*.txt" -print0 \ | |
| | xargs -0 grep -h -v "^mode:" >> coverage-contrib-merged.txt 2>/dev/null || true | |
| - name: Upload Coverage to Datadog | |
| if: always() && steps.restore.outcome == 'success' | |
| continue-on-error: true | |
| uses: DataDog/coverage-upload-github-action@d9548b1c3c4ab639d5d3ab29a1508af188975f77 | |
| with: | |
| api_key: ${{ steps.dd-sts.outputs.api_key }} | |
| files: coverage-contrib-merged.txt | |
| format: go-coverprofile | |
| smoke-test-tracer: | |
| name: Test dd-trace-go | |
| runs-on: ubuntu-latest | |
| env: | |
| PACKAGES: ./internal/... ./ddtrace/... ./profiler/... ./appsec/... ./instrumentation/... | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ inputs.ref || github.ref }} | |
| repository: DataDog/dd-trace-go | |
| - name: Setup Go and development tools | |
| uses: ./.github/actions/setup-go | |
| with: | |
| go-version: "1.25" | |
| tools-dir: ${{ github.workspace }}/_tools | |
| tools-bin: ${{ github.workspace }}/bin | |
| - name: Run tests | |
| run: | | |
| export PATH="${{ github.workspace }}/bin:${PATH}" | |
| # shellcheck disable=SC2086 | |
| gotestsum --junitfile "${TEST_RESULTS}/gotestsum-report-smoke-tracer.xml" -- -v $PACKAGES | |
| # Check for changes in the supported_configurations.json file | |
| - name: Supported Configurations Diff Check | |
| if: always() | |
| uses: ./.github/actions/supported_configurations_validation | |
| - name: Get Datadog credentials | |
| id: dd-sts | |
| if: always() | |
| continue-on-error: true | |
| uses: DataDog/dd-sts-action@7d2d231c02fd54a3da912e582ff87cb995d1fd30 | |
| with: | |
| policy: dd-trace-go | |
| - name: Upload the results to Datadog CI App | |
| if: always() | |
| continue-on-error: true | |
| uses: ./.github/actions/dd-ci-upload | |
| with: | |
| dd-api-key: ${{ steps.dd-sts.outputs.api_key }} | |
| path: ${{ env.TEST_RESULTS }} | |
| tags: go:1.25,arch:${{ runner.arch }},os:${{ runner.os }} | |
| check-gen-files: | |
| name: Check generated files and go mod files are up-to-date. | |
| runs-on: ubuntu-latest | |
| if: inputs.go-libddwaf-ref == '' | |
| env: | |
| # Users may build our library with GOTOOLCHAIN=local. If they do, and our | |
| # go.mod file specifies a newer Go version than their local toolchain, their | |
| # build will break. Run our tests with GOTOOLCHAIN=local to ensure that | |
| # our library builds with all of the Go versions we claim to support, | |
| # without having to download a newer one. | |
| GOTOOLCHAIN: local | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ inputs.ref || github.ref }} | |
| - name: Setup Go and development tools | |
| uses: ./.github/actions/setup-go | |
| with: | |
| go-version: "1.25" | |
| tools-dir: ${{ github.workspace }}/_tools | |
| tools-bin: ${{ github.workspace }}/bin | |
| - name: Run generate scripts and check diff | |
| run: |- | |
| export PATH="${{ github.workspace }}/bin:${PATH}" | |
| ./scripts/generate.sh | |
| ./scripts/fix_modules.sh | |
| git diff --exit-code | |
| # TODO: macos setup requirements (xcode tools installation, etc.) | |
| setup-requirements-linux: | |
| # Build and deployment setup smoke test of linux containers built from the | |
| # golang docker image to test that dd-trace-go doesn't need more than the | |
| # "out-of-the-box" images. It is expected require a few more tools when CGO | |
| # is enabled, but nothing more than gcc and the C library, but nothing more. | |
| # Anything more than this "standard Go build and deployment requirements" | |
| # must be considered breaking changes. | |
| name: 'Build and deployment requirements smoke tests' | |
| runs-on: ubuntu-latest | |
| if: github.repository_owner == 'DataDog' # only run on DataDog's repository, not in forks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # TODO: cross-compilation from/to different hardware architectures once | |
| # github provides native ARM runners. | |
| go: [ "1.26", "1.25" ] | |
| build-env: [ alpine, trixie, bookworm ] | |
| build-with-cgo: [ 0, 1 ] | |
| deployment-env: [ alpine, debian12, debian13, al2, al2023, busybox, scratchy ] | |
| include: | |
| # GitHub limits the number of matrix jobs to 256, so we need to reduce | |
| # it a bit, and we can reduce redundant tests. | |
| # 1. Building with `go mod vendoring` is not worth it on all the | |
| # possible build and deployment envs. | |
| - build-env: alpine | |
| build-with-vendoring: y | |
| build-with-cgo: 1 # cgo's build tag can impact the vendored files | |
| deployment-env: alpine | |
| - build-env: alpine | |
| build-with-vendoring: y | |
| build-with-cgo: 0 # cgo's build tag can impact the vendored files | |
| deployment-env: alpine | |
| # 2. Given the low blast radius of the busybox deployment environment | |
| # this is the only one where we accept the libdl.so.2 requirement. | |
| # For this reason, we add the datadog.no_waf build tag in this only | |
| # case to avoid the libdl.so.2 dependency. | |
| - deployment-env: busybox | |
| build-with-cgo: 1 | |
| build-tags: "datadog.no_waf" | |
| exclude: | |
| # Exclude "out of the box" cases requiring extra setup: | |
| # 1. Building with CGO enabled on alpine but deploying to a non-alpine | |
| # environment: the C library isn't located at the same place. | |
| - build-env: alpine | |
| build-with-cgo: 1 | |
| deployment-env: debian12 | |
| - build-env: alpine | |
| build-with-cgo: 1 | |
| deployment-env: debian13 | |
| - build-env: alpine | |
| build-with-cgo: 1 | |
| deployment-env: al2 | |
| - build-env: alpine | |
| build-with-cgo: 1 | |
| deployment-env: al2023 | |
| - build-env: alpine | |
| build-with-cgo: 1 | |
| deployment-env: busybox | |
| - build-env: alpine | |
| build-with-cgo: 1 | |
| deployment-env: scratchy | |
| # 2. Too old glibc on the deployment environment than on the build env | |
| - build-env: bookworm | |
| deployment-env: al2 | |
| - build-env: trixie | |
| deployment-env: al2 | |
| # 3. Build with CGO enabled and deploying to a scratch/busybox docker | |
| # image requires copying the dynamic lib dependencies (full example | |
| # provided at https://github.com/DataDog/appsec-go-test-app/blob/main/examples/docker/scratch/Dockerfile) | |
| - build-with-cgo: 1 | |
| deployment-env: scratchy | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ inputs.ref || github.ref }} | |
| # Manually specify the repository, which is necessary in the workflow_call situation, as the default is | |
| # otherwise the repository where the caller workflow started from. In this case, we need to check out the | |
| # repository where the called workflow is (i.e, this repository); but I don't know of a more elegant way to | |
| # obtain its name than hard-coding it. | |
| repository: DataDog/dd-trace-go | |
| - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 | |
| - name: Build | |
| uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 | |
| with: | |
| context: . | |
| file: ./internal/setup-smoke-test/Dockerfile | |
| push: false | |
| load: true | |
| tags: smoke-test | |
| build-args: | | |
| go=${{ matrix.go }} | |
| build_env=${{ matrix.build-env }} | |
| build_tags=${{ matrix.build-tags }} | |
| build_with_vendoring=${{ matrix.build-with-vendoring }} | |
| build_with_cgo=${{ matrix.build-with-cgo }} | |
| deployment_env=${{ matrix.deployment-env }} | |
| go_libddwaf_ref=${{ inputs.go-libddwaf-ref }} | |
| - name: Test | |
| run: docker run -p7777:7777 --rm smoke-test | |
| smoke-tests-done: | |
| name: Smoke Tests | |
| needs: | |
| - setup-requirements-linux | |
| - smoke-test-tracer | |
| - go-get-u | |
| runs-on: ubuntu-latest | |
| if: success() || failure() | |
| steps: | |
| - name: Success | |
| if: needs.setup-requirements-linux.result == 'success' && needs.smoke-test-tracer.result == 'success' && needs.go-get-u.result == 'success' | |
| run: echo "Success!" | |
| - name: Failure | |
| if: needs.setup-requirements-linux.result != 'success' || needs.smoke-test-tracer.result != 'success' || needs.go-get-u.result != 'success' | |
| uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 | |
| with: | |
| webhook-type: incoming-webhook | |
| payload: | | |
| { | |
| "text": "Smoke Tests Failed", | |
| "linux_result": "${{ needs.setup-requirements-linux.result }}", | |
| "core_result": "${{ needs.smoke-test-tracer.result }}", | |
| "contrib_result": "${{ needs.go-get-u.result }}", | |
| "blocks": [ | |
| { | |
| "type": "header", | |
| "text": { | |
| "type": "plain_text", | |
| "text": ":x: Smoke Tests Failed on Main!" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "fields": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Repository:*\n${{ github.repository }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Branch:*\n${{ github.ref_name }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Workflow:*\n${{ github.workflow }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Linux Result:*\n${{ needs.setup-requirements-linux.result }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Tracer Result:*\n${{ needs.smoke-test-tracer.result }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Contrib Test Result:*\n${{ needs.go-get-u.result }}" | |
| }, | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_SMOKE_TESTS_WEBHOOK_URL }} |