DataDog · dd-mergequeue · Mar 28, 2025 · Mar 27, 2025 · Mar 28, 2025
@@ -167,15 +167,15 @@ variables:
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
   DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v59857552-25d5753f
+  DATADOG_AGENT_BUILDIMAGES: v60221061-324ab098
   DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_WINBUILDIMAGES: v59857552-25d5753f
+  DATADOG_AGENT_WINBUILDIMAGES: v60221061-324ab098
   DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v59857552-25d5753f
+  DATADOG_AGENT_ARMBUILDIMAGES: v60221061-324ab098
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v59857552-25d5753f
+  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v60221061-324ab098
   DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v59857552-25d5753f
+  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v60221061-324ab098
   # New images to enable different version per image - not used yet
   CI_IMAGE_BTF_GEN: v59857552-25d5753f
   CI_IMAGE_BTF_GEN_SUFFIX: ""
@@ -218,6 +218,7 @@ variables:
   DOCKER_REGISTRY_URL: docker.io
   KITCHEN_INFRASTRUCTURE_FLAKES_RETRY: 2
   CLANG_LLVM_VER: 12.0.1
+  CLANG_BUILD_VERSION: ""
   KERNEL_MATRIX_TESTING_X86_AMI_ID: "ami-05b3973acf5422348"
   KERNEL_MATRIX_TESTING_ARM_AMI_ID: "ami-0b5f838a19d37fc61"
   RUN_E2E_TESTS: "auto" # Should be "off", "auto" or "on" it will change the trigger condition for new-e2e tests on branch != main

@@ -7,7 +7,20 @@
     !reference [.manual]
   stage: deps_build
   variables:
+    KUBERNETES_MEMORY_REQUEST: 12Gi
+    KUBERNETES_MEMORY_LIMIT: 12Gi
     KUBERNETES_CPU_REQUEST: 4
+    BUILD_VERSION: "v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
+    CC: "/opt/toolchains/${CMAKE_ARCH}/bin/${CMAKE_ARCH}-unknown-linux-gnu-gcc"
+    CXX: "/opt/toolchains/${CMAKE_ARCH}/bin/${CMAKE_ARCH}-unknown-linux-gnu-g++"
+    CMAKE_SYSTEM_PROCESSOR: "${CMAKE_ARCH}"
+    CMAKE_SYSTEM_NAME: Linux
+    CMAKE_RANLIB: "/opt/toolchains/${CMAKE_ARCH}/bin/${CMAKE_ARCH}-unknown-linux-gnu-ranlib"
+    CMAKE_AR: "/opt/toolchains/${CMAKE_ARCH}/bin/${CMAKE_ARCH}-unknown-linux-gnu-ar"
+    CMAKE_C_COMPILER: "/opt/toolchains/${CMAKE_ARCH}/bin/${CMAKE_ARCH}-unknown-linux-gnu-gcc"
+    CMAKE_CXX_COMPILER: "/opt/toolchains/${CMAKE_ARCH}/bin/${CMAKE_ARCH}-unknown-linux-gnu-g++"
+    CMAKE_FIND_ROOT_PATH: /opt/datadog/embedded
+    CMAKE_FIND_ROOT_PATH_MODE_PACKAGE: ONLY
   script:
     # use tmpdir to prevent git remote capture by clang build
     - mkdir /tmp/clangbuild && cd /tmp/clangbuild
@@ -28,11 +41,20 @@
     - patch -ruN ../llvm/CMakeLists.txt < $CI_PROJECT_DIR/pkg/ebpf/c/cmakelists.patch
     - patch -ruN ../llvm/lib/Analysis/ConstantFolding.cpp < $CI_PROJECT_DIR/pkg/ebpf/c/constant_folding.patch
     - |
-      cmake -DLLVM_ENABLE_PROJECTS=clang \
+      cmake \
+      -DLLVM_ENABLE_PROJECTS=clang \
       -DLLVM_TARGETS_TO_BUILD="BPF" \
       -DCMAKE_INSTALL_PREFIX=$DATADOG_AGENT_EMBEDDED_PATH \
       -G "Ninja" \
       -DCMAKE_BUILD_TYPE=MinSizeRel \
+      -DCMAKE_SYSTEM_PROCESSOR=$CMAKE_SYSTEM_PROCESSOR \
+      -DCMAKE_SYSTEM_NAME=$CMAKE_SYSTEM_NAME \
+      -DCMAKE_RANLIB=$CMAKE_RANLIB \
+      -DCMAKE_AR=$CMAKE_AR \
+      -DCMAKE_C_COMPILER=$CMAKE_C_COMPILER \
+      -DCMAKE_CXX_COMPILER=$CMAKE_CXX_COMPILER \
+      -DCMAKE_FIND_ROOT_PATH=$CMAKE_FIND_ROOT_PATH \
+      -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=$CMAKE_FIND_ROOT_PATH_MODE_PACKAGE \
       -DLLVM_BUILD_TOOLS=ON \
       -DLLVM_ENABLE_TERMINFO=OFF \
       -DLLVM_INCLUDE_EXAMPLES=OFF \
@@ -43,6 +65,7 @@
       -DLLVM_PARALLEL_COMPILE_JOBS=4 \
       -DLLVM_PARALLEL_LINK_JOBS=4 \
       -DLLVM_ENABLE_UNWIND_TABLES=OFF \
+      -Wno-dev \
       ../llvm
     - cmake --build . --target install
     # fail if references to glibc >= 2.18
@@ -53,33 +76,39 @@
     - diff $CI_PROJECT_DIR/pkg/ebpf/c/expected_targets.txt clang_targets.txt
     - $DATADOG_AGENT_EMBEDDED_PATH/bin/llc --version | tail -n +7 > llc_targets.txt # remove first 6 lines of llc version output; we only care about the registered targets
     - diff $CI_PROJECT_DIR/pkg/ebpf/c/expected_targets.txt llc_targets.txt
-    - $S3_CP_CMD $DATADOG_AGENT_EMBEDDED_PATH/bin/clang-${CLANG_MAJOR_VER} $S3_PERMANENT_ARTIFACTS_URI/clang-${CLANG_LLVM_VER}.${ARCH}
-    - $S3_CP_CMD $DATADOG_AGENT_EMBEDDED_PATH/bin/llc $S3_PERMANENT_ARTIFACTS_URI/llc-${CLANG_LLVM_VER}.${ARCH}
-    - $S3_CP_CMD $DATADOG_AGENT_EMBEDDED_PATH/bin/clang-${CLANG_MAJOR_VER} $S3_DD_AGENT_OMNIBUS_LLVM_URI/clang-${CLANG_LLVM_VER}.${ARCH} --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
-    - $S3_CP_CMD $DATADOG_AGENT_EMBEDDED_PATH/bin/llc $S3_DD_AGENT_OMNIBUS_LLVM_URI/llc-${CLANG_LLVM_VER}.${ARCH} --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
+    - ${S3_CP_CMD} "${DATADOG_AGENT_EMBEDDED_PATH}/bin/clang-${CLANG_MAJOR_VER}" "${S3_PERMANENT_ARTIFACTS_URI}/clang-${CLANG_LLVM_VER}.${ARCH}.${BUILD_VERSION}"
+    - ${S3_CP_CMD} "${DATADOG_AGENT_EMBEDDED_PATH}/bin/llc" "${S3_PERMANENT_ARTIFACTS_URI}/llc-${CLANG_LLVM_VER}.${ARCH}.${BUILD_VERSION}"
+    - ${S3_CP_CMD} "${DATADOG_AGENT_EMBEDDED_PATH}/bin/llvm-strip" "${S3_PERMANENT_ARTIFACTS_URI}/llvm-strip-${CLANG_LLVM_VER}.${ARCH}.${BUILD_VERSION}"
+    - ${S3_CP_CMD} "${DATADOG_AGENT_EMBEDDED_PATH}/bin/clang-${CLANG_MAJOR_VER}" "${S3_DD_AGENT_OMNIBUS_LLVM_URI}/clang-${CLANG_LLVM_VER}.${ARCH}.${BUILD_VERSION}" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
+    - ${S3_CP_CMD} "${DATADOG_AGENT_EMBEDDED_PATH}/bin/llc" "${S3_DD_AGENT_OMNIBUS_LLVM_URI}/llc-${CLANG_LLVM_VER}.${ARCH}.${BUILD_VERSION}" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
+    - ${S3_CP_CMD} "${DATADOG_AGENT_EMBEDDED_PATH}/bin/llvm-strip" "${S3_DD_AGENT_OMNIBUS_LLVM_URI}/llvm-strip-${CLANG_LLVM_VER}.${ARCH}.${BUILD_VERSION}" --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
     - mkdir -p $CI_PROJECT_DIR/.tmp
     - cp $DATADOG_AGENT_EMBEDDED_PATH/bin/clang-${CLANG_MAJOR_VER} $CI_PROJECT_DIR/.tmp/clang
     - cp $DATADOG_AGENT_EMBEDDED_PATH/bin/llc $CI_PROJECT_DIR/.tmp/llc
+    - cp $DATADOG_AGENT_EMBEDDED_PATH/bin/llvm-strip $CI_PROJECT_DIR/.tmp/llvm-strip
   timeout: 2h 00m
   artifacts:
     expire_in: 2 weeks
     paths:
       - $CI_PROJECT_DIR/.tmp/clang
       - $CI_PROJECT_DIR/.tmp/llc
+      - $CI_PROJECT_DIR/.tmp/llvm-strip
 
 build_clang_x64:
   extends: .build_clang_common
-  image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-17-x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   variables:
     ARCH: amd64
+    CMAKE_ARCH: x86_64
 
 build_clang_arm64:
   extends: .build_clang_common
-  image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux-glibc-2-23-arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:arm64"]
   variables:
     ARCH: arm64
+    CMAKE_ARCH: aarch64
 
 build_processed_btfhub_archive:
   rules:

@@ -24,8 +24,8 @@
     - !reference [.setup_deb_signing_key]
     - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz
     - mkdir -p /tmp/system-probe
-    - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf
-    - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf
+    - ${S3_CP_CMD} "${S3_PERMANENT_ARTIFACTS_URI}/clang-${CLANG_LLVM_VER}.${PACKAGE_ARCH}${CLANG_BUILD_VERSION}" /tmp/system-probe/clang-bpf
+    - ${S3_CP_CMD} "${S3_PERMANENT_ARTIFACTS_URI}/llc-${CLANG_LLVM_VER}.${PACKAGE_ARCH}${CLANG_BUILD_VERSION}" /tmp/system-probe/llc-bpf
     - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz
     - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf
     - dda inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor heroku

@@ -15,8 +15,8 @@
     # Thus, we move the artifacts at the end in a gitlab-friendly dir.
     - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz
     - mkdir -p /tmp/system-probe
-    - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf
-    - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf
+    - ${S3_CP_CMD} "${S3_PERMANENT_ARTIFACTS_URI}/clang-${CLANG_LLVM_VER}.${PACKAGE_ARCH}${CLANG_BUILD_VERSION}" /tmp/system-probe/clang-bpf
+    - ${S3_CP_CMD} "${S3_PERMANENT_ARTIFACTS_URI}/llc-${CLANG_LLVM_VER}.${PACKAGE_ARCH}${CLANG_BUILD_VERSION}" /tmp/system-probe/llc-bpf
     - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz
     - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf
     # NOTE: for now, we consider "ociru" to be a "redhat_target" in omnibus/lib/ostools.rb

@@ -8,8 +8,8 @@
   # Thus, we move the artifacts at the end in a gitlab-friendly dir.
   - tar -xf $CI_PROJECT_DIR/sysprobe-build-outputs.tar.xz
   - mkdir -p /tmp/system-probe
-  - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/clang-bpf
-  - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.${PACKAGE_ARCH} /tmp/system-probe/llc-bpf
+  - ${S3_CP_CMD} "${S3_PERMANENT_ARTIFACTS_URI}/clang-${CLANG_LLVM_VER}.${PACKAGE_ARCH}${CLANG_BUILD_VERSION}" /tmp/system-probe/clang-bpf
+  - ${S3_CP_CMD} "${S3_PERMANENT_ARTIFACTS_URI}/llc-${CLANG_LLVM_VER}.${PACKAGE_ARCH}${CLANG_BUILD_VERSION}" /tmp/system-probe/llc-bpf
   - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz /tmp/system-probe/minimized-btfs.tar.xz
   - chmod 0744 /tmp/system-probe/clang-bpf /tmp/system-probe/llc-bpf
   - dda inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor "$FLAVOR" --config-directory "$CONFIG_DIR" --install-directory "$INSTALL_DIR"

@@ -2,8 +2,8 @@
 .retrieve_sysprobe_deps:
   - mkdir -p $DATADOG_AGENT_EMBEDDED_PATH/bin
   - mkdir -p $DATADOG_AGENT_EMBEDDED_PATH/include
-  - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/clang-$CLANG_LLVM_VER.$ARCH /tmp/clang-bpf
-  - $S3_CP_CMD $S3_PERMANENT_ARTIFACTS_URI/llc-$CLANG_LLVM_VER.$ARCH /tmp/llc-bpf
+  - ${S3_CP_CMD} "${S3_PERMANENT_ARTIFACTS_URI}/clang-${CLANG_LLVM_VER}.${ARCH}${CLANG_BUILD_VERSION}" /tmp/clang-bpf
+  - ${S3_CP_CMD} "${S3_PERMANENT_ARTIFACTS_URI}/llc-${CLANG_LLVM_VER}.${ARCH}${CLANG_BUILD_VERSION}" /tmp/llc-bpf
 
 # Run tests for eBPF code
 .tests_linux_ebpf:

@@ -15,12 +15,14 @@
 from subprocess import check_output
 
 import requests
+import yaml
 from invoke.context import Context
 from invoke.exceptions import Exit
 from invoke.tasks import task
 
 from tasks.build_tags import UNIT_TEST_TAGS, add_fips_tags, get_default_build_tags
 from tasks.libs.build.ninja import NinjaWriter
+from tasks.libs.ciproviders.gitlab_api import ReferenceTag
 from tasks.libs.common.color import color_message
 from tasks.libs.common.git import get_commit_sha
 from tasks.libs.common.utils import (
@@ -82,7 +84,6 @@
     "arm64": "arm64",  # darwin
 }
 CURRENT_ARCH = arch_mapping.get(platform.machine(), "x64")
-CLANG_VERSION_RUNTIME = "12.0.1"
 CLANG_VERSION_SYSTEM_PREFIX = "12.0"
 # system-probe doesn't depend on any particular version of libpcap so use the latest one (as of 2024-10-28)
 # this version should be kept in sync with the one in the agent omnibus build
@@ -1388,6 +1389,16 @@ def run_ninja(
         ctx.run(f"ninja {explain_opt} -f {nf_path} {target}")
 
 
+def get_clang_version_and_build_version() -> tuple[str, str]:
+    gitlab_ci_file = Path(__file__).parent.parent / ".gitlab-ci.yml"
+    yaml.SafeLoader.add_constructor(ReferenceTag.yaml_tag, ReferenceTag.from_yaml)
+    with open(gitlab_ci_file) as f:
+        ci_config = yaml.safe_load(f)
+
+    ci_vars = ci_config['variables']
+    return ci_vars['CLANG_LLVM_VER'], ci_vars['CLANG_BUILD_VERSION']
+
+
 def setup_runtime_clang(
     ctx: Context, arch: Arch | None = None, target_dir: Path | str = "/opt/datadog-agent/embedded/bin"
 ) -> None:
@@ -1398,42 +1409,48 @@ def setup_runtime_clang(
     if arch is None:
         arch = Arch.local()
 
-    clang_bpf_path = target_dir / "clang-bpf"
-    llc_bpf_path = target_dir / "llc-bpf"
-    needs_clang_download, needs_llc_download = True, True
-
-    if not arch.is_cross_compiling() and sys.platform == "linux":
-        # We can check the version of clang and llc on the system, we have the same arch and can
-        # execute the binaries. This way we can omit the download if the binaries exist and the version
-        # matches the desired one
-        clang_res = ctx.run(f"{sudo} {clang_bpf_path} --version", warn=True)
-        if clang_res is not None and clang_res.ok:
-            clang_version_str = clang_res.stdout.split("\n")[0].split(" ")[2].strip()
-            needs_clang_download = clang_version_str != CLANG_VERSION_RUNTIME
-
-        llc_res = ctx.run(f"{sudo} {llc_bpf_path} --version", warn=True)
-        if llc_res is not None and llc_res.ok:
-            llc_version_str = llc_res.stdout.split("\n")[1].strip().split(" ")[2].strip()
-            needs_llc_download = llc_version_str != CLANG_VERSION_RUNTIME
-    else:
-        # If we're cross-compiling we cannot check the version of clang and llc on the system,
-        # so we download them only if they don't exist
-        needs_clang_download = not clang_bpf_path.exists()
-        needs_llc_download = not llc_bpf_path.exists()
+    clang_version, clang_build_version = get_clang_version_and_build_version()
+
+    runtime_binaries = {
+        "clang-bpf": {"url_prefix": "clang", "version_line": 0, "needs_download": False},
+        "llc-bpf": {"url_prefix": "llc", "version_line": 1, "needs_download": False},
+    }
+
+    for binary, meta in runtime_binaries.items():
+        binary_path = target_dir / binary
+        if not arch.is_cross_compiling() and sys.platform == "linux":
+            if not binary_path.exists() or binary_path.stat().st_size == 0:
+                print(f"'{binary}' missing")
+                runtime_binaries[binary]["needs_download"] = True
+                continue
+
+            # We can check the version of clang and llc on the system, we have the same arch and can
+            # execute the binaries. This way we can omit the download if the binaries exist and the version
+            # matches the desired one
+            res = ctx.run(f"{sudo} {binary_path} --version", warn=True, hide=True)
+            if res is not None and res.ok:
+                version_str = res.stdout.split("\n")[meta["version_line"]].strip().split(" ")[2].strip()
+                if version_str != clang_version:
+                    print(f"'{binary}' version '{version_str}' is not required version '{clang_version}'")
+                    runtime_binaries[binary]["needs_download"] = True
+        else:
+            # If we're cross-compiling we cannot check the version of clang and llc on the system,
+            # so we download them only if they don't exist
+            runtime_binaries[binary]["needs_download"] = not binary_path.exists() or binary_path.stat().st_size == 0
 
     if not target_dir.exists():
         ctx.run(f"{sudo} mkdir -p {target_dir}")
 
-    if needs_clang_download:
+    for binary, meta in runtime_binaries.items():
+        if not meta["needs_download"]:
+            continue
+
         # download correct version from dd-agent-omnibus S3 bucket
-        clang_url = f"https://dd-agent-omnibus.s3.amazonaws.com/llvm/clang-{CLANG_VERSION_RUNTIME}.{arch.name}"
-        ctx.run(f"{sudo} wget -q {clang_url} -O {clang_bpf_path}")
-        ctx.run(f"{sudo} chmod 0755 {clang_bpf_path}")
-
-    if needs_llc_download:
-        llc_url = f"https://dd-agent-omnibus.s3.amazonaws.com/llvm/llc-{CLANG_VERSION_RUNTIME}.{arch.name}"
-        ctx.run(f"{sudo} wget -q {llc_url} -O {llc_bpf_path}")
-        ctx.run(f"{sudo} chmod 0755 {llc_bpf_path}")
+        binary_url = f"https://dd-agent-omnibus.s3.amazonaws.com/llvm/{meta['url_prefix']}-{clang_version}.{arch.name}{clang_build_version}"
+        binary_path = target_dir / binary
+        print(f"'{binary}' downloading...")
+        ctx.run(f"{sudo} wget -nv {binary_url} -O {binary_path}")
+        ctx.run(f"{sudo} chmod 0755 {binary_path}")
 
 
 def verify_system_clang_version(ctx):