CI: Add multi-target build infra

Author: Alexey-Rivkin

.ci/jenkins/lib/build-multi-target-matrix.yaml

@@ -0,0 +1,112 @@
+# NIXL PR Testing Matrix Configuration
+#
+# This file defines the build matrix for NIXL PR validation using the multi-target Docker system.
+# Primary purpose: Fast feedback on code changes via build + test execution.
+#
+# Key Components:
+# - Single NIXL Image Build: Uses multi-stage Dockerfile (base -> nixl) with layer caching
+# - Integrated Testing: Runs C++ and Python tests directly in the built environment
+# - CI-Demo Integration: Base image auto-builds only when Dockerfile changes
+#
+# PR Testing Benefits:
+# - Fast feedback: Base layer cached when only code changes
+# - Full rebuild: When infrastructure (Dockerfile) changes
+# - No registry operations: Focus on validation, not distribution
+# - Multi-OS/arch testing: Ensures compatibility across platforms
+#
+
+---
+job: nixl-ci-build-multi-target
+
+# Registry configuration for base image reusability
+registry_host: urm.nvidia.com
+registry_path: /sw-nbu-swx-nixl-docker-local/ci
+registry_auth: svc-nixl-artifactory-token
+
+# Build settings
+failFast: false
+timeout_minutes: 120
+
+# Infrastructure requirements
+kubernetes:
+  cloud: il-ipp-blossom-prod
+  namespace: swx-media
+  limits: "{memory: 16Gi, cpu: 8000m}"
+  requests: "{memory: 8Gi, cpu: 4000m}"
+
+# Container images: Base for reusability + NIXL for PR testing
+runs_on_dockers:
+  - { name: "podman-v5.0.2", url: "quay.io/podman/stable:v5.0.2", privileged: true, category: 'tool' }
+
+  # BASE IMAGE - Built & Pushed for Reusability
+  # Registry: urm.nvidia.com/sw-nbu-swx-nixl-docker-local/ci/{arch}/nixl-base:base-{os}-{arch}
+  - { file: 'contrib/Dockerfile.base', name: "nixl-base", tag: "base-${os}-${arch}", arch: '${arch}', build_args: 'OS=${os} ARCH=${arch}', category: 'reusable' }
+
+  # NIXL TEST IMAGE - Uses base via FROM directive (PR testing only)
+  # Local only: Built from base + NIXL source for validation
+  - { file: 'contrib/Dockerfile.nixl', name: "nixl-pr-test", tag: "test-${os}-${arch}", arch: '${arch}', build_args: 'OS=${os} ARCH=${arch} REGISTRY_HOST=urm.nvidia.com REGISTRY_PATH=/sw-nbu-swx-nixl-docker-local/ci', category: 'test' }
+
+# Build matrix - test both architectures and OS versions
+matrix:
+  axes:
+    arch:
+      - x86_64
+      - aarch64
+    os:
+      - ubuntu22.04
+      - ubuntu24.04
+
+# Environment configuration
+env:
+  NIXL_INSTALL_DIR: "/usr/local/nixl"
+  NPROC: "16"
+
+# Task naming for parallel execution
+taskName: "${os}/${arch}/${axis_index}"
+
+credentials:
+  - credentialsId: 'svc-nixl-artifactory-token'
+    usernameVariable: 'ARTIFACTORY_USERNAME'
+    passwordVariable: 'ARTIFACTORY_PASSWORD'
+
+# Build pipeline steps
+steps:
+  - name: Test NIXL with PR Changes
+    parallel: true
+    containerSelector: "{ name: 'nixl-pr-test' }"
+    run: |
+      # Set up environment
+      cd /workspace/nixl
+      export ARCH=$(uname -m)
+      [ "$ARCH" = "arm64" ] && ARCH="aarch64"
+
+      .gitlab/test_cpp.sh ${NIXL_INSTALL_DIR}
+      .gitlab/test_python.sh ${NIXL_INSTALL_DIR}
+
+
+# Pipeline completion
+pipeline_stop:
+  shell: action
+  module: groovy
+  run: |
+    def jobStatus = currentBuild.result ?: 'SUCCESS'
+    def statusColor = jobStatus == 'SUCCESS' ? 'green' : 'red'
+
+    echo "NIXL PR testing completed with status: ${jobStatus}"
+
+        if (params.MAIL_TO) {
+        def userName = currentBuild.rawBuild.getCause(hudson.model.Cause.UserIdCause)?.userName ?: 'schedule'
+
+        mail(
+            to: params.MAIL_TO,
+            subject: "NIXL PR Testing [${env.BUILD_NUMBER}] - ${jobStatus}",
+            mimeType: 'text/html',
+            body: """
+                <h3>NIXL PR Testing Results</h3>
+                <p><b>Status:</b> <span style="color: ${statusColor};">${jobStatus}</span></p>
+                <p><b>Build:</b> <a href='${env.BUILD_URL}'>#${env.BUILD_NUMBER}</a></p>
+                <p><b>Images:</b> Base image pushed for reuse, NIXL tested with PR changes</p>
+                <p><a href='${env.BUILD_URL}console'>View Console Output</a></p>
+            """
+        )
+    }

.ci/jenkins/pipeline/proj-jjb.yaml

@@ -316,6 +316,71 @@
               parent-credentials: true
       script-path: "{jjb_jenkinsfile}"
 
+# Template for multi-target build job using the new unified Dockerfile approach
+- job-template:
+    name: "{jjb_proj}-build-multi-target"
+    project-type: pipeline
+    folder: "{jjb_folder}"
+    disabled: false
+    properties:
+        - github:
+            url: "{jjb_gh_url}"
+        - build-discarder:
+            days-to-keep: 14
+            num-to-keep: 1000
+        - inject:
+            keep-system-variables: true
+            properties-content: |
+              jjb_proj={jjb_proj}-build-multi-target
+    description: >
+      <b>NIXL PR Testing Pipeline</b><br/>
+      • Fast feedback for code changes using multi-target Docker system<br/>
+      • Builds NIXL with PR changes (base layer cached when possible)<br/>
+      • Supports both Ubuntu 22.04 and 24.04 on x86_64/aarch64<br/>
+      • Runs comprehensive C++ and Python test suites<br/>
+      <br/>
+      <i>Do NOT edit this job through the Jenkins GUI — managed by Jenkins Job Builder.</i>
+    concurrent: true
+    sandbox: true
+    parameters:
+        - string:
+            name: "sha1"
+            default: "{jjb_branch}"
+            description: "Git commit/branch/tag to build (default: main)"
+        - string:
+            name: "githubData"
+            default: ""
+            description: "GitHub webhook data (automatically set)"
+        - string:
+            name: "conf_file"
+            default: ".ci/jenkins/lib/build-multi-target-matrix.yaml"
+            description: "Build matrix configuration file"
+        - string:
+            name: "MAIL_TO"
+            default: ""
+            description: "Email address for build notifications (optional)"
+        - string:
+            name: "DEBUG"
+            default: 0
+            description: "Debug level (0-9) for verbose output"
+    pipeline-scm:
+        scm:
+            - git:
+                url: "{jjb_git}"
+                credentials-id: 'swx-jenkins_ssh_key'
+                branches: ['$sha1']
+                shallow-clone: false
+                do-not-fetch-tags: false
+                refspec: "+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/* +refs/tags/*:refs/remotes/origin/tags/*"
+                browser: githubweb
+                browser-url: "{jjb_git}"
+                submodule:
+                    disable: false
+                    recursive: true
+                    tracking: true
+                    parent-credentials: true
+        script-path: "{jjb_jenkinsfile}"
+
 # Project definition that instantiates the job templates
 # This section defines the actual jobs that will be created
 - project:
@@ -327,7 +392,8 @@
     jjb_branch: 'main'            # Default branch
     jjb_gh_url: 'https://github.com/ai-dynamo/nixl'  # GitHub web URL
     jobs:
-        - "{jjb_proj}-dispatcher"  # Create dispatcher job
-        - "{jjb_proj}-build"       # Create build job
-        - "{jjb_proj}-build-container"  # Create container builder job
-        - "{jjb_proj}-test"        # Create test job
+        # - "{jjb_proj}-dispatcher"  # Create dispatcher job
+        # - "{jjb_proj}-build"       # Create build job
+        # - "{jjb_proj}-build-container"  # Create container builder job
+        - "{jjb_proj}-build-multi-target"  # Create multi-target build job
+        # - "{jjb_proj}-test"        # Create test job

contrib/Dockerfile.base

@@ -0,0 +1,157 @@
+# Base Dockerfile for NIXL CI/CD system
+# Contains shared dependencies: system packages, UCX, Rust, AWS SDK, etc.
+# Used by both NIXL and NIXLBench builds
+
+ARG BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base"
+ARG BASE_IMAGE_TAG="25.06-cuda12.9-devel-ubuntu24.04"
+ARG OS="ubuntu24.04"
+ARG ARCH="x86_64"
+ARG DEFAULT_PYTHON_VERSION="3.12"
+ARG UCX_REF="v1.19.x"
+ARG NPROC="16"
+
+FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG}
+
+ARG ARCH
+ARG OS
+ARG DEFAULT_PYTHON_VERSION
+ARG NPROC
+ARG UCX_REF
+
+# Install common system packages, build tools, and RDMA/InfiniBand packages
+RUN apt-get update -y && \
+    apt-get install -y ubuntu-keyring && \
+    DEBIAN_FRONTEND=noninteractive apt-get -y install \
+    autotools-dev \
+    automake \
+    build-essential \
+    cmake \
+    etcd-client \
+    etcd-server \
+    flex \
+    ibverbs-providers \
+    ibverbs-utils \
+    libaio-dev \
+    libibumad-dev \
+    libibverbs-dev \
+    libclang-dev \
+    libcpprest-dev \
+    libcurl4-openssl-dev \
+    libgflags-dev \
+    libgrpc-dev \
+    libgrpc++-dev \
+    libgtest-dev \
+    libnuma-dev \
+    libprotobuf-dev \
+    librdmacm-dev \
+    libssl-dev \
+    libtool \
+    liburing-dev \
+    libz-dev \
+    ninja-build \
+    protobuf-compiler-grpc \
+    pybind11-dev \
+    python${DEFAULT_PYTHON_VERSION}-dev \
+    rdma-core \
+    uuid-dev \
+    zlib1g-dev
+
+# Add Mellanox repository and install DOCA packages
+RUN case "${ARCH}" in \
+        aarch64) ARCH_SUFFIX="arm64-sbsa" ;; \
+        x86_64) ARCH_SUFFIX="${ARCH}" ;; \
+    esac && \
+    export PKG_CONFIG_PATH="/opt/mellanox/doca/lib/${ARCH_SUFFIX}-linux-gnu/pkgconfig:/opt/mellanox/dpdk/lib/${ARCH_SUFFIX}-linux-gnu/pkgconfig:$PKG_CONFIG_PATH" && \
+    curl -fsSL https://linux.mellanox.com/public/repo/doca/3.0.0/${OS}/${ARCH_SUFFIX}/GPG-KEY-Mellanox.pub | \
+    gpg --dearmor | tee /usr/share/keyrings/mellanox-archive-keyring.gpg && \
+    echo "deb [signed-by=/usr/share/keyrings/mellanox-archive-keyring.gpg] https://linux.mellanox.com/public/repo/doca/3.0.0/${OS}/${ARCH_SUFFIX} ./" | \
+    tee /etc/apt/sources.list.d/mellanox.list && \
+    apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+        doca-all \
+        doca-sdk-common doca-sdk-dma doca-sdk-dpdk-bridge \
+        doca-sdk-eth doca-sdk-flow doca-sdk-rdma \
+        mlnx-dpdk mlnx-dpdk-dev
+
+# Install Ubuntu 24 specific DOCA packages
+RUN if [ "$OS" = "ubuntu24.04" ]; then \
+        apt-get install -y --no-install-recommends \
+            doca-sdk-gpunetio \
+            libdoca-sdk-gpunetio-dev; \
+    fi
+
+# Install AWS CLI
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && ./aws/install && rm -rf awscliv2.zip aws
+
+WORKDIR /workspace
+
+# Build and install etcd-cpp-apiv3
+RUN git clone https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \
+    cd etcd-cpp-apiv3 && \
+    sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-apiv3-config.cmake.in && \
+    sed -i '/^find_dependency(cpprestsdk)$/d' cmake/etcd-cpp-apiv3-config.cmake.in && \
+    mkdir build && cd build && \
+    cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local && \
+    make -j${NPROC} && \
+    make -j${NPROC} install
+
+# Build and install AWS SDK C++
+RUN git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.581 && \
+    mkdir aws_sdk_build && cd aws_sdk_build && \
+    cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local && \
+    make -j${NPROC} && \
+    make -j${NPROC} install
+
+# Install uv for Python virtual environment management
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+# Build and install UCX
+RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \
+    cd /usr/local/src && \
+    git clone https://github.com/openucx/ucx.git && \
+    cd ucx && \
+    git checkout $UCX_REF && \
+    ./autogen.sh && ./configure \
+        --prefix=/usr \
+        --with-rdmacm \
+        --with-verbs \
+        --with-mlx5-dv \
+        --with-dm \
+        --without-java \
+        --enable-optimizations \
+        --disable-logging \
+        --disable-debug \
+        --disable-assertions \
+        --disable-params-check \
+        --enable-mt && \
+    make -j${NPROC} && \
+    make -j${NPROC} install-strip && \
+    ldconfig
+
+# Install Rust toolchain
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.86.0
+
+RUN case "${ARCH}" in \
+        aarch64) RUSTARCH="aarch64-unknown-linux-gnu" ;; \
+        x86_64) RUSTARCH="x86_64-unknown-linux-gnu" ;; \
+    esac && \
+    wget --tries=3 --waitretry=5 \
+        "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init" \
+        "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init.sha256" && \
+    sha256sum -c rustup-init.sha256 && \
+    chmod +x rustup-init && \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${RUSTARCH} && \
+    rm rustup-init* && \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME
+
+# Final cleanup
+RUN rm -rf etcd-cpp-apiv3 aws-sdk-cpp aws_sdk_build /usr/local/src/ucx && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Set up environment variables
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH