CI: Add multi-target build infra

Author: Alexey-Rivkin

.ci/jenkins/lib/build-multi-target-matrix.yaml

@@ -0,0 +1,127 @@
+# NIXL PR Testing Matrix Configuration
+#
+# This file defines the build matrix for NIXL PR validation using the multi-target Docker system.
+# Primary purpose: Fast feedback on code changes via build + test execution.
+#
+# Key Components:
+# - Single NIXL Image Build: Uses multi-stage Dockerfile (base -> nixl) with layer caching
+# - Integrated Testing: Runs C++ and Python tests directly in the built environment
+# - CI-Demo Integration: Base image auto-builds only when Dockerfile changes
+#
+# PR Testing Benefits:
+# - Fast feedback: Base layer cached when only code changes
+# - Full rebuild: When infrastructure (Dockerfile) changes
+# - No registry operations: Focus on validation, not distribution
+# - Multi-OS/arch testing: Ensures compatibility across platforms
+#
+
+---
+job: nixl-ci-build-multi-target
+
+# Registry configuration for base image reusability
+registry_host: urm.nvidia.com
+registry_path: /sw-nbu-swx-nixl-docker-local/ci
+registry_auth: svc-nixl-artifactory-token
+
+# Build settings
+failFast: false
+timeout_minutes: 120
+
+# Infrastructure requirements
+kubernetes:
+  cloud: il-ipp-blossom-prod
+  namespace: swx-media
+  limits: "{memory: 16Gi, cpu: 8000m}"
+  requests: "{memory: 8Gi, cpu: 4000m}"
+
+# Container images: Base for reusability + NIXL for PR testing
+runs_on_dockers:
+  - { name: "podman-v5.0.2", url: "quay.io/podman/stable:v5.0.2", privileged: true, category: 'tool' }
+
+  # BASE IMAGE - Built & Pushed for Reusability
+  # Registry: urm.nvidia.com/sw-nbu-swx-nixl-docker-local/ci/{arch}/nixl-base:base-{os}-{arch}
+  # Contains: UCX, DOCA, AWS SDK, Rust toolchain, all dependencies
+  - { file: 'contrib/Dockerfile.multi-target', name: "nixl-base", tag: "base-${os}-${arch}", target: 'base', arch: '${arch}', build_args: 'OS=${os}', category: 'reusable' }
+
+  # NIXL TEST IMAGE - Uses base via multi-stage (PR testing only)
+  # Local only: Built from base + NIXL source for validation
+  - { file: 'contrib/Dockerfile.multi-target', name: "nixl-pr-test", tag: "test-${os}-${arch}", target: 'nixl', arch: '${arch}', build_args: 'OS=${os}', category: 'test' }
+
+# Build matrix - test both architectures and OS versions
+matrix:
+  axes:
+    arch:
+      - x86_64
+      - aarch64
+    os:
+      - ubuntu22.04
+      - ubuntu24.04
+
+# Environment configuration
+env:
+  NIXL_INSTALL_DIR: "/usr/local/nixl"
+  NPROC: "16"
+
+# Task naming for parallel execution
+taskName: "${os}/${arch}/${axis_index}"
+
+credentials:
+  - credentialsId: 'svc-nixl-artifactory-token'
+    usernameVariable: 'ARTIFACTORY_USERNAME'
+    passwordVariable: 'ARTIFACTORY_PASSWORD'
+
+# Build pipeline steps
+steps:
+  - name: Test NIXL with PR Changes
+    parallel: true
+    containerSelector: "{ name: 'nixl-pr-test' }"
+    run: |
+      echo "=== Testing NIXL with PR Changes ==="
+
+      # Set up environment
+      cd /workspace/nixl
+      export ARCH=$(uname -m)
+      [ "$ARCH" = "arm64" ] && ARCH="aarch64"
+
+      echo "=== Running C++ Tests ==="
+      if [ -f .gitlab/test_cpp.sh ]; then
+        .gitlab/test_cpp.sh ${NIXL_INSTALL_DIR}
+      else
+        echo 'C++ test script not found, skipping C++ tests'
+      fi
+
+      echo "=== Running Python Tests ==="
+      if [ -f .gitlab/test_python.sh ]; then
+        .gitlab/test_python.sh ${NIXL_INSTALL_DIR}
+      else
+        echo 'Python test script not found, skipping Python tests'
+      fi
+
+      echo "✅ All tests completed successfully - PR is ready!"
+
+# Pipeline completion
+pipeline_stop:
+  shell: action
+  module: groovy
+  run: |
+    def jobStatus = currentBuild.result ?: 'SUCCESS'
+    def statusColor = jobStatus == 'SUCCESS' ? 'green' : 'red'
+
+    echo "NIXL PR testing completed with status: ${jobStatus}"
+
+        if (params.MAIL_TO) {
+        def userName = currentBuild.rawBuild.getCause(hudson.model.Cause.UserIdCause)?.userName ?: 'schedule'
+
+        mail(
+            to: params.MAIL_TO,
+            subject: "NIXL PR Testing [${env.BUILD_NUMBER}] - ${jobStatus}",
+            mimeType: 'text/html',
+            body: """
+                <h3>NIXL PR Testing Results</h3>
+                <p><b>Status:</b> <span style="color: ${statusColor};">${jobStatus}</span></p>
+                <p><b>Build:</b> <a href='${env.BUILD_URL}'>#${env.BUILD_NUMBER}</a></p>
+                <p><b>Images:</b> Base image pushed for reuse, NIXL tested with PR changes</p>
+                <p><a href='${env.BUILD_URL}console'>View Console Output</a></p>
+            """
+        )
+    }

.ci/jenkins/pipeline/proj-jjb.yaml

@@ -316,6 +316,71 @@
               parent-credentials: true
       script-path: "{jjb_jenkinsfile}"
 
+# Template for multi-target build job using the new unified Dockerfile approach
+- job-template:
+    name: "{jjb_proj}-build-multi-target"
+    project-type: pipeline
+    folder: "{jjb_folder}"
+    disabled: false
+    properties:
+        - github:
+            url: "{jjb_gh_url}"
+        - build-discarder:
+            days-to-keep: 14
+            num-to-keep: 1000
+        - inject:
+            keep-system-variables: true
+            properties-content: |
+              jjb_proj={jjb_proj}-build-multi-target
+    description: >
+      <b>NIXL PR Testing Pipeline</b><br/>
+      • Fast feedback for code changes using multi-target Docker system<br/>
+      • Builds NIXL with PR changes (base layer cached when possible)<br/>
+      • Supports both Ubuntu 22.04 and 24.04 on x86_64/aarch64<br/>
+      • Runs comprehensive C++ and Python test suites<br/>
+      <br/>
+      <i>Do NOT edit this job through the Jenkins GUI — managed by Jenkins Job Builder.</i>
+    concurrent: true
+    sandbox: true
+    parameters:
+        - string:
+            name: "sha1"
+            default: "{jjb_branch}"
+            description: "Git commit/branch/tag to build (default: main)"
+        - string:
+            name: "githubData"
+            default: ""
+            description: "GitHub webhook data (automatically set)"
+        - string:
+            name: "conf_file"
+            default: ".ci/jenkins/lib/build-multi-target-matrix.yaml"
+            description: "Build matrix configuration file"
+        - string:
+            name: "MAIL_TO"
+            default: ""
+            description: "Email address for build notifications (optional)"
+        - string:
+            name: "DEBUG"
+            default: 0
+            description: "Debug level (0-9) for verbose output"
+    pipeline-scm:
+        scm:
+            - git:
+                url: "{jjb_git}"
+                credentials-id: 'swx-jenkins_ssh_key'
+                branches: ['$sha1']
+                shallow-clone: false
+                do-not-fetch-tags: false
+                refspec: "+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/* +refs/tags/*:refs/remotes/origin/tags/*"
+                browser: githubweb
+                browser-url: "{jjb_git}"
+                submodule:
+                    disable: false
+                    recursive: true
+                    tracking: true
+                    parent-credentials: true
+        script-path: "{jjb_jenkinsfile}"
+
 # Project definition that instantiates the job templates
 # This section defines the actual jobs that will be created
 - project:
@@ -327,7 +392,8 @@
     jjb_branch: 'main'            # Default branch
     jjb_gh_url: 'https://github.com/ai-dynamo/nixl'  # GitHub web URL
     jobs:
-        - "{jjb_proj}-dispatcher"  # Create dispatcher job
-        - "{jjb_proj}-build"       # Create build job
-        - "{jjb_proj}-build-container"  # Create container builder job
-        - "{jjb_proj}-test"        # Create test job
+        # - "{jjb_proj}-dispatcher"  # Create dispatcher job
+        # - "{jjb_proj}-build"       # Create build job
+        # - "{jjb_proj}-build-container"  # Create container builder job
+        - "{jjb_proj}-build-multi-target"  # Create multi-target build job
+        # - "{jjb_proj}-test"        # Create test job