Merge branch 'main' into pylint-ci

Author: roiedanino

.ci/dockerfiles/Dockerfile.gpu_test

@@ -19,6 +19,7 @@
 #   _LOGIN: Username (default: svc-nixl)
 #   _GROUP: Group name (default: hardware)
 #   _HOME: Home directory path (default: /home/svc-nixl)
+#   WORKSPACE: Workspace directory path
 #
 
 ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:25.02-py3
@@ -31,6 +32,7 @@ ARG _GID=30
 ARG _LOGIN=svc-nixl
 ARG _GROUP=hardware
 ARG _HOME=/home/$_LOGIN
+ARG WORKSPACE
 
 # Labels for documentation
 LABEL maintainer="NVIDIA NIXL Team"
@@ -57,6 +59,9 @@ RUN mkdir -p /etc/sudoers.d && \
     chmod 440 /etc/sudoers.d/${_LOGIN} && \
     chown root:root /etc/sudoers.d/${_LOGIN}
 
+# Copy workspace into container (workaround for files disappearing from workspace)
+COPY --chown="${_UID}":"${_GID}" . ${WORKSPACE}
+
 # Set working directory
 WORKDIR ${_HOME}
 

.ci/jenkins/lib/test-matrix.yaml

@@ -65,7 +65,7 @@ steps:
   - name: Build GPU Test Environment
     parallel: false
     run: |
-      docker build -t "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}" -f .ci/dockerfiles/Dockerfile.gpu_test --build-arg BASE_IMAGE=${image} .
+      docker build -t "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}" -f .ci/dockerfiles/Dockerfile.gpu_test --build-arg BASE_IMAGE=${image} --build-arg WORKSPACE=${WORKSPACE} .
     onfail: docker image rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
 
   - name: Run GPU Test Environment
@@ -79,8 +79,6 @@ steps:
         --gpus all \
         --device=/dev/infiniband \
         --device=/dev/gdrdrv \
-        -v ${WORKSPACE}:${WORKSPACE} \
-        -w ${WORKSPACE} \
         "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
     onfail: |
       docker rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"

.ci/jenkins/pipeline/proj-jjb.yaml

@@ -252,7 +252,7 @@
       - string:
           name: "NIXL_VERSION"
           default: "{jjb_branch}"
-          description: "NIXL version to use (tag like 0.4.1, branch name, or commit hash)"
+          description: "NIXL version to use (tag like 0.5.0, branch name, or commit hash)"
       - string:
           name: "UCX_VERSION"
           default: "v1.19.x"

.ci/scripts/common.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Common functions for CI scripts
+#
+
+#
+# Set initial port number for client/server applications to be updated with
+# function below
+#
+tcp_port_range=1000
+min_port_number=10500
+max_port_number=65535
+
+# GITLAB CI
+if [ -n "$CI_CONCURRENT_ID" ]; then
+    nixl_concurrent_id=$CI_CONCURRENT_ID
+# Jenkins CI
+elif [ -n "$EXECUTOR_NUMBER" ]; then
+    nixl_concurrent_id=$EXECUTOR_NUMBER
+else
+    # Fallback to random number if both CI_CONCURRENT_ID and EXECUTOR_NUMBER are not set
+    nixl_concurrent_id=$((RANDOM % $(((max_port_number - min_port_number) / tcp_port_range))))
+fi
+
+echo nixl_concurrent_id="$nixl_concurrent_id"
+
+# First half of the port range is used for shell script tests
+tcp_port_min=$((min_port_number + nixl_concurrent_id * tcp_port_range))
+tcp_port_max=$((tcp_port_min + tcp_port_range / 2))
+
+get_next_tcp_port() {
+    local port_file="/tmp/nixl_tcp_port_${nixl_concurrent_id}"
+
+    if [ ! -f "$port_file" ]; then
+        echo "$tcp_port_min" > "$port_file"
+    fi
+
+    local current_port
+    current_port=$(cat "$port_file")
+    local next_port=$((current_port + 1))
+
+    # Check if the port is already in use
+    while ss -tuln | grep -q :$next_port; do
+        next_port=$((next_port + 1))
+    done
+
+    if [ "$next_port" -ge "$tcp_port_max" ]; then
+        next_port="$tcp_port_min"
+    fi
+
+    echo "$next_port" > "$port_file"
+
+    echo "$next_port"
+}
+
+# Second half of the port range is used for gtest
+gtest_offset=$((tcp_port_range / 2))
+# shellcheck disable=SC2034
+min_gtest_port=$((tcp_port_min + gtest_offset))
+# shellcheck disable=SC2034
+max_gtest_port=$((tcp_port_max + gtest_offset))

.gitlab/test_cpp.sh

@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+# shellcheck disable=SC1091
+. "$(dirname "$0")/../.ci/scripts/common.sh"
+
 set -e
 set -x
 TEXT_YELLOW="\033[1;33m"
@@ -57,8 +61,13 @@ ibv_devinfo || true
 uname -a || true
 
 echo "==== Running ETCD server ===="
-export NIXL_ETCD_ENDPOINTS="http://127.0.0.1:2379"
-etcd --listen-client-urls ${NIXL_ETCD_ENDPOINTS} --advertise-client-urls ${NIXL_ETCD_ENDPOINTS} &
+etcd_port=$(get_next_tcp_port)
+etcd_peer_port=$(get_next_tcp_port)
+export NIXL_ETCD_ENDPOINTS="http://127.0.0.1:${etcd_port}"
+export NIXL_ETCD_PEER_URLS="http://127.0.0.1:${etcd_peer_port}"
+etcd --listen-client-urls ${NIXL_ETCD_ENDPOINTS} --advertise-client-urls ${NIXL_ETCD_ENDPOINTS} \
+     --listen-peer-urls ${NIXL_ETCD_PEER_URLS} --initial-advertise-peer-urls ${NIXL_ETCD_PEER_URLS} \
+     --initial-cluster default=${NIXL_ETCD_PEER_URLS} &
 sleep 5
 
 echo "==== Running C++ tests ===="
@@ -76,13 +85,17 @@ cd ${INSTALL_DIR}
 
 ./bin/ucx_backend_multi
 ./bin/serdes_test
-./bin/gtest
+
+# shellcheck disable=SC2154
+./bin/gtest --min-tcp-port="$min_gtest_port" --max-tcp-port="$max_gtest_port"
 ./bin/test_plugin
 
 # Run NIXL client-server test
-./bin/nixl_test target 127.0.0.1 1234&
+nixl_test_port=$(get_next_tcp_port)
+
+./bin/nixl_test target 127.0.0.1 "$nixl_test_port"&
 sleep 1
-./bin/nixl_test initiator 127.0.0.1 1234
+./bin/nixl_test initiator 127.0.0.1 "$nixl_test_port"
 
 echo "${TEXT_YELLOW}==== Disabled tests==="
 echo "./bin/md_streamer disabled"

.gitlab/test_plugins.sh

@@ -14,6 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1091
+. "$(dirname "$0")/../.ci/scripts/common.sh"
+
 set -e
 set -x
 
@@ -45,4 +48,5 @@ uname -a || true
 
 echo "==== Running Plugins Gtest tests ===="
 cd ${INSTALL_DIR}
-./bin/plugins_gtest
+# shellcheck disable=SC2154
+./bin/plugins_gtest --min-tcp-port="$min_gtest_port" --max-tcp-port="$max_gtest_port"

.gitlab/test_python.sh

@@ -14,6 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# shellcheck disable=SC1091
+. "$(dirname "$0")/../.ci/scripts/common.sh"
+
 set -e
 set -x
 
@@ -52,8 +55,13 @@ pip3 install --break-system-packages pytest-timeout
 pip3 install --break-system-packages zmq
 
 echo "==== Running ETCD server ===="
-export NIXL_ETCD_ENDPOINTS="http://127.0.0.1:2379"
-etcd --listen-client-urls ${NIXL_ETCD_ENDPOINTS} --advertise-client-urls ${NIXL_ETCD_ENDPOINTS} &
+etcd_port=$(get_next_tcp_port)
+etcd_peer_port=$(get_next_tcp_port)
+export NIXL_ETCD_ENDPOINTS="http://127.0.0.1:${etcd_port}"
+export NIXL_ETCD_PEER_URLS="http://127.0.0.1:${etcd_peer_port}"
+etcd --listen-client-urls ${NIXL_ETCD_ENDPOINTS} --advertise-client-urls ${NIXL_ETCD_ENDPOINTS} \
+     --listen-peer-urls ${NIXL_ETCD_PEER_URLS} --initial-advertise-peer-urls ${NIXL_ETCD_PEER_URLS} \
+     --initial-cluster default=${NIXL_ETCD_PEER_URLS} &
 sleep 5
 
 echo "==== Running python tests ===="
@@ -66,10 +74,12 @@ python3 test/python/prep_xfer_perf.py list
 python3 test/python/prep_xfer_perf.py array
 
 echo "==== Running python examples ===="
+blocking_send_recv_port=$(get_next_tcp_port)
+
 cd examples/python
-python3 blocking_send_recv_example.py --mode="target" --ip=127.0.0.1 --port=1234&
+python3 blocking_send_recv_example.py --mode="target" --ip=127.0.0.1 --port="$blocking_send_recv_port"&
 sleep 5
-python3 blocking_send_recv_example.py --mode="initiator" --ip=127.0.0.1 --port=1234
+python3 blocking_send_recv_example.py --mode="initiator" --ip=127.0.0.1 --port="$blocking_send_recv_port"
 
 python3 query_mem_example.py
 

CODEOWNERS

@@ -14,17 +14,17 @@ CODEOWNERS @ai-dynamo/Devops @ai-dynamo/nixl-maintainers
 /SECURITY.md @ai-dynamo/Devops
 
 # Bindings
-/src/bindings/rust @roiedanino @gleon99
+/src/bindings/rust @roiedanino @gleon99 @mkhazraee
 
 # UCX Plugins
 /src/plugins/ucx* @brminich @yosefe @gleon99
 /src/utils/ucx @brminich @yosefe @gleon99
 
 # Storage Plugins
-/src/plugins/posix @w1ldptr @barneuman @etoledano
-/src/plugins/hf3fs @w1ldptr @barneuman @etoledano
-/src/plugins/cuda_gds @w1ldptr @barneuman @etoledano
-/src/plugins/obj @w1ldptr @barneuman @etoledano
+/src/plugins/posix @w1ldptr @barneuman @etoledano vvenkates27
+/src/plugins/hf3fs @w1ldptr @barneuman @etoledano vvenkates27
+/src/plugins/cuda_gds @w1ldptr @barneuman @etoledano vvenkates27
+/src/plugins/obj @w1ldptr @barneuman @etoledano vvenkates27
 
 # Benchmarks
-/benchmark/nixlbench @aranadive @ovidiusm @brminich
+/benchmark @aranadive @ovidiusm @brminich

Cargo.lock

@@ -125,3 +125,24 @@ Example:
 ```
 
 The workers automatically coordinate ranks through ETCD as they connect.
+
+### Benchmarking the OBJ Plugin
+
+For OBJ plugin benchmarking run etcd-server and a single nixlbench instance.
+
+Example:
+```bash
+AWS_ACCESS_KEY_ID=<access_key> AWS_SECRET_ACCESS_KEY=<secret_key> AWS_DEFAULT_REGION=<region> /tmp/nixlbench/nixlbench --etcd-endpoints http://<etcd-server>:2379 --backend OBJ --obj_bucket_name <bucket_name>
+```
+Access key, secret access key, default region and bucket name are mandatory fields.
+Use your own valid credentials.
+
+Transfer times are higher than local storage so it is advisable to use less iterations than the default values.
+
+Example:
+```bash
+AWS_ACCESS_KEY_ID=<access_key> AWS_SECRET_ACCESS_KEY=<secret_key> AWS_DEFAULT_REGION=<region> /tmp/nixlbench/nixlbench --etcd-endpoints http://etcd-server:2379 --backend OBJ --obj_bucket_name nixl-ci-test --warmup_iter 32 --num_iter 32 --large_blk_iter_ftr 2
+```
+
+The default benchmark command tests write. To test read ops add the flag: `--op_type READ`.
+To test tranfer data validity add the flag: `--check_consistency true`.