Merge branch 'main' into rust_test_sync_manager

Author: evgeny-leksikov

CODEOWNERS

@@ -31,6 +31,6 @@ CODEOWNERS @ai-dynamo/Devops @ai-dynamo/nixl-maintainers
 /benchmark @aranadive @ovidiusm @brminich
 
 # Libfabric Plugins
-/src/plugins/libfabric* @yexiang-aws @akkart-aws @fengjica
-/src/utils/libfabric @yexiang-aws @akkart-aws @fengjica
-/test/unit/utils/libfabric @yexiang-aws @akkart-aws @fengjica
+/src/plugins/libfabric* @amitrad-aws @yexiang-aws @akkart-aws @fengjica
+/src/utils/libfabric @amitrad-aws @yexiang-aws @akkart-aws @fengjica
+/test/unit/utils/libfabric @amitrad-aws @yexiang-aws @akkart-aws @fengjica

README.md

@@ -144,6 +144,8 @@ Common build options:
 - `disable_gds_backend`: Disable GDS backend (default: false)
 - `cudapath_inc`, `cudapath_lib`: Custom CUDA paths
 - `static_plugins`: Comma-separated list of plugins to build statically
+- `enable_plugins`: Comma-separated list of plugins to build (e.g. `-Denable_plugins=UCX,POSIX`). Cannot be used with `disable_plugins`.
+- `disable_plugins`: Comma-separated list of plugins to exclude (e.g. `-Ddisable_plugins=GDS`). Cannot be used with `enable_plugins`.
 
 ### Building Documentation
 

benchmark/kvbench/pyproject.toml

@@ -18,7 +18,7 @@ name = "NIXLKVBench"
 version = "0.2.0"
 description = "Benchmarking utility for testing KVCache transfers in large language models (LLMs) using the NIXL runtime."
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "click==8.1.7",
     "pytest==7.4.4",

benchmark/nixlbench/contrib/Dockerfile

@@ -189,6 +189,8 @@ RUN uv pip install --upgrade meson meson-python pybind11 patchelf pyYAML click t
 RUN CUDA_SHORT_VERSION=cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .) && \
     FLAGS="--index-url https://download.pytorch.org/whl/$CUDA_SHORT_VERSION" && \
     uv pip install $FLAGS torch torchvision torchaudio
+# Upgrade setuptools to latest version for compatibility with PEP 639 (license format)
+RUN uv pip install --upgrade 'setuptools>=80.9.0'
 
 COPY --from=nixl . /workspace/nixl
 COPY --from=nixlbench . /workspace/nixlbench

contrib/Dockerfile

@@ -24,8 +24,11 @@ ARG OS=${OS:-ubuntu24}
 ARG ARCH="x86_64"
 ARG DEFAULT_PYTHON_VERSION="3.12"
 ARG UCX_REF="v1.20.x"
+ARG BUILD_NIXL_EP="false"
+ARG RDMA_CORE_PREFIX="/usr"
 ARG UCX_PREFIX="/usr"
 ARG UCX_PLUGIN_DIR="$UCX_PREFIX/lib/ucx"
+ARG DOCA_PREFIX="/opt/mellanox/doca"
 ARG NIXL_PREFIX="/usr/local/nixl"
 ARG NIXL_PLUGIN_DIR="$NIXL_PREFIX/lib/$ARCH-linux-gnu/plugins"
 ARG NPROC
@@ -122,8 +125,17 @@ RUN rm -rf /opt/hpcx/ucx
 
 RUN cd /usr/local/src && \
      git clone https://github.com/openucx/ucx.git && \
-     cd ucx && 			     \
-     git checkout $UCX_REF &&        \
+     cd ucx &&                       \
+     if [ "$BUILD_NIXL_EP" = "true" ]; then \
+         echo "=== BUILD_NIXL_EP=true: Using latest UCX master branch (ignoring UCX_REF=$UCX_REF) ===" && \
+         UCX_COMMIT=$(git rev-parse --short HEAD) && \
+         echo "Using UCX commit: $UCX_COMMIT" && \
+         EXPERIMENTAL_API_PARAM="--enable-experimental-api"; \
+     else                            \
+         echo "=== Using UCX_REF=$UCX_REF ===" && \
+         git checkout $UCX_REF &&    \
+         EXPERIMENTAL_API_PARAM="";   \
+     fi &&                           \
      ./autogen.sh && \
      ./contrib/configure-release-mt  \
          --prefix=$UCX_PREFIX        \
@@ -133,6 +145,7 @@ RUN cd /usr/local/src && \
          --enable-optimizations      \
          --enable-cma                \
          --enable-devel-headers      \
+         $EXPERIMENTAL_API_PARAM      \
          --with-cuda=/usr/local/cuda \
          --with-verbs                \
          --with-dm                   \
@@ -191,19 +204,30 @@ RUN uv pip install --upgrade meson meson-python pybind11 patchelf pyYAML click t
 # Install PyTorch
 RUN export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \
     uv pip install torch torchvision torchaudio
+# Upgrade setuptools to latest version for compatibility with PEP 639 (license format)
+RUN uv pip install --upgrade 'setuptools>=80.9.0'
 
 WORKDIR /workspace/nixl
 COPY . /workspace/nixl
 
-ENV LD_LIBRARY_PATH=/usr/local/lib:$LIBFABRIC_INSTALL_PATH/lib:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH=/usr/local/lib:$LIBFABRIC_INSTALL_PATH/lib:$RDMA_CORE_PREFIX/lib:$LD_LIBRARY_PATH
 
 # Install pybind11 via apt
 RUN apt-get update && apt-get install -y --no-install-recommends pybind11-dev
 
+# Set PKG_CONFIG_PATH for NIXL EP dependencies (rdma-core, UCX, DOCA)
+ENV PKG_CONFIG_PATH=$RDMA_CORE_PREFIX/lib/$ARCH-linux-gnu/pkgconfig:$UCX_PREFIX/lib/pkgconfig:$DOCA_PREFIX/lib/$ARCH-linux-gnu/pkgconfig:$PKG_CONFIG_PATH
+
 ENV NIXL_PREFIX=$NIXL_PREFIX
 RUN rm -rf build && \
     mkdir build && \
-    meson setup -Dlibfabric_path=$LIBFABRIC_INSTALL_PATH build/ --prefix=$NIXL_PREFIX --buildtype=$BUILD_TYPE && \
+    if [ "$BUILD_NIXL_EP" = "true" ]; then \
+        echo "=== BUILD_NIXL_EP=true: Building NIXL with NIXL EP support ===" && \
+        NIXL_EP_FLAG="-Dbuild_nixl_ep=true"; \
+    else \
+        NIXL_EP_FLAG=""; \
+    fi && \
+    meson setup -Ducx_path=$UCX_PREFIX -Dlibfabric_path=$LIBFABRIC_INSTALL_PATH $NIXL_EP_FLAG build/ --prefix=$NIXL_PREFIX --buildtype=$BUILD_TYPE && \
     cd build && \
     ninja && \
     ninja install
@@ -212,6 +236,10 @@ RUN echo "$NIXL_PREFIX/lib/$ARCH-linux-gnu" > /etc/ld.so.conf.d/nixl.conf && \
     echo "$NIXL_PLUGIN_DIR" >> /etc/ld.so.conf.d/nixl.conf && \
     ldconfig
 
+# Set environment variables for NIXL EP
+ENV NIXL_PLUGIN_DIR=$NIXL_PLUGIN_DIR
+ENV PYTHONPATH=/workspace/nixl/build/examples/device/ep
+
 RUN cd src/bindings/rust && cargo build --release --locked
 
 # Build wheel using the build-wheel.sh script for better UCX plugin bundling and library management

contrib/Dockerfile.manylinux

@@ -18,7 +18,7 @@ ARG BASE_IMAGE
 ARG BASE_IMAGE_TAG
 FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG}
 
-ARG DEFAULT_PYTHON_VERSION="3.12"
+ARG DEFAULT_PYTHON_VERSION="3.14"
 ARG ARCH="x86_64"
 ARG UCX_REF="v1.20.x"
 ARG LIBFABRIC_VERSION="v1.21.0"
@@ -244,6 +244,8 @@ RUN uv pip install --upgrade meson meson-python pybind11 patchelf pyYAML click s
 # Install PyTorch
 RUN export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \
     uv pip install torch torchvision torchaudio
+# Upgrade setuptools to latest version for compatibility with PEP 639 (license format)
+RUN uv pip install --upgrade 'setuptools>=80.9.0'
 
 COPY . /workspace/nixl
 WORKDIR /workspace/nixl
@@ -267,10 +269,11 @@ RUN echo "/usr/local/nixl/lib/$ARCH-linux-gnu" > /etc/ld.so.conf.d/nixl.conf &&
 
 # Create the wheel
 # No need to specifically add path to libcuda.so here, meson finds the stubs and links them
-ARG WHL_PYTHON_VERSIONS="3.9,3.10,3.11,3.12,3.13"
+ARG WHL_PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14"
 ARG WHL_PLATFORM="manylinux_2_28_$ARCH"
 RUN IFS=',' read -ra PYTHON_VERSIONS <<< "$WHL_PYTHON_VERSIONS" && \
     export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \
+    export UV_INDEX_STRATEGY=unsafe-best-match && \
     mkdir -p dist && \
     for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do \
         export PATH=$VIRTUAL_ENV/bin:$PATH && \

contrib/build-container.sh

@@ -36,6 +36,7 @@ WHL_BASE=manylinux_2_39
 WHL_PLATFORM=${WHL_BASE}_${ARCH}
 WHL_PYTHON_VERSIONS="3.12"
 UCX_REF=${UCX_REF:-v1.20.x}
+BUILD_NIXL_EP="false"
 OS="ubuntu24"
 NPROC=${NPROC:-$(nproc)}
 if [ "$CI" = "true" ]; then
@@ -122,6 +123,9 @@ get_options() {
             # Master branch (v1.20) also containing EFA SRD support
             UCX_REF=9d2b88a1f67faf9876f267658bd077b379b8bb76
             ;;
+        --build-nixl-ep)
+            BUILD_NIXL_EP=true
+            ;;
         --arch)
             if [ "$2" ]; then
                 ARCH=$2
@@ -169,7 +173,13 @@ show_build_options() {
     echo "Container arch: ${ARCH}"
     echo "Python Versions for wheel build: ${WHL_PYTHON_VERSIONS}"
     echo "Wheel Platform: ${WHL_PLATFORM}"
-    echo "UCX Ref: ${UCX_REF}"
+    if [ "$BUILD_NIXL_EP" = "true" ]; then
+        echo "UCX Ref: master (latest) - BUILD_NIXL_EP enabled"
+        echo "NIXL EP: Enabled"
+    else
+        echo "UCX Ref: ${UCX_REF}"
+        echo "NIXL EP: Disabled"
+    fi
     echo "Build Type: ${BUILD_TYPE}"
 }
 
@@ -184,6 +194,7 @@ show_help() {
     echo "  [--tag tag for image]"
     echo "  [--python-versions python versions to build for, comma separated]"
     echo "  [--ucx-upstream use ucx master branch]"
+    echo "  [--build-nixl-ep build NIXL with NIXL EP support (uses latest UCX master)]"
     echo "  [--arch [x86_64|aarch64] to select target architecture]"
     echo "  [--dockerfile path to a dockerfile to use]"
     exit 0
@@ -210,6 +221,7 @@ BUILD_ARGS+=" --build-arg WHL_PYTHON_VERSIONS=$WHL_PYTHON_VERSIONS"
 BUILD_ARGS+=" --build-arg WHL_PLATFORM=$WHL_PLATFORM"
 BUILD_ARGS+=" --build-arg ARCH=$ARCH"
 BUILD_ARGS+=" --build-arg UCX_REF=$UCX_REF"
+BUILD_ARGS+=" --build-arg BUILD_NIXL_EP=$BUILD_NIXL_EP"
 BUILD_ARGS+=" --build-arg NPROC=$NPROC"
 BUILD_ARGS+=" --build-arg OS=$OS"
 BUILD_ARGS+=" --build-arg BUILD_TYPE=$BUILD_TYPE"

meson.build

@@ -21,6 +21,56 @@ project('nixl', 'CPP', version: '0.8.0',
     meson_version: '>= 0.64.0'
 )
 
+enable_plugins_opt = get_option('enable_plugins')
+disable_plugins_opt = get_option('disable_plugins')
+is_explicit_enable = enable_plugins_opt != ''
+
+if enable_plugins_opt != '' and disable_plugins_opt != ''
+    error('Cannot specify both enable_plugins and disable_plugins options')
+endif
+
+all_plugins = ['UCX', 'LIBFABRIC', 'POSIX', 'OBJ', 'GDS', 'GDS_MT', 'MOONCAKE', 'HF3FS', 'GUSLI', 'GPUNETIO', 'UCCL']
+
+enabled_plugins = {}
+
+if enable_plugins_opt != ''
+    enabled_plugins_list = []
+    foreach p : enable_plugins_opt.split(',')
+        p_stripped = p.strip()
+        if p_stripped not in all_plugins
+            error('Requested plugin "' + p_stripped + '" is not available. Available plugins are: ' + ', '.join(all_plugins))
+        endif
+        enabled_plugins_list += [p_stripped]
+    endforeach
+
+    foreach plugin : all_plugins
+        enabled_plugins += {plugin: plugin in enabled_plugins_list}
+    endforeach
+
+    message('Building only selected plugins: ' + enable_plugins_opt)
+elif disable_plugins_opt != ''
+    disabled_plugins_list = []
+    foreach p : disable_plugins_opt.split(',')
+        p_stripped = p.strip()
+        if p_stripped not in all_plugins
+            error('Cannot exclude unknown plugin "' + p_stripped + '". Available plugins are: ' + ', '.join(all_plugins))
+        endif
+        disabled_plugins_list += [p_stripped]
+    endforeach
+
+    foreach plugin : all_plugins
+        enabled_plugins += {plugin: plugin not in disabled_plugins_list}
+    endforeach
+
+    message('Building all plugins except: ' + disable_plugins_opt)
+else
+    foreach plugin : all_plugins
+        enabled_plugins += {plugin: true}
+    endforeach
+
+    message('Building all available plugins')
+endif
+
 # set up some global vars for compiler, platform, configuration, etc.
 cpp = meson.get_compiler('cpp')
 fs = import('fs')
@@ -110,7 +160,7 @@ if cuda_dep.found()
     nvcc_flags_link += ['-gencode=arch=compute_90,code=sm_90']
     add_project_link_arguments(nvcc_flags_link, language: 'cuda')
     message('nvcc version: ' + nvcc.version())
-    if nvcc.version().version_compare('>=12.8') and nvcc.version().version_compare('<13.0')
+    if nvcc.version().version_compare('>=12.8')
         doca_gpunetio_dep = dependency('doca-gpunetio', required : false)
     else
         warning('GPUNETIO plugin not supported in CUDA version: ' + nvcc.version())
@@ -262,7 +312,7 @@ if get_option('buildtype') == 'debug'
     run_command('truncate', '-s 0', plugfile, check: true)
 endif
 
-nixl_inc_dirs = include_directories('src/api/cpp', 'src/api/cpp/backend', 'src/infra', 'src/core')
+nixl_inc_dirs = include_directories('src/api/cpp', 'src/api/cpp/backend', 'src/infra', 'src/core', 'src/core/telemetry')
 nixl_gpu_inc_dirs = include_directories('src/api/gpu/ucx')
 plugins_inc_dirs = include_directories('src/plugins')
 utils_inc_dirs = include_directories('src/utils')

meson_options.txt

@@ -25,6 +25,8 @@ option('cudapath_inc', type: 'string', value: '', description: 'Include path for
 option('cudapath_lib', type: 'string', value: '', description: 'Library path for CUDA')
 option('cudapath_stub', type: 'string', value: '', description: 'Extra Stub path for CUDA')
 option('static_plugins', type: 'string', value: '', description: 'Plugins to be built-in, comma-separated')
+option('enable_plugins', type: 'string', value: '', description: 'Comma-separated list of plugins to build. Default (empty) builds all available plugins.')
+option('disable_plugins', type: 'string', value: '', description: 'Comma-separated list of plugins to exclude from build. Cannot be used with enable_plugins.')
 option('build_docs', type: 'boolean', value: false, description: 'Build Doxygen documentation')
 option('log_level', type: 'combo', choices: ['trace', 'debug', 'info', 'warning', 'error', 'fatal', 'auto'], value: 'auto', description: 'Log Level (auto: auto-detect based on build type: trace for debug builds, info for release builds)')
 option('rust', type: 'boolean', value: false, description: 'Build Rust bindings')

pyproject.toml

@@ -14,18 +14,19 @@
 # limitations under the License.
 
 [build-system]
-requires = ["meson-python", "pybind11", "patchelf", "pyyaml", "types-PyYAML", "pytest", "build", "setuptools"]
+requires = ["meson-python", "pybind11", "patchelf", "pyyaml", "types-PyYAML", "pytest", "build", "setuptools>=80.9.0"]
 build-backend = "mesonpy"
 
 [project]
-name = 'nixl-cu12'
-version = '0.8.0'
-description = 'NIXL Python API'
-readme = 'README.md'
-license = {file = 'LICENSE'}
-requires-python = '>=3.9'
+name = "nixl-cu12"
+version = "0.8.0"
+description = "NIXL Python API"
+readme = "README.md"
+license = "Apache-2.0"
+license-files = ["LICENSE"]
+requires-python = ">=3.10"
 authors = [
-  {name = 'NIXL Developers', email = 'nixl-developers@nvidia.com'}
+  {name = "NIXL Developers", email = "nixl-developers@nvidia.com"}
 ]
 
 dependencies = ["torch", "numpy"]
@@ -34,4 +35,4 @@ dependencies = ["torch", "numpy"]
 profile = "black"
 
 [tool.meson-python.args]
-setup = ['-Dinstall_headers=false']
+setup = ["-Dinstall_headers=false"]