|
| 1 | +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# SPDX-License-Identifier: Apache-2.0 |
| 3 | +# |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +# you may not use this file except in compliance with the License. |
| 6 | +# You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | + |
| 16 | +ARG BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base" |
| 17 | +ARG BASE_IMAGE_TAG="25.03-cuda12.8-devel-ubuntu24.04" |
| 18 | + |
| 19 | +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} as nixl-base |
| 20 | + |
| 21 | +# Set default OS if not provided |
| 22 | +ARG OS="ubuntu24" |
| 23 | +ARG ARCH="x86_64" |
| 24 | +ARG DEFAULT_PYTHON_VERSION="3.12" |
| 25 | +ARG UCX_REF="v1.19.x" |
| 26 | +ARG UCX_PREFIX="/usr" |
| 27 | +ARG UCX_PLUGIN_DIR="$UCX_PREFIX/lib/ucx" |
| 28 | +ARG NIXL_PREFIX="/usr/local/nixl" |
| 29 | +ARG NIXL_PLUGIN_DIR="$NIXL_PREFIX/lib/$ARCH-linux-gnu/plugins" |
| 30 | +ARG NPROC="10" |
| 31 | + |
| 32 | +RUN PY_VER=$(if [ "${OS}" = "ubuntu22" ]; then echo "3.10"; elif [ "${OS}" = "ubuntu24" ]; then echo "3.12"; else echo "3.12"; fi) && \ |
| 33 | + apt-get update -y && \ |
| 34 | + apt-get install -y ubuntu-keyring && \ |
| 35 | + apt-get update -y && \ |
| 36 | + DEBIAN_FRONTEND=noninteractive apt-get -y install \ |
| 37 | + automake \ |
| 38 | + autotools-dev \ |
| 39 | + build-essential \ |
| 40 | + clang \ |
| 41 | + cmake \ |
| 42 | + etcd-client \ |
| 43 | + etcd-server \ |
| 44 | + flex \ |
| 45 | + libaio-dev \ |
| 46 | + libclang-dev \ |
| 47 | + libcpprest-dev \ |
| 48 | + libcurl4-openssl-dev \ |
| 49 | + libgflags-dev \ |
| 50 | + libgtest-dev \ |
| 51 | + libgrpc++-dev \ |
| 52 | + libgrpc-dev \ |
| 53 | + libprotobuf-dev \ |
| 54 | + libssl-dev \ |
| 55 | + libtool \ |
| 56 | + liburing-dev \ |
| 57 | + libz-dev \ |
| 58 | + ninja-build \ |
| 59 | + pybind11-dev \ |
| 60 | + protobuf-compiler-grpc \ |
| 61 | + python${PY_VER}-dev \ |
| 62 | + python3-pip \ |
| 63 | + uuid-dev \ |
| 64 | + zlib1g-dev |
| 65 | + |
| 66 | +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \ |
| 67 | + --reinstall libibverbs-dev rdma-core ibverbs-utils libibumad-dev \ |
| 68 | + libnuma-dev librdmacm-dev ibverbs-providers |
| 69 | + |
| 70 | +WORKDIR /workspace |
| 71 | +RUN git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ |
| 72 | + cd etcd-cpp-apiv3 && \ |
| 73 | + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ |
| 74 | + mkdir build && cd build && \ |
| 75 | + cmake .. -DBUILD_ETCD_CORE_ONLY=ON -DCMAKE_BUILD_TYPE=Release && make -j${NPROC:-$(nproc)} && make install |
| 76 | + |
| 77 | +RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.581 && \ |
| 78 | + mkdir aws_sdk_build && cd aws_sdk_build && \ |
| 79 | + cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local && \ |
| 80 | + make -j${NPROC:-$(nproc)} && make install |
| 81 | + |
| 82 | +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ |
| 83 | + |
| 84 | +ENV RUSTUP_HOME=/usr/local/rustup \ |
| 85 | + CARGO_HOME=/usr/local/cargo \ |
| 86 | + PATH=/usr/local/cargo/bin:$PATH \ |
| 87 | + RUST_VERSION=1.86.0 \ |
| 88 | + RUSTARCH=${ARCH}-unknown-linux-gnu |
| 89 | + |
| 90 | +# Download rustup-init and its checksum for the target architecture |
| 91 | +RUN wget --tries=3 --waitretry=5 \ |
| 92 | + "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init" \ |
| 93 | + "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init.sha256" && \ |
| 94 | + sha256sum -c rustup-init.sha256 && \ |
| 95 | + chmod +x rustup-init && \ |
| 96 | + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${RUSTARCH} && \ |
| 97 | + rm rustup-init* && \ |
| 98 | + chmod -R a+w $RUSTUP_HOME $CARGO_HOME |
| 99 | + |
| 100 | +# Add Mellanox repository and install packages |
| 101 | +RUN ARCH_SUFFIX=$(if [ "${ARCH}" = "aarch64" ]; then echo "arm64-sbsa"; else echo "${ARCH}"; fi) && \ |
| 102 | + # Map OS names to Mellanox repository format |
| 103 | + MELLANOX_OS=$(if [ "${OS}" = "ubuntu22" ]; then echo "ubuntu22.04"; elif [ "${OS}" = "ubuntu24" ]; then echo "ubuntu24.04"; else echo "${OS}"; fi) && \ |
| 104 | + export PKG_CONFIG_PATH="/opt/mellanox/doca/lib/${ARCH_SUFFIX}-linux-gnu/pkgconfig:/opt/mellanox/dpdk/lib/${ARCH_SUFFIX}-linux-gnu/pkgconfig:$PKG_CONFIG_PATH" && \ |
| 105 | + curl -fsSL https://linux.mellanox.com/public/repo/doca/3.0.0/${MELLANOX_OS}/${ARCH_SUFFIX}/GPG-KEY-Mellanox.pub | \ |
| 106 | + gpg --dearmor | tee /usr/share/keyrings/mellanox-archive-keyring.gpg && \ |
| 107 | + echo "deb [signed-by=/usr/share/keyrings/mellanox-archive-keyring.gpg] https://linux.mellanox.com/public/repo/doca/3.0.0/${MELLANOX_OS}/${ARCH_SUFFIX} ./" | \ |
| 108 | + tee /etc/apt/sources.list.d/mellanox.list && \ |
| 109 | + apt-get update -y && \ |
| 110 | + apt-get install -y --no-install-recommends \ |
| 111 | + mlnx-dpdk mlnx-dpdk-dev \ |
| 112 | + doca-sdk-common doca-sdk-dma doca-sdk-dpdk-bridge \ |
| 113 | + doca-sdk-eth doca-sdk-flow doca-sdk-rdma doca-all |
| 114 | + |
| 115 | +RUN if [ "$OS" = "ubuntu24" ]; then \ |
| 116 | + apt-get install -y --no-install-recommends \ |
| 117 | + doca-sdk-gpunetio libdoca-sdk-gpunetio-dev; \ |
| 118 | + fi |
| 119 | + |
| 120 | +RUN rm -rf /usr/lib/ucx /opt/hpcx/ucx && \ |
| 121 | + cd /usr/local/src && \ |
| 122 | + git clone https://github.com/openucx/ucx.git && \ |
| 123 | + cd ucx && \ |
| 124 | + git checkout $UCX_REF && \ |
| 125 | + ./autogen.sh && ./configure \ |
| 126 | + --prefix=$UCX_PREFIX \ |
| 127 | + --enable-shared \ |
| 128 | + --disable-static \ |
| 129 | + --disable-doxygen-doc \ |
| 130 | + --enable-optimizations \ |
| 131 | + --enable-cma \ |
| 132 | + --enable-devel-headers \ |
| 133 | + --with-cuda=/usr/local/cuda \ |
| 134 | + --with-verbs \ |
| 135 | + --with-dm \ |
| 136 | + --with-gdrcopy=/usr/local \ |
| 137 | + --with-efa \ |
| 138 | + --enable-mt && \ |
| 139 | + make -j${NPROC:-$(nproc)} && \ |
| 140 | + make -j${NPROC:-$(nproc)} install-strip && \ |
| 141 | + ldconfig |
| 142 | + |
| 143 | +RUN if [ "$OS" = "ubuntu22" ]; then \ |
| 144 | + pip --no-cache-dir install --upgrade pip "meson>=0.64.0"; \ |
| 145 | + fi |
| 146 | + |
| 147 | +FROM nixl-base as nixl-build |
| 148 | + |
| 149 | +WORKDIR /workspace/nixl |
| 150 | +COPY . /workspace/nixl |
| 151 | + |
| 152 | +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH |
| 153 | + |
| 154 | +ENV VIRTUAL_ENV=/workspace/nixl/.venv |
| 155 | +RUN rm -rf $VIRTUAL_ENV && uv venv $VIRTUAL_ENV --python $DEFAULT_PYTHON_VERSION && \ |
| 156 | + # pybind11 pip install needed for ubuntu 22.04 |
| 157 | + uv pip install --upgrade "meson>=0.64.0" pybind11 patchelf |
| 158 | + |
| 159 | +# Install pybind11 via apt |
| 160 | +# RUN apt-get update && apt-get install -y --no-install-recommends pybind11-dev |
| 161 | + |
| 162 | +ENV NIXL_PREFIX=$NIXL_PREFIX |
| 163 | +RUN rm -rf build && \ |
| 164 | + mkdir build && \ |
| 165 | + uv run meson setup build/ --prefix=$NIXL_PREFIX && \ |
| 166 | + cd build && \ |
| 167 | + ninja && \ |
| 168 | + ninja install |
| 169 | + |
| 170 | +RUN echo "$NIXL_PREFIX/lib/$ARCH-linux-gnu" > /etc/ld.so.conf.d/nixl.conf && \ |
| 171 | + echo "$NIXL_PLUGIN_DIR" >> /etc/ld.so.conf.d/nixl.conf && \ |
| 172 | + ldconfig |
| 173 | + |
| 174 | +RUN cd src/bindings/rust && cargo build --release --locked |
| 175 | + |
| 176 | +# Build wheel using the build-wheel.sh script for better UCX plugin bundling and library management |
| 177 | +RUN ./contrib/build-wheel.sh \ |
| 178 | + --python-version $DEFAULT_PYTHON_VERSION \ |
| 179 | + --platform manylinux_2_39_$ARCH \ |
| 180 | + --ucx-plugins-dir $UCX_PLUGIN_DIR \ |
| 181 | + --nixl-plugins-dir $NIXL_PLUGIN_DIR \ |
| 182 | + --output-dir /workspace/nixl/dist |
| 183 | + |
| 184 | +RUN uv pip install dist/nixl-*cp${DEFAULT_PYTHON_VERSION//./}*.whl |
0 commit comments