feat: add fast variant of CI build and smoke test

.github/scripts/spur-dist-build.sh

@@ -2,13 +2,21 @@
 set -euo pipefail
 
 # Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), clones the repo at
-# the current SHA, and runs dist-build with a CI-scoped image name and tarball
-# path. The clone and tarball are left in place for spur-smoke-test.sh to use;
-# spur-smoke-test.sh owns the final cleanup.
+# the current SHA, and runs the requested dist-build target with a CI-scoped
+# image name and tarball path. The clone and tarball are left in place for
+# spur-smoke-test.sh to use; spur-smoke-test.sh owns the final cleanup.
 #
 # On failure, cleans up immediately so no stale state is left behind.
 
-SHA="${1:?usage: $0 <full-sha>}"
+SHA="${1:?usage: $0 <full-sha> [dist-build|dist-build-fast]}"
+AIC_DIST_BUILD_TARGET="${2:-dist-build}"
+case "${AIC_DIST_BUILD_TARGET}" in
+    dist-build | dist-build-fast) ;;
+    *)
+        echo "ERROR: unsupported build target: ${AIC_DIST_BUILD_TARGET}" >&2
+        exit 2
+        ;;
+esac
 SHORT="${SHA:0:7}"
 REPO="https://github.com/ROCm/rocm-aic.git"
 AIC_IMAGE="rocm-aic-ci-${SHORT}:latest"
@@ -21,6 +29,7 @@ ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
     SHA="${SHA}" \
     REPO="${REPO}" \
     AIC_IMAGE="${AIC_IMAGE}" \
+    AIC_DIST_BUILD_TARGET="${AIC_DIST_BUILD_TARGET}" \
     AIC_SHARED_NFS="${AIC_SHARED_NFS}" \
     AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
     SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" \
@@ -46,13 +55,13 @@ git checkout "${SHA}"
 
 mkdir -p "${TARBALL_DIR}"
 
-echo "=== Running dist-build (AIC_SPUR_CLUSTER=1, AIC_IMAGE=${AIC_IMAGE}) ==="
+echo "=== Running ${AIC_DIST_BUILD_TARGET} (AIC_SPUR_CLUSTER=1, AIC_IMAGE=${AIC_IMAGE}) ==="
 AIC_SPUR_CLUSTER=1 \
     AIC_IMAGE="${AIC_IMAGE}" \
     AIC_IMAGE_DIR="${TARBALL_DIR}" \
-    make dist-build
+    make "${AIC_DIST_BUILD_TARGET}"
 
-echo "=== dist-build complete — tarball in ${TARBALL_DIR} ==="
+echo "=== ${AIC_DIST_BUILD_TARGET} complete — tarball in ${TARBALL_DIR} ==="
 REMOTE
 
 echo "Build succeeded for ${SHORT}"

.github/scripts/spur-smoke-test.sh

@@ -1,13 +1,21 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and runs smoke-test
-# against the tarball produced by spur-dist-build.sh for the same SHA.
-# The clone and tarball are left in place for spur-tiny-test.sh (the next stage)
-# to use; spur-tiny-test.sh owns the final cleanup.  On failure, cleans up
-# immediately so no stale state is left behind.
+# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and
+# runs the requested smoke-test target against the tarball produced by
+# spur-dist-build.sh for the same SHA. The clone and tarball are left in place
+# for spur-tiny-test.sh (the next stage) to use; spur-tiny-test.sh owns the final
+# cleanup. On failure, cleans up immediately so no stale state is left behind.
 
-SHA="${1:?usage: $0 <full-sha>}"
+SHA="${1:?usage: $0 <full-sha> [smoke-test|smoke-test-fast]}"
+AIC_SMOKE_TEST_TARGET="${2:-smoke-test}"
+case "${AIC_SMOKE_TEST_TARGET}" in
+    smoke-test | smoke-test-fast) ;;
+    *)
+        echo "ERROR: unsupported smoke-test target: ${AIC_SMOKE_TEST_TARGET}" >&2
+        exit 2
+        ;;
+esac
 SHORT="${SHA:0:7}"
 AIC_IMAGE="rocm-aic-ci-${SHORT}:latest"
 AIC_SPUR_HOST="${AIC_SPUR_HOST:?AIC_SPUR_HOST must be set (e.g. via GitHub repo variable)}"
@@ -20,6 +28,7 @@ ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
     SHA="${SHA}" \
     REPO="${REPO}" \
     AIC_IMAGE="${AIC_IMAGE}" \
+    AIC_SMOKE_TEST_TARGET="${AIC_SMOKE_TEST_TARGET}" \
     AIC_SHARED_NFS="${AIC_SHARED_NFS}" \
     AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
     SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" \
@@ -49,13 +58,13 @@ if [[ ! -d "${WORKDIR}" || "${ACTUAL_SHA}" != "${SHA}" ]]; then
     git -C "${WORKDIR}" checkout "${SHA}"
 fi
 
-echo "=== Running smoke-test (AIC_IMAGE=${AIC_IMAGE}) ==="
+echo "=== Running ${AIC_SMOKE_TEST_TARGET} (AIC_IMAGE=${AIC_IMAGE}) ==="
 AIC_SPUR_CLUSTER=1 \
     AIC_IMAGE="${AIC_IMAGE}" \
     AIC_IMAGE_DIR="${TARBALL_DIR}" \
-    make -C "${WORKDIR}" smoke-test
+    make -C "${WORKDIR}" "${AIC_SMOKE_TEST_TARGET}"
 
-echo "=== smoke-test complete ==="
+echo "=== ${AIC_SMOKE_TEST_TARGET} complete ==="
 REMOTE
 
 echo "Smoke test passed for ${SHORT}"

.github/workflows/aic-amd-dist-build.yml → .github/workflows/aic-amd-dist-build-fast.yml

@@ -98,8 +98,8 @@ jobs:
               description: 'Building image on the SPUR head node...',
             });
 
-      - name: Build image on the SPUR head node
-        run: bash /usr/local/lib/aic-ci/spur-dist-build.sh "${{ needs.authorize.outputs.sha }}"
+      - name: Build fast image on the SPUR head node
+        run: bash /usr/local/lib/aic-ci/spur-dist-build.sh "${{ needs.authorize.outputs.sha }}" dist-build-fast
 
       - name: Set commit status on success
         if: success() && needs.authorize.outputs.pr_number != ''
@@ -171,8 +171,8 @@ jobs:
               description: 'Running smoke test on the SPUR head node...',
             });
 
-      - name: Run smoke test on the SPUR head node
-        run: bash /usr/local/lib/aic-ci/spur-smoke-test.sh "${{ needs.authorize.outputs.sha }}"
+      - name: Run fast smoke test on the SPUR head node
+        run: bash /usr/local/lib/aic-ci/spur-smoke-test.sh "${{ needs.authorize.outputs.sha }}" smoke-test-fast
 
       - name: Set commit status on success
         if: success() && needs.authorize.outputs.pr_number != ''

.slurm/run-build-distribute.sh

@@ -160,6 +160,7 @@ AIC_DAY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
 # Infinity-Storage hardware -- if hipFile fails to build for them, narrow this
 # to the CDNA set "gfx90a;gfx942;gfx950".  Override via AIC_ROCM_ARCH.
 AIC_ROCM_ARCH="${AIC_ROCM_ARCH:-gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1150;gfx1151;gfx1200;gfx1201}"
+AIC_UCX_FAST="${AIC_UCX_FAST:-}"
 AIC_IMAGE="${AIC_IMAGE:-rocm-aic:latest}"
 AIC_IMAGE_DIR="${AIC_IMAGE_DIR:-/scratch/${USER}/images}"
 AIC_SPUR_CLUSTER="${AIC_SPUR_CLUSTER:-0}"
@@ -538,6 +539,7 @@ tmp="${tarball}.partial.\$\$"
 set +o pipefail
 docker buildx build --builder ${AIC_BUILDX_BUILDER} --progress=plain --output type=docker,dest=- \
     --build-arg ROCM_ARCH="${AIC_ROCM_ARCH}" \
+    --build-arg AIC_UCX_FAST="${AIC_UCX_FAST}" \
     ${_secret_arg} \
     ${_cache_args} \
     -f "${AIC_DAY_DIR}/docker/Dockerfile" \
@@ -567,6 +569,7 @@ cd "${AIC_DAY_DIR}"
 ${_builder_setup}
 ${_build_program} \
     --build-arg ROCM_ARCH="${AIC_ROCM_ARCH}" \
+    --build-arg AIC_UCX_FAST="${AIC_UCX_FAST}" \
     ${_secret_arg} \
     ${_cache_args} \
     -f "${AIC_DAY_DIR}/docker/Dockerfile" \

Makefile

@@ -128,6 +128,8 @@ DIST := $(CURDIR)/.slurm/run-build-distribute.sh
 AIC_CI_LIB_DIR    ?= /usr/local/lib/aic-ci
 AIC_CI_SCRIPT_DIR := $(CURDIR)/.github/scripts
 
+AIC_FAST_ARCH ?= gfx950
+
 # ---- SPUR cluster overrides ------------------------------------------------
 # When AIC_SPUR_CLUSTER=1, default storage paths to AIC_SHARED_NFS (the NFS
 # volume shared across all SPUR compute nodes) instead of /scratch (not present
@@ -185,7 +187,8 @@ EXPORT_TARBALL ?= $(CURDIR)/$(EXPORT_PREFIX)-$(_GEN_DATE)-$(_GIT_SHORT_REV)$(_GI
 .PHONY: help ensure-compose build up up-batch up-gds-l1 up-gds-l1-batch down logs logs-lmcache logs-vllm \
         ps shell-lmcache shell-vllm restart-vllm restart-lmcache cliff plot venv \
         monitoring-up monitoring-down monitoring-logs monitoring-build-exporters \
-        dist-build dist-build-exporters dist-push smoke-test tiny-test install-ci-scripts cliff-submit cliff-short \
+        dist-build dist-build-fast dist-build-exporters dist-push smoke-test smoke-test-fast \
+        tiny-test install-ci-scripts cliff-submit cliff-short \
         cliff-long-64k cliff-long-128k \
         export _check_hf_token _prep_dirs _check_gds_slab
 
@@ -219,11 +222,13 @@ help:
 	@echo "Distribute / cliff targets (Slurm; wrap .slurm/ scripts + sbatch):"
 	@echo "  (dist-build/dist-build-exporters/smoke-test submit via sbatch and log to logs/<job-id>/)"
 	@echo "  make dist-build        Build image (+ fabric exporters) on a Slurm build node, save tarballs"
+	@echo "  make dist-build-fast   Single-arch dev build (AIC_FAST_ARCH=$(AIC_FAST_ARCH), no exporters) -- faster iteration"
 	@echo "  make dist-build-exporters  Build ONLY the nvme/rdma exporter images (no main rebuild)"
 	@echo "  make dist-push         Tag + push the built image (needs AIC_PUSH_REF)"
 	@echo "  make smoke-test        Load + smoke-test the image on a GPU+NVMe node"
 	@echo "                         (also sanity-checks exporters + writes a Prometheus TSDB"
 	@echo "                          to logs/<job-id>/prometheus; AIC_SMOKE_EXPORTERS=0 skips)"
+	@echo "  make smoke-test-fast   Smoke-test the single-arch dev image (AIC_FAST_ARCH=$(AIC_FAST_ARCH))"
 	@echo "  make tiny-test         End-to-end serve check (MP stack + tiny model, one completion)"
 	@echo "  make install-ci-scripts  Deploy .github/scripts/spur-*.sh to $(AIC_CI_LIB_DIR) (sudo if needed)"
 	@echo "  make cliff-submit      sbatch the full 3-arm cliff sweep -> logs/<job-id>/"
@@ -433,6 +438,11 @@ dist-build:                    # Build image (+ fabric exporters) on a Slurm bui
 	@[ "$(AIC_BUILD_EXPORTERS)" = "0" ] || "$(DIST)" build-exporters \
 	    || echo "WARNING: fabric-exporter build failed (optional; main image is built). Retry on a node with Docker Hub access, or set AIC_BUILD_EXPORTERS=0."
 
+dist-build-fast:               # Single-arch (AIC_FAST_ARCH) dev build -- fast edit-build loop
+	@# A cut-down version of dist-build.
+	@$(MAKE) --no-print-directory dist-build \
+	    AIC_ROCM_ARCH='$(AIC_FAST_ARCH)' AIC_BUILD_EXPORTERS=0 AIC_UCX_FAST=1
+
 dist-build-exporters:          # Build ONLY the fabric exporters (no main-image rebuild)
 	@# Rebuild just the nvme/rdma exporter images -- e.g. after `make dist-build`
 	@# succeeded for the main image but the exporter step failed for lack of Docker
@@ -446,6 +456,12 @@ dist-push:                     # Tag + push the built image to a registry (needs
 smoke-test:                    # Load + smoke-test the image on a GPU+NVMe node
 	"$(DIST)" test
 
+smoke-test-fast:               # Smoke-test the single-arch (AIC_FAST_ARCH) dev image
+	@# A cut-down version of smoke-test.
+	@# Must pin the SAME AIC_ROCM_ARCH as dist-build-fast.
+	@$(MAKE) --no-print-directory smoke-test \
+	    AIC_ROCM_ARCH='$(AIC_FAST_ARCH)' AIC_SMOKE_EXPORTERS=0
+
 tiny-test:                     # End-to-end serve check: MP stack + a tiny model, one real completion
 	@# Stages Qwen/Qwen2.5-0.5B-Instruct, brings up the compose MP stack (nvme arm),
 	@# waits for the endpoint, and asserts one non-empty chat completion.  Fast

docker/Dockerfile

@@ -246,6 +246,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     mkdir -p build && cd build && \
     cmake -DCMAKE_HIP_PLATFORM=amd -DAIS_INSTALL_TOOLS=ON \
           -DBUILD_TESTING=OFF \
+          -DAIS_INSTALL_EXAMPLES=OFF \
           -DGPU_TARGETS="${ROCM_ARCH}" \
           -DCMAKE_HIP_ARCHITECTURES="${ROCM_ARCH}" .. && \
     cmake --build . -j"${BUILD_JOBS:-$(nproc)}" && \
@@ -268,6 +269,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # ----- 6. NIXL (ai-dynamo/nixl upstream + nixl-rocm-ais-mt.patch) ------------
 # Clone upstream ai-dynamo/nixl, then apply nixl-rocm-ais-mt.patch to add ROCm/HIP
 # detection and the AIS_MT plugin tree.
+ARG AIC_UCX_FAST=
 COPY docker/scripts /tmp/nixl-scripts
 COPY patches/nixl/ /tmp/nixl-patches/
 # Apply every patch under patches/nixl/ (lexical order) to the fresh checkout.
@@ -292,6 +294,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         NIXL_INSTALL_PREFIX=/opt/nixl \
         NIXL_BUILD_WHEEL=1 NIXL_WHEEL_DIR=/wheels \
         BUILD_JOBS="${BUILD_JOBS}" \
+        AIC_UCX_FAST="${AIC_UCX_FAST}" \
         UV_SYSTEM_CERTS=1 \
         /tmp/nixl-scripts/build-nixl.sh
 

docker/docker-compose.yml

@@ -81,6 +81,7 @@ services:
       args:
         ROCM_ARCH: ${ROCM_ARCH}
         BUILD_JOBS: ${BUILD_JOBS:-}
+        AIC_UCX_FAST: ${AIC_UCX_FAST:-}
     image: ${IMAGE_NAME:-rocm-aic}
     container_name: aic-lmcache
     # Cross-container HIP IPC (LMCacheMPConnector registering vLLM's KV tensors) on

docker/scripts/build-nixl.sh

@@ -63,17 +63,34 @@ cd ucx-src
 rm -rf build
 mkdir build
 cd build
-../configure \
-	--prefix="${UCX_PREFIX}" \
-	--enable-shared \
-	--disable-static \
-	--disable-doxygen-doc \
-	--enable-optimizations \
-	--enable-devel-headers \
-	--with-rocm="${ROCM_PATH}" \
-	--with-verbs \
-	--with-dm \
+# UCX configure flags.
+UCX_CONFIGURE_FLAGS=(
+	--prefix="${UCX_PREFIX}"
+	--enable-shared
+	--disable-static
+	--disable-doxygen-doc
+	--enable-devel-headers
+	--with-rocm="${ROCM_PATH}"
+	--with-verbs
+	--with-dm
 	--enable-mt
+)
+# Improve compile time for dist-build-fast target.
+if [[ "${AIC_UCX_FAST:-0}" == "1" ]]; then
+	UCX_CONFIGURE_FLAGS+=(
+		--disable-logging
+		--disable-debug
+		--disable-assertions
+		--disable-params-check
+		--without-knem
+		--without-xpmem
+		--without-ugni
+		--without-java
+	)
+else
+	UCX_CONFIGURE_FLAGS+=(--enable-optimizations)
+fi
+../configure "${UCX_CONFIGURE_FLAGS[@]}"
 make -j"${BUILD_JOBS}"
 make install
 ldconfig
@@ -89,6 +106,9 @@ MESON_EXTRA=(
 	"-Drocm_ais_path=${AIS_PATH}"
 	"-Ducx_path=${UCX_PREFIX}"
 	"-Ddisable_gds_backend=true"
+	"-Dbuild_tests=false" # disable unused components
+	"-Dbuild_examples=false"
+	"-Ddisable_plugins=OBJ,AZURE_BLOB"
 	"--prefix=${NIXL_INSTALL_PREFIX}"
 )