Stress test

Signed-off-by: Ovidiu Mara <[email protected]>

Author: ovidiusm

.ci/jenkins/lib/test-matrix.yaml

@@ -38,7 +38,7 @@ env:
   UCX_VERSION: v1.19.0
   NPROC: "16"
   # Manual timeout - ci-demo doesn't handle docker exec
-  TEST_TIMEOUT: 30
+  TEST_TIMEOUT: 90
 
 steps:
   - name: Get Environment Info
@@ -98,15 +98,21 @@ steps:
   - name: Test CPP
     parallel: false
     run: |
-      timeout ${TEST_TIMEOUT}m docker exec -w ${WORKSPACE} "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}" /bin/bash -c ".gitlab/test_cpp.sh ${INSTALL_DIR}"
+      for i in $(seq 1 10)
+      do
+        timeout ${TEST_TIMEOUT}m docker exec -w ${WORKSPACE} "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}" /bin/bash -c ".gitlab/test_cpp.sh ${INSTALL_DIR}"
+      done
     onfail: |
       docker rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
       docker image rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
 
   - name: Test Python
     parallel: false
     run: |
-      timeout ${TEST_TIMEOUT}m docker exec -w ${WORKSPACE} "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}" /bin/bash -c ".gitlab/test_python.sh ${INSTALL_DIR}"
+      for i in $(seq 1 10)
+      do
+        timeout ${TEST_TIMEOUT}m docker exec -w ${WORKSPACE} "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}" /bin/bash -c ".gitlab/test_python.sh ${INSTALL_DIR}"
+      done
     onfail: |
       docker rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
       docker image rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
@@ -122,7 +128,10 @@ steps:
   - name: Test Rust
     parallel: false
     run: |
-      timeout ${TEST_TIMEOUT}m docker exec -w ${WORKSPACE} "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}" /bin/bash -c ".gitlab/test_rust.sh ${INSTALL_DIR}"
+      for i in $(seq 1 10)
+      do
+        timeout ${TEST_TIMEOUT}m docker exec -w ${WORKSPACE} "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}" /bin/bash -c ".gitlab/test_rust.sh ${INSTALL_DIR}"
+      done
     always: |
       docker rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
       docker image rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"

.gitlab/build.sh

@@ -159,6 +159,11 @@ ninja -C nixl_build && ninja -C nixl_build install
 # TODO(kapila): Copy the nixl.pc file to the install directory if needed.
 # cp ${BUILD_DIR}/nixl.pc ${INSTALL_DIR}/lib/pkgconfig/nixl.pc
 
+pip3 install --break-system-packages .
+pip3 install --break-system-packages pytest
+pip3 install --break-system-packages pytest-timeout
+pip3 install --break-system-packages zmq
+
 cd benchmark/nixlbench
 meson setup nixlbench_build -Dnixl_path=${INSTALL_DIR} -Dprefix=${INSTALL_DIR}
 ninja -C nixlbench_build && ninja -C nixlbench_build install

.gitlab/test_cpp.sh

@@ -30,10 +30,6 @@ else
     SUDO=""
 fi
 
-$SUDO apt-get update
-$SUDO apt-get -qq install -y libaio-dev
-
-
 # Parse commandline arguments with first argument being the install directory.
 INSTALL_DIR=$1
 
@@ -101,8 +97,10 @@ kill -s SIGINT $telePID
 nixl_test_port=$(get_next_tcp_port)
 
 ./bin/nixl_test target 127.0.0.1 "$nixl_test_port"&
+target_pid=$!
 sleep 1
 ./bin/nixl_test initiator 127.0.0.1 "$nixl_test_port"
+wait $target_pid
 
 echo "${TEXT_YELLOW}==== Disabled tests==="
 echo "./bin/md_streamer disabled"

.gitlab/test_python.sh

@@ -35,8 +35,6 @@ else
     SUDO=""
 fi
 
-$SUDO apt-get -qq install liburing-dev
-
 ARCH=$(uname -m)
 [ "$ARCH" = "arm64" ] && ARCH="aarch64"
 
@@ -51,11 +49,6 @@ export NIXL_PREFIX=${INSTALL_DIR}
 # Raise exceptions for logging errors
 export NIXL_DEBUG_LOGGING=yes
 
-pip3 install --break-system-packages .
-pip3 install --break-system-packages pytest
-pip3 install --break-system-packages pytest-timeout
-pip3 install --break-system-packages zmq
-
 echo "==== Running ETCD server ===="
 etcd_port=$(get_next_tcp_port)
 etcd_peer_port=$(get_next_tcp_port)
@@ -83,9 +76,11 @@ blocking_send_recv_port=$(get_next_tcp_port)
 mkdir -p /tmp/telemetry_test
 
 python3 blocking_send_recv_example.py --mode="target" --ip=127.0.0.1 --port="$blocking_send_recv_port"&
+target_pid=$!
 sleep 5
 NIXL_TELEMETRY_ENABLE=y NIXL_TELEMETRY_DIR=/tmp/telemetry_test \
 python3 blocking_send_recv_example.py --mode="initiator" --ip=127.0.0.1 --port="$blocking_send_recv_port"
+wait $target_pid
 
 python3 telemetry_reader.py --telemetry_path /tmp/telemetry_test/initiator &
 telePID=$!

test/nixl/nixl_test.cpp

@@ -105,11 +105,13 @@ static void targetThread(nixlAgent &agent, nixl_opt_args_t *extra_params, int th
         if (!rc)
             std::this_thread::sleep_for(std::chrono::milliseconds(10));
     }
-    if (!rc)
+    if (!rc) {
         std::cerr << "Thread " << thread_id << " UCX Transfer failed, buffers are different\n";
-    else
+        exit(-1);
+    } else {
         std::cout << "Thread " << thread_id << " Transfer completed and Buffers match with Initiator\n"
                   << "Thread " << thread_id << " UCX Transfer Success!!!\n";
+    }
 
     std::cout << "Thread " << thread_id << " Cleanup..\n";
     agent.deregisterMem(dram_for_ucx, extra_params);