Skip to content

Commit 43ffb37

Browse files
roiedaninoroiedanino
authored
.CI/SCRIPTS: Avoid tcp port collisions, unify duplicated code in CI test scripts (#568)
* .CI/SCRIPTS: Avoid tcp port collisions, unify duplicated code in CI test scripts Signed-off-by: Roie Danino <[email protected]> * .CI/SCRIPTS: fixed common.sh macros definitions Signed-off-by: Roie Danino <[email protected]> * .CI/SCRIPTS: get_next_server_port returns the port, refactoring Signed-off-by: Roie Danino <[email protected]> * .CI/SCRIPTS: fixed identations Signed-off-by: Roie Danino <[email protected]> * .CI/SCRIPTS: using env variables from jenkins and gitlab ci Signed-off-by: Roie Danino <[email protected]> * CONTRIB: trying to pass gitlab CI_CONCURRENT_ID env to test scripts Signed-off-by: Roie Danino <[email protected]> * CONTRIB: reverted some changes Signed-off-by: Roie Danino <[email protected]> --------- Signed-off-by: Roie Danino <[email protected]>
1 parent 54294d2 commit 43ffb37

File tree

10 files changed

+234
-35
lines changed

10 files changed

+234
-35
lines changed

‎.ci/scripts/common.sh

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/bin/bash
2+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
#
18+
# Common functions for CI scripts
19+
#
20+
21+
#
22+
# Set initial port number for client/server applications to be updated with
23+
# function below
24+
#
25+
tcp_port_range=1000
26+
min_port_number=10500
27+
max_port_number=65535
28+
29+
# GITLAB CI
30+
if [ -n "$CI_CONCURRENT_ID" ]; then
31+
nixl_concurrent_id=$CI_CONCURRENT_ID
32+
# Jenkins CI
33+
elif [ -n "$EXECUTOR_NUMBER" ]; then
34+
nixl_concurrent_id=$EXECUTOR_NUMBER
35+
else
36+
# Fallback to random number if both CI_CONCURRENT_ID and EXECUTOR_NUMBER are not set
37+
nixl_concurrent_id=$((RANDOM % $(((max_port_number - min_port_number) / tcp_port_range))))
38+
fi
39+
40+
echo nixl_concurrent_id="$nixl_concurrent_id"
41+
42+
# First half of the port range is used for shell script tests
43+
tcp_port_min=$((min_port_number + nixl_concurrent_id * tcp_port_range))
44+
tcp_port_max=$((tcp_port_min + tcp_port_range / 2))
45+
46+
get_next_tcp_port() {
47+
local port_file="/tmp/nixl_tcp_port_${nixl_concurrent_id}"
48+
49+
if [ ! -f "$port_file" ]; then
50+
echo "$tcp_port_min" > "$port_file"
51+
fi
52+
53+
local current_port
54+
current_port=$(cat "$port_file")
55+
local next_port=$((current_port + 1))
56+
57+
# Check if the port is already in use
58+
while ss -tuln | grep -q :$next_port; do
59+
next_port=$((next_port + 1))
60+
done
61+
62+
if [ "$next_port" -ge "$tcp_port_max" ]; then
63+
next_port="$tcp_port_min"
64+
fi
65+
66+
echo "$next_port" > "$port_file"
67+
68+
echo "$next_port"
69+
}
70+
71+
# Second half of the port range is used for gtest
72+
gtest_offset=$((tcp_port_range / 2))
73+
# shellcheck disable=SC2034
74+
min_gtest_port=$((tcp_port_min + gtest_offset))
75+
# shellcheck disable=SC2034
76+
max_gtest_port=$((tcp_port_max + gtest_offset))

‎.gitlab/test_cpp.sh

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17+
18+
# shellcheck disable=SC1091
19+
. "$(dirname "$0")/../.ci/scripts/common.sh"
20+
1721
set -e
1822
set -x
1923
TEXT_YELLOW="\033[1;33m"
@@ -57,8 +61,13 @@ ibv_devinfo || true
5761
uname -a || true
5862

5963
echo "==== Running ETCD server ===="
60-
export NIXL_ETCD_ENDPOINTS="http://127.0.0.1:2379"
61-
etcd --listen-client-urls ${NIXL_ETCD_ENDPOINTS} --advertise-client-urls ${NIXL_ETCD_ENDPOINTS} &
64+
etcd_port=$(get_next_tcp_port)
65+
etcd_peer_port=$(get_next_tcp_port)
66+
export NIXL_ETCD_ENDPOINTS="http://127.0.0.1:${etcd_port}"
67+
export NIXL_ETCD_PEER_URLS="http://127.0.0.1:${etcd_peer_port}"
68+
etcd --listen-client-urls ${NIXL_ETCD_ENDPOINTS} --advertise-client-urls ${NIXL_ETCD_ENDPOINTS} \
69+
--listen-peer-urls ${NIXL_ETCD_PEER_URLS} --initial-advertise-peer-urls ${NIXL_ETCD_PEER_URLS} \
70+
--initial-cluster default=${NIXL_ETCD_PEER_URLS} &
6271
sleep 5
6372

6473
echo "==== Running C++ tests ===="
@@ -76,13 +85,17 @@ cd ${INSTALL_DIR}
7685

7786
./bin/ucx_backend_multi
7887
./bin/serdes_test
79-
./bin/gtest
88+
89+
# shellcheck disable=SC2154
90+
./bin/gtest --min-tcp-port="$min_gtest_port" --max-tcp-port="$max_gtest_port"
8091
./bin/test_plugin
8192

8293
# Run NIXL client-server test
83-
./bin/nixl_test target 127.0.0.1 1234&
94+
nixl_test_port=$(get_next_tcp_port)
95+
96+
./bin/nixl_test target 127.0.0.1 "$nixl_test_port"&
8497
sleep 1
85-
./bin/nixl_test initiator 127.0.0.1 1234
98+
./bin/nixl_test initiator 127.0.0.1 "$nixl_test_port"
8699

87100
echo "${TEXT_YELLOW}==== Disabled tests==="
88101
echo "./bin/md_streamer disabled"

‎.gitlab/test_plugins.sh

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17+
# shellcheck disable=SC1091
18+
. "$(dirname "$0")/../.ci/scripts/common.sh"
19+
1720
set -e
1821
set -x
1922

@@ -45,4 +48,5 @@ uname -a || true
4548

4649
echo "==== Running Plugins Gtest tests ===="
4750
cd ${INSTALL_DIR}
48-
./bin/plugins_gtest
51+
# shellcheck disable=SC2154
52+
./bin/plugins_gtest --min-tcp-port="$min_gtest_port" --max-tcp-port="$max_gtest_port"

‎.gitlab/test_python.sh

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17+
# shellcheck disable=SC1091
18+
. "$(dirname "$0")/../.ci/scripts/common.sh"
19+
1720
set -e
1821
set -x
1922

@@ -51,8 +54,13 @@ pip3 install --break-system-packages pytest-timeout
5154
pip3 install --break-system-packages zmq
5255

5356
echo "==== Running ETCD server ===="
54-
export NIXL_ETCD_ENDPOINTS="http://127.0.0.1:2379"
55-
etcd --listen-client-urls ${NIXL_ETCD_ENDPOINTS} --advertise-client-urls ${NIXL_ETCD_ENDPOINTS} &
57+
etcd_port=$(get_next_tcp_port)
58+
etcd_peer_port=$(get_next_tcp_port)
59+
export NIXL_ETCD_ENDPOINTS="http://127.0.0.1:${etcd_port}"
60+
export NIXL_ETCD_PEER_URLS="http://127.0.0.1:${etcd_peer_port}"
61+
etcd --listen-client-urls ${NIXL_ETCD_ENDPOINTS} --advertise-client-urls ${NIXL_ETCD_ENDPOINTS} \
62+
--listen-peer-urls ${NIXL_ETCD_PEER_URLS} --initial-advertise-peer-urls ${NIXL_ETCD_PEER_URLS} \
63+
--initial-cluster default=${NIXL_ETCD_PEER_URLS} &
5664
sleep 5
5765

5866
echo "==== Running python tests ===="
@@ -65,10 +73,12 @@ python3 test/python/prep_xfer_perf.py list
6573
python3 test/python/prep_xfer_perf.py array
6674

6775
echo "==== Running python examples ===="
76+
blocking_send_recv_port=$(get_next_tcp_port)
77+
6878
cd examples/python
69-
python3 blocking_send_recv_example.py --mode="target" --ip=127.0.0.1 --port=1234&
79+
python3 blocking_send_recv_example.py --mode="target" --ip=127.0.0.1 --port="$blocking_send_recv_port"&
7080
sleep 5
71-
python3 blocking_send_recv_example.py --mode="initiator" --ip=127.0.0.1 --port=1234
81+
python3 blocking_send_recv_example.py --mode="initiator" --ip=127.0.0.1 --port="$blocking_send_recv_port"
7282

7383
python3 query_mem_example.py
7484

‎test/gtest/common.cpp

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,14 @@
1919
#include <iostream>
2020
#include <iomanip>
2121
#include <cassert>
22+
#include <cstring>
23+
#include <memory>
2224
#include <stack>
2325
#include <optional>
26+
#include <netinet/in.h>
27+
#include <sys/socket.h>
28+
#include <unistd.h>
29+
#include <random>
2430

2531
namespace gtest {
2632

@@ -72,4 +78,54 @@ ScopedEnv::Variable::~Variable()
7278
}
7379
}
7480

81+
PortAllocator &
82+
PortAllocator::instance() {
83+
static PortAllocator _instance;
84+
return _instance;
85+
}
86+
87+
void
88+
PortAllocator::set_min_port(uint16_t min_port) {
89+
_min_port = min_port;
90+
_port = _min_port;
91+
}
92+
93+
void
94+
PortAllocator::set_max_port(uint16_t max_port) {
95+
_max_port = max_port;
96+
}
97+
98+
bool
99+
PortAllocator::is_port_available(uint16_t port) {
100+
struct sockaddr_in addr = {
101+
.sin_family = AF_INET, .sin_port = htons(port), .sin_addr = {.s_addr = INADDR_ANY}};
102+
103+
const auto sock_fd = socket(AF_INET, SOCK_STREAM, 0);
104+
const auto ret = bind(sock_fd, (struct sockaddr *)&addr, sizeof(addr));
105+
close(sock_fd);
106+
return ret == 0;
107+
}
108+
109+
uint16_t
110+
PortAllocator::next_tcp_port() {
111+
PortAllocator &instance = PortAllocator::instance();
112+
std::lock_guard<std::mutex> lock(instance._mutex);
113+
const int port_range = instance._max_port - instance._min_port;
114+
115+
for (int scanned = 0; scanned < port_range; scanned++) {
116+
if (is_port_available(instance._port)) {
117+
return instance._port++;
118+
}
119+
120+
instance._port++;
121+
122+
if (instance._port >= instance._max_port) {
123+
instance._port = instance._min_port;
124+
}
125+
}
126+
127+
throw std::runtime_error("No port available in range: " + std::to_string(instance._min_port) +
128+
" - " + std::to_string(instance._max_port));
129+
}
130+
75131
} // namespace gtest

‎test/gtest/common.h

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,11 @@
2020
#include <iostream>
2121
#include <iomanip>
2222
#include <cassert>
23+
#include <cstdint>
24+
#include <memory>
2325
#include <stack>
2426
#include <optional>
27+
#include <mutex>
2528

2629
namespace gtest {
2730
constexpr const char *
@@ -63,6 +66,39 @@ class ScopedEnv {
6366
std::stack<Variable> m_vars;
6467
};
6568

69+
class PortAllocator {
70+
public:
71+
static constexpr uint16_t MIN_PORT = 10500;
72+
static constexpr uint16_t MAX_PORT = 65535;
73+
74+
private:
75+
PortAllocator() = default;
76+
~PortAllocator() = default;
77+
PortAllocator(const PortAllocator &other) = delete;
78+
void
79+
operator=(const PortAllocator &) = delete;
80+
81+
public:
82+
static uint16_t
83+
next_tcp_port();
84+
static PortAllocator &
85+
instance();
86+
87+
void
88+
set_min_port(uint16_t min_port);
89+
void
90+
set_max_port(uint16_t max_port);
91+
92+
private:
93+
static bool
94+
is_port_available(uint16_t port);
95+
96+
std::mutex _mutex;
97+
uint16_t _port = MIN_PORT;
98+
uint16_t _min_port = MIN_PORT;
99+
uint16_t _max_port = MAX_PORT;
100+
};
101+
66102
} // namespace gtest
67103

68104
#endif /* TEST_GTEST_COMMON_H */

‎test/gtest/main.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,12 @@
1515
* limitations under the License.
1616
*/
1717
#include "plugin_manager.h"
18+
#include "common.h"
1819
#include <gtest/gtest.h>
20+
#include <iostream>
21+
#include <sstream>
22+
#include <vector>
23+
#include <string>
1924

2025
namespace gtest {
2126
std::vector<std::string> SplitWithDelimiter(const std::string &str,
@@ -30,6 +35,19 @@ std::vector<std::string> SplitWithDelimiter(const std::string &str,
3035
return tokens;
3136
}
3237

38+
void
39+
ParseTcpPortRange(const std::string &arg) {
40+
if (arg.find("--min-tcp-port=") == 0) {
41+
const std::string min_port = SplitWithDelimiter(arg, '=').back();
42+
PortAllocator::instance().set_min_port(std::stoi(min_port));
43+
}
44+
45+
if (arg.find("--max-tcp-port=") == 0) {
46+
const std::string max_port = SplitWithDelimiter(arg, '=').back();
47+
PortAllocator::instance().set_max_port(std::stoi(max_port));
48+
}
49+
}
50+
3351
void ParseArguments(int argc, char **argv) {
3452
for (int i = 1; i < argc; ++i) {
3553
if (std::string(argv[i]).find("--tests_plugin_dirs=") == 0) {
@@ -42,6 +60,8 @@ void ParseArguments(int argc, char **argv) {
4260
}
4361
}
4462
}
63+
64+
ParseTcpPortRange(argv[i]);
4565
}
4666
}
4767

‎test/gtest/metadata_exchange.cpp

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -40,22 +40,6 @@
4040

4141
namespace gtest {
4242
namespace metadata_exchange {
43-
44-
namespace {
45-
46-
int getRandomPort()
47-
{
48-
static constexpr int min_port = 10000;
49-
static constexpr int max_port = 65535;
50-
static std::random_device rd;
51-
static std::mt19937 gen(rd());
52-
static std::uniform_int_distribution<int> distr(min_port, max_port);
53-
54-
return distr(gen);
55-
}
56-
57-
}; // unnamed namespace
58-
5943
class MemBuffer {
6044
public:
6145
MemBuffer(size_t size) :
@@ -136,11 +120,9 @@ class MetadataExchangeTestFixture : public testing::Test {
136120

137121
void SetUp() override
138122
{
139-
int port_base = getRandomPort();
140-
141123
// Create two agents
142124
for (int i = 0; i < AGENT_COUNT_; i++) {
143-
int port = port_base + i;
125+
const auto port = PortAllocator::next_tcp_port();
144126
std::string name = "agent_" + std::to_string(i);
145127
nixlAgentConfig cfg(false, true, port, nixl_thread_sync_t::NIXL_THREAD_SYNC_STRICT);
146128

‎test/gtest/plugins/meson.build

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ if not aws_s3.found()
2020
endif
2121

2222
plugins_test_exe = executable('plugins_gtest',
23-
sources : ['../main.cpp', 'obj_plugin.cpp'],
23+
sources : ['../main.cpp', '../common.cpp', 'obj_plugin.cpp'],
2424
include_directories: [nixl_inc_dirs, utils_inc_dirs, plugins_inc_dirs, '.'],
2525
dependencies : [nixl_dep, gtest_dep, absl_strings_dep, absl_time_dep, plugin_deps,
2626
obj_backend_interface],

0 commit comments

Comments
 (0)