Skip to content

Commit 71ee0a8

Browse files
Lt-FlashLt-Flash
committed
clusterer_controller: the multicast controller, added to clusterer
The nodes of a cluster declared with cluster_options use_controller=1 find each other over encrypted IPv4 multicast, elect a master, receive their node ids and BIN addresses from it, and fail the cluster's sharing tags over; every other cluster behaves as with clusterer. clusterer's side (per-cluster node ids, controller-managed clusters and sharing tags, BIN messages that cannot override the controller) is in clusterer's files, the controller's own code in cl_ctr_*. Loading clusterer.so beside it is refused. With its unit, network-namespace and real-cluster tests, the packages (opensips-module-contrib-clusterer-controller) and the README.
1 parent f20cfc4 commit 71ee0a8

53 files changed

Lines changed: 17093 additions & 180 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎modules-contrib/clusterer_controller/Makefile‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,23 @@ NAME=clusterer_controller.so
66
#DEFS+= -DCLUSTERER_DBG
77
#DEFS+= -DCLUSTERER_EXTRA_BIN_DBG
88

9+
# Linux-only: the controller's timers are timerfds and its multicast socket is
10+
# pinned to its interface with SO_BINDTODEVICE
11+
ifneq ($(OS),linux)
12+
$(error clusterer_controller builds on Linux only (timerfd, SO_BINDTODEVICE) - drop it from include_modules on $(OS))
13+
endif
14+
15+
# libsodium is a hard requirement: it provides the payload AEAD
16+
# (XChaCha20-Poly1305), the bootstrap KDF (Argon2id), the Noise_NNpsk0 join
17+
# handshake, and X25519 / HKDF-SHA256 / RNG. It is linked dynamically (distro
18+
# static libs are usually non-PIC), so target hosts need the libsodium runtime
19+
# package (e.g. `apt install libsodium23`).
20+
SODIUM_EXISTS := $(shell pkg-config --exists libsodium 2>/dev/null && echo yes)
21+
ifeq ($(SODIUM_EXISTS),yes)
22+
DEFS += $(shell pkg-config --cflags libsodium)
23+
LIBS += $(shell pkg-config --libs libsodium)
24+
else
25+
$(error clusterer_controller requires libsodium (install libsodium-dev / libsodium-devel))
26+
endif
27+
928
include ../../Makefile.modules

‎modules-contrib/clusterer_controller/README.md‎

Lines changed: 1429 additions & 67 deletions
Large diffs are not rendered by default.

‎modules-contrib/clusterer_controller/cl_ctr.h‎

Lines changed: 1175 additions & 0 deletions
Large diffs are not rendered by default.

‎modules-contrib/clusterer_controller/cl_ctr_caps.c‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
/*
2+
* clusterer_controller - capability bits
3+
*
4+
* Copyright (C) 2026 Yury Kirsanov
5+
*
6+
* This file is part of opensips, a free SIP server.
7+
*
8+
* opensips is free software; you can redistribute it and/or modify
9+
* it under the terms of the GNU General Public License as published by
10+
* the Free Software Foundation; either version 2 of the License, or
11+
* (at your option) any later version.
12+
*
13+
* opensips is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* GNU General Public License for more details.
17+
*
18+
* You should have received a copy of the GNU General Public License
19+
* along with this program; if not, write to the Free Software
20+
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21+
*
22+
*/
23+
24+
#include "cl_ctr.h"
25+
#include "cl_ctr_caps.h"
26+
27+
/* only the master hears every ALIVE: it relays the shared bits in MASTER_ALIVE;
28+
* a build older than the field advertises nothing, so is never asked */
29+
30+
static const struct {
31+
uint32_t bit;
32+
const char *name;
33+
} cl_ctr_cap_names[] = {
34+
{ CL_CTR_CAP_PROBE, "probe" },
35+
{ CL_CTR_CAP_MAINT_SET, "maint_set" },
36+
{ CL_CTR_CAP_MAINT_FULL, "maint_full" },
37+
};
38+
39+
#define CL_CTR_CAP_NAMES (int)(sizeof(cl_ctr_cap_names) / sizeof(cl_ctr_cap_names[0]))
40+
41+
/* live as cl_ctr_live_count() counts it; caller holds the lock */
42+
static int cl_ctr_caps_live_locked(cl_ctr_cluster_t *cl, const cl_ctr_peer_t *e,
43+
time_t cutoff)
44+
{
45+
if (e->last_seen < cutoff)
46+
return 0;
47+
if (cl->forgiven_at && e->last_seen == cl->forgiven_at &&
48+
strcmp(e->ip, my_ip) != 0)
49+
return 0;
50+
return 1;
51+
}
52+
53+
/* the election window in force - the worker's adopted query_time is
54+
* mirrored in shm for the other processes (MI, script) */
55+
static time_t cl_ctr_caps_cutoff(cl_ctr_cluster_t *cl)
56+
{
57+
int qt = cl->peers->eff_query_time > 0 ? cl->peers->eff_query_time
58+
: query_time;
59+
return cl_ctr_now() - (time_t)(qt * CL_CTR_ELECT_FACTOR);
60+
}
61+
62+
uint32_t cl_ctr_table_caps(cl_ctr_cluster_t *cl)
63+
{
64+
time_t cutoff = cl_ctr_caps_cutoff(cl);
65+
uint32_t caps = CL_CTR_CAPS_MINE;
66+
int i;
67+
68+
lock_start_read(cl->peers->lock);
69+
for (i = 0; i < cl->peers->count; i++) {
70+
cl_ctr_peer_t *e = &cl->peers->entries[i];
71+
if (strcmp(e->ip, my_ip) == 0 || !cl_ctr_caps_live_locked(cl, e, cutoff))
72+
continue;
73+
caps &= e->caps;
74+
}
75+
lock_stop_read(cl->peers->lock);
76+
return caps;
77+
}
78+
79+
uint32_t cl_ctr_cluster_caps(cl_ctr_cluster_t *cl)
80+
{
81+
uint32_t relayed;
82+
int im;
83+
84+
lock_start_read(cl->peers->lock);
85+
im = cl_ctr_i_am_master_locked(cl);
86+
relayed = cl->peers->cluster_caps;
87+
lock_stop_read(cl->peers->lock);
88+
return im ? cl_ctr_table_caps(cl) : (relayed & CL_CTR_CAPS_MINE);
89+
}
90+
91+
/* on the master: how many live members lack @cap, the first one's IP in @ip;
92+
* 0 elsewhere - a member does not hear the others */
93+
static int cl_ctr_live_lacking(cl_ctr_cluster_t *cl, uint32_t cap, char *ip,
94+
int ip_sz)
95+
{
96+
time_t cutoff = cl_ctr_caps_cutoff(cl);
97+
int i, n = 0;
98+
99+
ip[0] = '\0';
100+
lock_start_read(cl->peers->lock);
101+
if (!cl_ctr_i_am_master_locked(cl)) {
102+
lock_stop_read(cl->peers->lock);
103+
return 0;
104+
}
105+
for (i = 0; i < cl->peers->count; i++) {
106+
cl_ctr_peer_t *e = &cl->peers->entries[i];
107+
if (strcmp(e->ip, my_ip) == 0 || !cl_ctr_caps_live_locked(cl, e, cutoff))
108+
continue;
109+
if (e->caps & cap)
110+
continue;
111+
if (n++ == 0)
112+
snprintf(ip, ip_sz, "%s", e->ip);
113+
}
114+
lock_stop_read(cl->peers->lock);
115+
return n;
116+
}
117+
118+
int cl_ctr_mi_add_caps(mi_item_t *obj, const char *name, int name_len,
119+
uint32_t caps)
120+
{
121+
mi_item_t *arr = add_mi_array(obj, (char *)name, name_len);
122+
int k;
123+
124+
if (!arr)
125+
return -1;
126+
for (k = 0; k < CL_CTR_CAP_NAMES; k++)
127+
if ((caps & cl_ctr_cap_names[k].bit) &&
128+
add_mi_string(arr, NULL, 0, cl_ctr_cap_names[k].name,
129+
strlen(cl_ctr_cap_names[k].name)) < 0)
130+
return -1;
131+
return 0;
132+
}
133+
134+
int cl_ctr_maint_supported(cl_ctr_cluster_t *cl, int remote, int st,
135+
char *why, int why_sz)
136+
{
137+
uint32_t have = cl_ctr_cluster_caps(cl);
138+
char ip[CL_CTR_MAX_IP_LEN + 1];
139+
int n;
140+
141+
if (remote && !(have & CL_CTR_CAP_MAINT_SET)) {
142+
n = cl_ctr_live_lacking(cl, CL_CTR_CAP_MAINT_SET, ip, sizeof ip);
143+
snprintf(why, why_sz, "not every live member runs a build with remote "
144+
"maintenance (no maint_set capability%s%s%s) - finish the "
145+
"upgrade first", n ? ", " : "", ip, n ? " among them" : "");
146+
return -1;
147+
}
148+
if (st == CL_CTR_MAINT_FULL && !(have & CL_CTR_CAP_MAINT_FULL)) {
149+
n = cl_ctr_live_lacking(cl, CL_CTR_CAP_MAINT_FULL, ip, sizeof ip);
150+
snprintf(why, why_sz, "not every live member acts on the full level "
151+
"(no maint_full capability%s%s%s) - they would keep "
152+
"replicating to the node", n ? ", " : "", ip,
153+
n ? " among them" : "");
154+
return -1;
155+
}
156+
return 0;
157+
}

‎modules-contrib/clusterer_controller/cl_ctr_caps.h‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* clusterer_controller - capability bits: declarations
3+
*
4+
* Copyright (C) 2026 Yury Kirsanov
5+
*
6+
* This file is part of opensips, a free SIP server.
7+
*
8+
* opensips is free software; you can redistribute it and/or modify
9+
* it under the terms of the GNU General Public License as published by
10+
* the Free Software Foundation; either version 2 of the License, or
11+
* (at your option) any later version.
12+
*
13+
* opensips is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* GNU General Public License for more details.
17+
*
18+
* You should have received a copy of the GNU General Public License
19+
* along with this program; if not, write to the Free Software
20+
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21+
*
22+
*/
23+
24+
#ifndef CL_CTR_CAPS_H
25+
#define CL_CTR_CAPS_H
26+
27+
#include "cl_ctr.h"
28+
29+
/* advertised on ALIVE after the maintenance byte; used only towards nodes with
30+
* the bit (or while all live members have it) - append, never renumber */
31+
#define CL_CTR_CAP_PROBE 0x00000001u /* answers MASTER_PROBE */
32+
#define CL_CTR_CAP_MAINT_SET 0x00000002u /* applies MAINT_SET to itself */
33+
#define CL_CTR_CAP_MAINT_FULL 0x00000004u /* acts on the full level */
34+
/* a test build with -DCL_CTR_CAPS_MINE=0 stands in for an older one of the
35+
* same wire version (netns suite, T27) */
36+
#ifndef CL_CTR_CAPS_MINE
37+
#define CL_CTR_CAPS_MINE (CL_CTR_CAP_PROBE | CL_CTR_CAP_MAINT_SET | \
38+
CL_CTR_CAP_MAINT_FULL)
39+
#endif
40+
#define CL_CTR_CAPS_SZ 4
41+
42+
/* bits every live member in this node's table shares, self included -
43+
* meaningful on the master, which hears them all */
44+
uint32_t cl_ctr_table_caps(cl_ctr_cluster_t *cl);
45+
/* bits every live member shares: computed on the master, as last relayed on
46+
* a member */
47+
uint32_t cl_ctr_cluster_caps(cl_ctr_cluster_t *cl);
48+
/* an MI array @name of the capability names set in @caps */
49+
int cl_ctr_mi_add_caps(mi_item_t *obj, const char *name, int name_len,
50+
uint32_t caps);
51+
/* can maintenance @st be set here or on another node (@remote), given what every
52+
* live member shares? 0, or -1 with the reason in @why */
53+
int cl_ctr_maint_supported(cl_ctr_cluster_t *cl, int remote, int st,
54+
char *why, int why_sz);
55+
56+
#endif /* CL_CTR_CAPS_H */

0 commit comments

Comments
 (0)