tm: render the anycast Via cid from the live node id, not a frozen one

tm_init_cluster() read this node cluster id once, at mod_init, and baked
it into the ";cid=<id>" Via parameter (and into a cached tm_node_id used
to decide whether an anycast reply is ours). That assumes the id is known
and stable by mod_init, which holds for a statically configured clusterer
but not for a controller-managed one: there the id is assigned at runtime,
after the node joins, and can change on re-election. So mod_init froze the
still-unassigned id (-1, printed as its unsigned form 18446744073709551615)
into every outgoing Via, and every node compared incoming cids against -1
- t_anycast_replicate() could never route a reply to the owning node.

Read the id live instead: pre-build only the fixed ";<param>=" prefix at
init, and have tm_via_cid() append the current get_my_id() per request
(returning no parameter while the node still has no id); compare incoming
cids against the live get_my_id() too. cl_get_my_id() already returns the
runtime id, so this needs nothing from the caller and self-corrects across
re-elections.

Author: Lt-Flash

modules/tm/cluster.c

@@ -32,7 +32,9 @@ str tm_cid;
 int tm_repl_cluster = 0;
 int tm_repl_auto_cancel = 1;
 str tm_cluster_param = str_init(TM_CLUSTER_DEFAULT_PARAM);
-static int tm_node_id = 0;
+/* length of the fixed ";<param>=" prefix of tm_cid; the node id is appended
+ * live per-request by tm_via_cid() */
+static int tm_cid_prefix_len;
 static str tm_repl_cap = str_init("tm-repl");
 
 struct clusterer_binds cluster_api;
@@ -256,8 +258,6 @@ static void receive_tm_repl(bin_packet_t *packet)
 
 int tm_init_cluster(void)
 {
-	str cid;
-
 	if (tm_repl_cluster == 0) {
 		LM_DBG("tm_replication_cluster not set - not engaging!\n");
 		return 0;
@@ -282,11 +282,13 @@ int tm_init_cluster(void)
 		/* overwrite structure to disable clusterer */
 		goto cluster_error;
 	}
-	tm_node_id = cluster_api.get_my_id();
-
-	/* build the via param */
-	cid.s = int2str(tm_node_id, &cid.len);
-	tm_cid.s = pkg_malloc(1/*;*/ + tm_cluster_param.len + 1/*=*/ + cid.len);
+	/* Pre-build only the fixed ";<param>=" prefix. The node id is NOT read
+	 * here: under a controller-managed cluster it is assigned at runtime
+	 * (after this mod_init) and may change on re-election, so freezing it
+	 * now would stamp a stale (typically -1) id into every Via. tm_via_cid()
+	 * appends the current id per-request instead. Room is left for the
+	 * largest id an int2str() can render. */
+	tm_cid.s = pkg_malloc(1/*;*/ + tm_cluster_param.len + 1/*=*/ + INT2STR_MAX_LEN);
 	if (!tm_cid.s) {
 		LM_ERR("out of pkg memory!\n");
 		goto cluster_error;
@@ -296,8 +298,7 @@ int tm_init_cluster(void)
 	memcpy(tm_cid.s + tm_cid.len, tm_cluster_param.s, tm_cluster_param.len);
 	tm_cid.len += tm_cluster_param.len;
 	tm_cid.s[tm_cid.len++] = '=';
-	memcpy(tm_cid.s + tm_cid.len, cid.s, cid.len);
-	tm_cid.len += cid.len;
+	tm_cid_prefix_len = tm_cid.len;
 
 	return 0;
 
@@ -306,6 +307,28 @@ int tm_init_cluster(void)
 	return -1;
 }
 
+str *tm_via_cid(void)
+{
+	char *id_s;
+	int   id, id_len;
+
+	if (!tm_cluster_enabled())
+		return NULL;
+
+	/* read this node's id live - it is only known once the node has been
+	 * assigned one in the cluster (immediately for a static clusterer,
+	 * after the runtime join for a controller-managed one) and can change
+	 * on re-election; advertise no cid until we actually have one */
+	id = cluster_api.get_my_id();
+	if (id < 0)
+		return NULL;
+
+	id_s = int2str((unsigned long)id, &id_len);
+	memcpy(tm_cid.s + tm_cid_prefix_len, id_s, id_len);
+	tm_cid.len = tm_cid_prefix_len + id_len;
+	return &tm_cid;
+}
+
 #define TM_BIN_PUSH(_t, _f, _d) \
 	do { \
 		if (bin_push_##_t(&packet, _f) < 0) { \
@@ -522,7 +545,7 @@ int tm_reply_replicate(struct sip_msg *msg)
 	/* if there was no parameter, or it was, but it was ours, handle it */
 	if (cid < 0)
 		return 0;
-	if (cid == tm_node_id) {
+	if (cid == cluster_api.get_my_id()) {
 		LM_DBG("reply should be processed by us (%d)\n", cid);
 		return 0;
 	}

modules/tm/cluster.h

@@ -51,7 +51,10 @@ int tm_anycast_cancel(struct sip_msg *msg);
 /* returns true if clusterer is enabled */
 #define tm_cluster_enabled() (cluster_api.register_capability != 0)
 
-/* returns the via parameter for the cluster */
-#define tm_via_cid() (tm_cluster_enabled()?&tm_cid:0)
+/* Returns the ";<param>=<node_id>" Via parameter for the cluster, rendered
+ * with this node's CURRENT clusterer id (read live, not frozen at init - the
+ * id is assigned at runtime under a controller-managed cluster and may change
+ * on re-election). NULL when clustering is off or no id is assigned yet. */
+str *tm_via_cid(void);
 
 #endif /* _TM_CLUSTER_H_ */