mem: add HG_MALLOC, a hugepage-backed slab allocator, with CPU pinning

A fifth core allocator, selectable at runtime with -a/-s/-k HG_MALLOC
alongside F_MALLOC, Q_MALLOC, HP_MALLOC and F_PARALLEL_MALLOC, plus the
CPU pinning it needs to be worth having.

Under load OpenSIPS spends a surprising share of its CPU inside the
allocator - profiling a topology-hiding proxy put allocator self-time
between 9.5% and 22.8% of total CPU depending on which of the existing
four was in use. That is a large enough budget to be worth attacking
directly rather than by shaving call paths elsewhere.

The design takes two things the existing allocators do not. First, huge
pages: the arena is acquired through a four-tier ladder - MAP_HUGETLB,
then MADV_HUGEPAGE, then MADV_COLLAPSE, then plain 4K - and each tier is
verified through /proc rather than assumed from a kernel version or a
sysfs setting, degrading quietly and logging what it actually got.
Second, and this turns out to matter more: allocation is served from
fixed size classes off a per-process free stack, so the hot path takes no
lock and touches no shared cache line. Measuring the tiers separately
showed huge pages are only about a fifth of the win - tier 4, with huge
pages entirely disabled, still beat F_MALLOC by 2.8x - so the allocator
is worth using on machines with no hugetlb pool at all.

Keeping free lists per process only pays while a process stays on one
CPU, which OpenSIPS had no way to arrange: there was no affinity support
in the core. So this adds it. pin_workers pins every worker round-robin;
pin_udp_cpus / pin_tcp_cpus / pin_timer_cpus / pin_module_cpus confine one
kind of process to a CPU list; and a UDP listener may name pin_cpus of
its own, which is finer still and lets a public interface and a private
one keep off each other's cores. Precedence runs listener, then process
type, then unpinned. Every choice is intersected with the set the process
is already allowed to run on, read back with sched_getaffinity(), so a
cpuset or container can only ever narrow it, never widen it. The CPU is
picked by occupancy over the processes running at that moment rather than
from a process-table slot, which is what makes it behave under
auto-scaling, where slots are freed and reused.

One process is deliberately not pinned to a single CPU: TCP main is
multithreaded - its IO pool defaults to one thread per online CPU - so a
single-CPU pin would serialise every one of those threads onto one core.
It is confined to the whole pin_tcp_cpus list instead, before its threads
are created so they all inherit it, and it does not count as occupying a
slot when the single-threaded TCP workers pick theirs.

Measured on the topology-hiding path, HG_MALLOC costs 10-24% less CPU
than F_MALLOC at matched load in containers, and 20-45% less on a
dispatcher load balancer holding 20,000 calls in flight, with the gap
widening as the load rises. It is not free: the arena is pre-faulted and
mlock()ed at startup, so it is resident from the first second, and the
size classes need roughly twice the shm F_MALLOC does for the same
traffic. Both are documented, along with what running short looks like -
loud in the log, but not fatal, which is the trap.

CPU pinning is Linux-only and compiled out elsewhere with a warning
rather than guessed at. The allocator itself is portable: ARM alignment
and page-size assumptions are handled and asserted, and it has been
cross-compiled for aarch64 and armv7, though not yet run on ARM hardware.

Author: Lt-Flash

Makefile.conf.template

@@ -97,6 +97,11 @@ DEFS+= -DQ_MALLOC #Quality assurance memory allocator with runtime safety checks
 DEFS+= -DHP_MALLOC #High performance allocator with fine-grained locking
 DEFS+= -DDBG_MALLOC #Include additional, debug-enabled allocator flavors
 DEFS+= -DF_PARALLEL_MALLOC #Fast memory allocator with paralel buckets
+DEFS+= -DHG_MALLOC #Hugepage-backed slab allocator with a lock-free fast path
+# HG_MALLOC reserves and pins its own arena, serves fixed size classes out of
+# it, and never hands memory back from one class to another - so it needs
+# roughly twice the -m that F_MALLOC needs for the same traffic, and says so
+# via carve_chunk errors when it does not have it. See mem/README.hg_malloc
 #DEFS+= -DQM_DBG_MALLOC_HIST=3 #Size of debug history for Q_MALLOC (default is 1)
 #DEFS+= -DNO_DEBUG #Compile out all debug messages
 #DEFS+= -DNO_LOG #Compile out all logging

Makefile.defs

@@ -524,10 +524,12 @@ endif
 ifeq (,$(findstring F_MALLOC, $(DEFS)))
 ifeq (,$(findstring Q_MALLOC, $(DEFS)))
 ifeq (,$(findstring HP_MALLOC, $(DEFS)))
+ifeq (,$(findstring HG_MALLOC, $(DEFS)))
 	DEFS+= -DQ_MALLOC
 endif
 endif
 endif
+endif
 
 ifneq (,$(MEM_STATS_HDR))
 EXPECTED_MD5=$(shell [ -f $(MEM_STATS_HDR) ] && tail -n 1 $(MEM_STATS_HDR) | awk '{print $$4}')
@@ -558,6 +560,10 @@ MKTAGS=ctags -R .
 #		decently fast allocator with extra sanity checks + safety buffers
 # -DHP_MALLOC
 #		high performance allocator optimized for shared memory multiprocessing
+# -DHG_MALLOC
+#		hugepage-backed slab allocator; fixed size classes bump-carved from
+#		its own huge-page (or best-effort degraded) reservation, no fallback
+#		to another allocator on exhaustion - increase -m/-M instead
 # -DDBG_MALLOC
 #       will cause each of the enabled allocators to also include a "debugging"
 #       flavor of itself;  by running a debugging version of an allocator, you

cfg.lex

@@ -246,6 +246,11 @@ RPM_MEM_FILE "restart_persistency_cache_file"
 RPM_MEM_SIZE "restart_persistency_size"
 MEMLOG		"memlog"|"mem_log"
 MEMDUMP		"memdump"|"mem_dump"
+PIN_WORKERS	"pin_workers"|"cpu_pinning"
+PIN_UDP_CPUS	"pin_udp_cpus"
+PIN_TCP_CPUS	"pin_tcp_cpus"
+PIN_TIMER_CPUS	"pin_timer_cpus"
+PIN_MODULE_CPUS	"pin_module_cpus"
 SHM_MEMLOG_SIZE			"shm_memlog_size"
 EXECMSGTHRESHOLD		"execmsgthreshold"|"exec_msg_threshold"
 EXECDNSTHRESHOLD		"execdnsthreshold"|"exec_dns_threshold"
@@ -318,6 +323,7 @@ TICK		\'
 SLASH		"/"
 AS			{EAT_ABLE}("as"|"AS"){EAT_ABLE}
 USE_WORKERS	{EAT_ABLE}("use_workers"|"USE_WORKERS"){EAT_ABLE}
+PIN_CPUS	{EAT_ABLE}("pin_cpus"|"PIN_CPUS"){EAT_ABLE}
 SOCK_TOS	{EAT_ABLE}("tos"|"TOS"){EAT_ABLE}
 USE_AUTO_SCALING_PROFILE {EAT_ABLE}("use_auto_scaling_profile"|"USE_AUTO_SCALING_PROFILE"){EAT_ABLE}
 SCALE_UP_TO		{EAT_ABLE}("scale"|"SCALE"){EAT_ABLE}+("up"|"UP"){EAT_ABLE}+("to"|"TO"){EAT_ABLE}
@@ -480,6 +486,11 @@ SPACE		[ ]
 <INITIAL>{RPM_MEM_SIZE}	{ count(); yylval.strval=yytext; return RPM_MEM_SIZE; }
 <INITIAL>{MEMLOG}	{ count(); yylval.strval=yytext; return MEMLOG; }
 <INITIAL>{MEMDUMP}	{ count(); yylval.strval=yytext; return MEMDUMP; }
+<INITIAL>{PIN_WORKERS}	{ count(); yylval.strval=yytext; return PIN_WORKERS; }
+<INITIAL>{PIN_UDP_CPUS}	{ count(); yylval.strval=yytext; return PIN_UDP_CPUS; }
+<INITIAL>{PIN_TCP_CPUS}	{ count(); yylval.strval=yytext; return PIN_TCP_CPUS; }
+<INITIAL>{PIN_TIMER_CPUS}	{ count(); yylval.strval=yytext; return PIN_TIMER_CPUS; }
+<INITIAL>{PIN_MODULE_CPUS}	{ count(); yylval.strval=yytext; return PIN_MODULE_CPUS; }
 <INITIAL>{SHM_MEMLOG_SIZE}	{ count(); yylval.strval=yytext; return SHM_MEMLOG_SIZE; }
 <INITIAL>{EXECMSGTHRESHOLD}	{ count(); yylval.strval=yytext; return EXECMSGTHRESHOLD; }
 <INITIAL>{EXECDNSTHRESHOLD}	{ count(); yylval.strval=yytext; return EXECDNSTHRESHOLD; }
@@ -601,6 +612,7 @@ SPACE		[ ]
 <INITIAL>{COMMA}		{ count(); return COMMA; }
 <INITIAL>{SEMICOLON}	{ count(); return SEMICOLON; }
 <INITIAL>{USE_WORKERS}  { count(); return USE_WORKERS; }
+<INITIAL>{PIN_CPUS}  { count(); return PIN_CPUS; }
 <INITIAL>{SOCK_TOS}	{ count(); return SOCK_TOS; }
 <INITIAL>{USE_AUTO_SCALING_PROFILE}  { count(); return USE_AUTO_SCALING_PROFILE; }
 <INITIAL>{COLON}	{ count(); return COLON; }

cfg.y

@@ -182,6 +182,7 @@ extern char *finame;
 struct listen_param {
 	enum si_flags flags;
 	int workers;
+	char *pin_cpus;
 	int tos;
 	struct socket_id *socket;
 	char *tag;
@@ -352,6 +353,11 @@ extern int cfg_parse_only_routes;
 %token RPM_MEM_SIZE
 %token MEMLOG
 %token MEMDUMP
+%token PIN_WORKERS
+%token PIN_UDP_CPUS
+%token PIN_TCP_CPUS
+%token PIN_TIMER_CPUS
+%token PIN_MODULE_CPUS
 %token SHM_MEMLOG_SIZE
 %token EXECMSGTHRESHOLD
 %token EXECDNSTHRESHOLD
@@ -459,6 +465,7 @@ extern int cfg_parse_only_routes;
 %token SLASH
 %token AS
 %token USE_WORKERS
+%token PIN_CPUS
 %token SOCK_TOS
 %token USE_AUTO_SCALING_PROFILE
 %token MAX
@@ -761,6 +768,9 @@ socket_def_param: ANYCAST { IFOR();
 				| USE_WORKERS NUMBER { IFOR();
 					p_tmp.workers=$2;
 					}
+				| PIN_CPUS STRING { IFOR();
+					p_tmp.pin_cpus=$2;
+					}
 				| SOCK_TOS NUMBER { IFOR();
 					p_tmp.tos=$2;
 					}
@@ -1257,6 +1267,16 @@ assign_stm: LOGLEVEL EQUAL snumber { IFOR();
 		| SHM_MEMLOG_SIZE EQUAL error { yyerror("int value expected"); }
 		| MEMDUMP EQUAL snumber { IFOR(); memdump=$3; }
 		| MEMDUMP EQUAL error { yyerror("int value expected"); }
+		| PIN_WORKERS EQUAL snumber { IFOR(); pin_workers=$3; }
+		| PIN_WORKERS EQUAL error { yyerror("int value expected"); }
+		| PIN_UDP_CPUS EQUAL STRING { IFOR(); pin_udp_cpus=$3; pin_workers=1; }
+		| PIN_UDP_CPUS EQUAL error { yyerror("string value expected"); }
+		| PIN_TCP_CPUS EQUAL STRING { IFOR(); pin_tcp_cpus=$3; pin_workers=1; }
+		| PIN_TCP_CPUS EQUAL error { yyerror("string value expected"); }
+		| PIN_TIMER_CPUS EQUAL STRING { IFOR(); pin_timer_cpus=$3; pin_workers=1; }
+		| PIN_TIMER_CPUS EQUAL error { yyerror("string value expected"); }
+		| PIN_MODULE_CPUS EQUAL STRING { IFOR(); pin_module_cpus=$3; pin_workers=1; }
+		| PIN_MODULE_CPUS EQUAL error { yyerror("string value expected"); }
 		| EXECMSGTHRESHOLD EQUAL NUMBER {  IFOR();execmsgthreshold=$3; }
 		| EXECMSGTHRESHOLD EQUAL error { yyerror("int value expected"); }
 		| EXECDNSTHRESHOLD EQUAL NUMBER { IFOR(); execdnsthreshold=$3; }
@@ -2793,6 +2813,7 @@ static void fill_socket_id(struct listen_param *param, struct socket_id *s)
 	while (s) {
 		s->flags |= param->flags;
 		s->workers = param->workers;
+		s->pin_cpus = param->pin_cpus;
 		s->tos = param->tos;
 		s->auto_scaling_profile = param->auto_scaling_profile;
 		s->tag = param->tag;

globals.c

@@ -95,6 +95,13 @@ int check_via =  0;
 /* debugging level for memory stats */
 int memlog = L_DBG + 11;
 int memdump = L_DBG + 10;
+/* pin each worker to one CPU (0 = off). See pt.c pin_worker_to_cpu(). */
+int pin_workers = 0;
+/* optional per-process-type CPU lists, e.g. "0-7". NULL = all allowed CPUs */
+char *pin_udp_cpus = NULL;
+char *pin_tcp_cpus = NULL;
+char *pin_timer_cpus = NULL;
+char *pin_module_cpus = NULL;
 unsigned int shm_memlog_size = 0;
 /* debugging in case msg processing takes. too long disabled by default */
 int execmsgthreshold = 0;

globals.h

@@ -121,6 +121,11 @@ extern int is_pre_daemon;
 
 extern int memlog;  /*!< debugging level for printing memory debugs */
 extern int memdump; /*!< debugging level for dumping memory status */
+extern int pin_workers; /*!< pin each worker process to a single CPU */
+extern char *pin_udp_cpus;    /*!< CPU list for UDP workers */
+extern char *pin_tcp_cpus;    /*!< CPU list for TCP workers */
+extern char *pin_timer_cpus;  /*!< CPU list for timer processes */
+extern char *pin_module_cpus; /*!< CPU list for module processes */
 extern unsigned int shm_memlog_size;
 extern int execmsgthreshold;  /*!< Maximum number of microseconds a SIP msg processing can last
 						before triggering Warning log */

help_msg.h

@@ -96,6 +96,7 @@ Options:\n\
                       F_PARALLEL_MALLOC   F_PARALLEL_MALLOC_DBG\n\
                       Q_MALLOC            Q_MALLOC_DBG\n\
                       HP_MALLOC           HP_MALLOC_DBG\n\
+                      HG_MALLOC           HG_MALLOC_DBG\n\
     -k allocator The pkg memory allocator to use (overrides -a)\n\
     -s allocator The shared memory allocator to use (overrides -a)\n\
     -e allocator The restart-persistent memory allocator to use (overrides -a)\n\

ip_addr.h

@@ -154,6 +154,7 @@ struct socket_id {
 	int proto;
 	int port;
 	int workers;
+	char* pin_cpus;
 	int tos;
 	enum si_flags flags;
 	struct socket_id* next;