mem: HG_MALLOC - hugepage slab allocator with elastic, auto-scaling arenas

Combines the v1 allocator (OpenSIPS#4126: size-class slabs over a self-made
hugepage reservation, per-process lock-free free caches, the CPU
affinity subsystem that keeps them warm) with the v2 rewrite (in-page
buddy grid, class-dedicated blocks, ownership checks on every header
read, event-driven GC, corruption counters) and extends them with v3:
arenas that grow and shrink at runtime between -m/-M INIT and a
pre-fork reserved CAP.

Growth commits whole 16 MB granules by mlock, bounded by three
independent limits (profile target, backing-tier capacity, host-RAM
floor with pkg deltas charged per worker); shrink releases whole-free
top granules back to the kernel (MADV_REMOVE returns hugetlb pages to
the pool). Policy reuses the auto_scaling_profile grammar via
shm_auto_scaling_profile / pkg_auto_scaling_profile; a latched
GROW-BLOCKED state raises E_CORE_SHM_GROW_BLOCKED; a dry-run mode
narrates decisions without acting. With no cap configured the arena is
fixed and behaviour is unchanged.

Includes modules/hgstress (multi-process stamp/verify soak driver) and
the full operator/design documentation under mem/.

Author: Lt-Flash

Makefile.conf.template

@@ -18,6 +18,7 @@
 #db_oracle= Provides Oracle connectivity for OpenSIPS. | Development library of OCI, typically instantclient-sdk-10.2.0.3
 #db_perlvdb= Provides a virtualization framework for OpenSIPS's database access. | Perl library development files, typically libperl-dev
 #db_postgres= Provides Postgres connectivity for OpenSIPS | PostgreSQL library and development library - typically libpq5 and libpq-dev
+#db_redis= Provides Redis (single instance or cluster) row storage for OpenSIPS | Redis client library, hiredis
 #db_sqlite= Provides SQLite connectivity for OpenSIPS | SQLite library and development library - typically libsqlite3 and libsqlite3-dev
 #db_unixodbc= Allows to use the unixodbc package with OpenSIPS | ODBC library and ODBC development library
 #dialplan= Implements generic string translations based on matching and replacement rules | PCRE development library, typically libpcre2-dev
@@ -76,7 +77,7 @@
 #uuid= UUID generator | uuid-dev
 
 # Modules omitted from the default build, generally due to external dependencies.
-exclude_modules?= aaa_diameter aaa_radius auth_jwt auth_web3 b2b_logic_xml cachedb_cassandra cachedb_couchbase cachedb_dynamodb cachedb_memcached cachedb_mongodb cachedb_redis carrierroute cgrates compression cpl_c db_berkeley db_http db_mysql db_oracle db_perlvdb db_postgres db_sqlite db_unixodbc dialplan emergency event_rabbitmq event_kafka event_sqs h350 httpd http2d identity jabber json launch_darkly ldap lua mi_xmlrpc mmgeoip opentelemetry osp perl pi_http presence presence_dialoginfo presence_mwi presence_reginfo presence_xml presence_dfks proto_ipsec proto_sctp proto_tls proto_wss pua pua_bla pua_dialoginfo pua_mi pua_reginfo pua_usrloc pua_xmpp python regex rabbitmq_consumer rest_client rls rtp.io siprec sngtc snmpstats stir_shaken tls_mgm tls_openssl tls_wolfssl uuid xcap xcap_client xml xmpp
+exclude_modules?= aaa_diameter aaa_radius auth_jwt auth_web3 b2b_logic_xml cachedb_cassandra cachedb_couchbase cachedb_dynamodb cachedb_memcached cachedb_mongodb cachedb_redis carrierroute cgrates clusterer_controller compression cpl_c db_berkeley db_http db_mysql db_oracle db_perlvdb db_postgres db_redis db_sqlite db_unixodbc dialplan emergency event_kafka event_rabbitmq event_sqs h350 http2d httpd identity jabber json launch_darkly ldap lua mi_xmlrpc mmgeoip opentelemetry osp perl pi_http presence presence_dfks presence_dialoginfo presence_mwi presence_reginfo presence_xml proto_ipsec proto_sctp proto_tls proto_wss pua pua_bla pua_dialoginfo pua_mi pua_reginfo pua_usrloc pua_xmpp python rabbitmq_consumer regex rest_client rls rtp.io siprec sngtc snmpstats stir_shaken tls_mgm tls_openssl tls_wolfssl uuid xcap xcap_client xml xmpp
 
 # Modules forced into the build, even when also listed in exclude_modules.
 include_modules?=
@@ -97,6 +98,17 @@ DEFS+= -DQ_MALLOC #Quality assurance memory allocator with runtime safety checks
 DEFS+= -DHP_MALLOC #High performance allocator with fine-grained locking
 DEFS+= -DDBG_MALLOC #Include additional, debug-enabled allocator flavors
 DEFS+= -DF_PARALLEL_MALLOC #Fast memory allocator with paralel buckets
+DEFS+= -DHG_MALLOC #Hugepage-backed slab allocator with a lock-free fast path
+# HG_MALLOC reserves its arena up front on 2 MB huge pages, pre-faults and
+# mlock()s it, and serves fixed size classes from per-thread caches so the
+# common path takes no lock. A block belongs to one size class while in use;
+# once it drains completely it returns to the class-agnostic buddy and can be
+# re-carved for a different class, so the carve shrinks again after a burst.
+# Because the arena is pinned, resident memory equals -m plus -M x processes
+# from the first second - unlike the mmap-backed allocators, oversizing is not
+# cheap. It needs a huge-page pool of at least (-m + -M x procs) / 2 pages and
+# a raised memlock limit; without them it drops to a lower tier and keeps
+# running. See mem/README.hg_malloc and mem/README.hg_arena_v2
 #DEFS+= -DQM_DBG_MALLOC_HIST=3 #Size of debug history for Q_MALLOC (default is 1)
 #DEFS+= -DNO_DEBUG #Compile out all debug messages
 #DEFS+= -DNO_LOG #Compile out all logging

Makefile.defs

@@ -524,10 +524,12 @@ endif
 ifeq (,$(findstring F_MALLOC, $(DEFS)))
 ifeq (,$(findstring Q_MALLOC, $(DEFS)))
 ifeq (,$(findstring HP_MALLOC, $(DEFS)))
+ifeq (,$(findstring HG_MALLOC, $(DEFS)))
 	DEFS+= -DQ_MALLOC
 endif
 endif
 endif
+endif
 
 ifneq (,$(MEM_STATS_HDR))
 EXPECTED_MD5=$(shell [ -f $(MEM_STATS_HDR) ] && tail -n 1 $(MEM_STATS_HDR) | awk '{print $$4}')
@@ -558,6 +560,10 @@ MKTAGS=ctags -R .
 #		decently fast allocator with extra sanity checks + safety buffers
 # -DHP_MALLOC
 #		high performance allocator optimized for shared memory multiprocessing
+# -DHG_MALLOC
+#		hugepage-backed slab allocator; fixed size classes bump-carved from
+#		its own huge-page (or best-effort degraded) reservation, no fallback
+#		to another allocator on exhaustion - increase -m/-M instead
 # -DDBG_MALLOC
 #       will cause each of the enabled allocators to also include a "debugging"
 #       flavor of itself;  by running a debugging version of an allocator, you

cfg.lex

@@ -246,6 +246,11 @@ RPM_MEM_FILE "restart_persistency_cache_file"
 RPM_MEM_SIZE "restart_persistency_size"
 MEMLOG		"memlog"|"mem_log"
 MEMDUMP		"memdump"|"mem_dump"
+PIN_WORKERS	"pin_workers"|"cpu_pinning"
+PIN_UDP_CPUS	"pin_udp_cpus"
+PIN_TCP_CPUS	"pin_tcp_cpus"
+PIN_TIMER_CPUS	"pin_timer_cpus"
+PIN_MODULE_CPUS	"pin_module_cpus"
 SHM_MEMLOG_SIZE			"shm_memlog_size"
 EXECMSGTHRESHOLD		"execmsgthreshold"|"exec_msg_threshold"
 WORKER_REACTOR_TIMEOUT	"workerreactortimeout"|"worker_reactor_timeout"
@@ -290,6 +295,10 @@ DB_DEFAULT_URL "db_default_url"
 DB_MAX_ASYNC_CONNECTIONS "db_max_async_connections"
 DISABLE_503_TRANSLATION "disable_503_translation"
 AUTO_SCALING_PROFILE "auto_scaling_profile"
+SHM_AUTO_SCALING_PROFILE "shm_auto_scaling_profile"
+PKG_AUTO_SCALING_PROFILE "pkg_auto_scaling_profile"
+HG_RAM_FLOOR_MB "hg_ram_floor_mb"
+HG_AUTOSCALE_DRY_RUN "hg_autoscale_dry_run"
 AUTO_SCALING_CYCLE "auto_scaling_cycle"
 TIMER_WORKERS "timer_workers"
 
@@ -319,6 +328,7 @@ TICK		\'
 SLASH		"/"
 AS			{EAT_ABLE}("as"|"AS"){EAT_ABLE}
 USE_WORKERS	{EAT_ABLE}("use_workers"|"USE_WORKERS"){EAT_ABLE}
+PIN_CPUS	{EAT_ABLE}("pin_cpus"|"PIN_CPUS"){EAT_ABLE}
 SOCK_TOS	{EAT_ABLE}("tos"|"TOS"){EAT_ABLE}
 USE_AUTO_SCALING_PROFILE {EAT_ABLE}("use_auto_scaling_profile"|"USE_AUTO_SCALING_PROFILE"){EAT_ABLE}
 SCALE_UP_TO		{EAT_ABLE}("scale"|"SCALE"){EAT_ABLE}+("up"|"UP"){EAT_ABLE}+("to"|"TO"){EAT_ABLE}
@@ -481,6 +491,11 @@ SPACE		[ ]
 <INITIAL>{RPM_MEM_SIZE}	{ count(); yylval.strval=yytext; return RPM_MEM_SIZE; }
 <INITIAL>{MEMLOG}	{ count(); yylval.strval=yytext; return MEMLOG; }
 <INITIAL>{MEMDUMP}	{ count(); yylval.strval=yytext; return MEMDUMP; }
+<INITIAL>{PIN_WORKERS}	{ count(); yylval.strval=yytext; return PIN_WORKERS; }
+<INITIAL>{PIN_UDP_CPUS}	{ count(); yylval.strval=yytext; return PIN_UDP_CPUS; }
+<INITIAL>{PIN_TCP_CPUS}	{ count(); yylval.strval=yytext; return PIN_TCP_CPUS; }
+<INITIAL>{PIN_TIMER_CPUS}	{ count(); yylval.strval=yytext; return PIN_TIMER_CPUS; }
+<INITIAL>{PIN_MODULE_CPUS}	{ count(); yylval.strval=yytext; return PIN_MODULE_CPUS; }
 <INITIAL>{SHM_MEMLOG_SIZE}	{ count(); yylval.strval=yytext; return SHM_MEMLOG_SIZE; }
 <INITIAL>{EXECMSGTHRESHOLD}	{ count(); yylval.strval=yytext; return EXECMSGTHRESHOLD; }
 <INITIAL>{WORKER_REACTOR_TIMEOUT}	{ count(); yylval.strval=yytext; return WORKER_REACTOR_TIMEOUT; }
@@ -544,6 +559,14 @@ SPACE		[ ]
 									return DISABLE_503_TRANSLATION; }
 <INITIAL>{AUTO_SCALING_PROFILE}	{	count(); yylval.strval=yytext;
 									return AUTO_SCALING_PROFILE; }
+<INITIAL>{SHM_AUTO_SCALING_PROFILE}	{	count(); yylval.strval=yytext;
+									return SHM_AUTO_SCALING_PROFILE; }
+<INITIAL>{PKG_AUTO_SCALING_PROFILE}	{	count(); yylval.strval=yytext;
+									return PKG_AUTO_SCALING_PROFILE; }
+<INITIAL>{HG_RAM_FLOOR_MB}	{	count(); yylval.strval=yytext;
+									return HG_RAM_FLOOR_MB; }
+<INITIAL>{HG_AUTOSCALE_DRY_RUN}	{	count(); yylval.strval=yytext;
+									return HG_AUTOSCALE_DRY_RUN; }
 <INITIAL>{AUTO_SCALING_CYCLE}	{	count(); yylval.strval=yytext;
 									return AUTO_SCALING_CYCLE; }
 <INITIAL>{TIMER_WORKERS}	{	count(); yylval.strval=yytext;
@@ -603,6 +626,7 @@ SPACE		[ ]
 <INITIAL>{COMMA}		{ count(); return COMMA; }
 <INITIAL>{SEMICOLON}	{ count(); return SEMICOLON; }
 <INITIAL>{USE_WORKERS}  { count(); return USE_WORKERS; }
+<INITIAL>{PIN_CPUS}  { count(); return PIN_CPUS; }
 <INITIAL>{SOCK_TOS}	{ count(); return SOCK_TOS; }
 <INITIAL>{USE_AUTO_SCALING_PROFILE}  { count(); return USE_AUTO_SCALING_PROFILE; }
 <INITIAL>{COLON}	{ count(); return COLON; }

cfg.y

@@ -183,6 +183,7 @@ extern char *finame;
 struct listen_param {
 	enum si_flags flags;
 	int workers;
+	char *pin_cpus;
 	int tos;
 	struct socket_id *socket;
 	char *tag;
@@ -353,6 +354,11 @@ extern int cfg_parse_only_routes;
 %token RPM_MEM_SIZE
 %token MEMLOG
 %token MEMDUMP
+%token PIN_WORKERS
+%token PIN_UDP_CPUS
+%token PIN_TCP_CPUS
+%token PIN_TIMER_CPUS
+%token PIN_MODULE_CPUS
 %token SHM_MEMLOG_SIZE
 %token EXECMSGTHRESHOLD
 %token WORKER_REACTOR_TIMEOUT
@@ -407,6 +413,10 @@ extern int cfg_parse_only_routes;
 %token ASYNC_TOKEN
 %token LAUNCH_TOKEN
 %token AUTO_SCALING_PROFILE
+%token SHM_AUTO_SCALING_PROFILE
+%token PKG_AUTO_SCALING_PROFILE
+%token HG_RAM_FLOOR_MB
+%token HG_AUTOSCALE_DRY_RUN
 %token AUTO_SCALING_CYCLE
 %token TIMER_WORKERS
 
@@ -461,6 +471,7 @@ extern int cfg_parse_only_routes;
 %token SLASH
 %token AS
 %token USE_WORKERS
+%token PIN_CPUS
 %token SOCK_TOS
 %token USE_AUTO_SCALING_PROFILE
 %token MAX
@@ -763,6 +774,9 @@ socket_def_param: ANYCAST { IFOR();
 				| USE_WORKERS NUMBER { IFOR();
 					p_tmp.workers=$2;
 					}
+				| PIN_CPUS STRING { IFOR();
+					p_tmp.pin_cpus=$2;
+					}
 				| SOCK_TOS NUMBER { IFOR();
 					p_tmp.tos=$2;
 					}
@@ -1259,6 +1273,16 @@ assign_stm: LOGLEVEL EQUAL snumber { IFOR();
 		| SHM_MEMLOG_SIZE EQUAL error { yyerror("int value expected"); }
 		| MEMDUMP EQUAL snumber { IFOR(); memdump=$3; }
 		| MEMDUMP EQUAL error { yyerror("int value expected"); }
+		| PIN_WORKERS EQUAL snumber { IFOR(); pin_workers=$3; }
+		| PIN_WORKERS EQUAL error { yyerror("int value expected"); }
+		| PIN_UDP_CPUS EQUAL STRING { IFOR(); pin_udp_cpus=$3; pin_workers=1; }
+		| PIN_UDP_CPUS EQUAL error { yyerror("string value expected"); }
+		| PIN_TCP_CPUS EQUAL STRING { IFOR(); pin_tcp_cpus=$3; pin_workers=1; }
+		| PIN_TCP_CPUS EQUAL error { yyerror("string value expected"); }
+		| PIN_TIMER_CPUS EQUAL STRING { IFOR(); pin_timer_cpus=$3; pin_workers=1; }
+		| PIN_TIMER_CPUS EQUAL error { yyerror("string value expected"); }
+		| PIN_MODULE_CPUS EQUAL STRING { IFOR(); pin_module_cpus=$3; pin_workers=1; }
+		| PIN_MODULE_CPUS EQUAL error { yyerror("string value expected"); }
 		| EXECMSGTHRESHOLD EQUAL NUMBER {  IFOR();execmsgthreshold=$3; }
 		| EXECMSGTHRESHOLD EQUAL error { yyerror("int value expected"); }
 		| WORKER_REACTOR_TIMEOUT EQUAL NUMBER { IFOR(); worker_reactor_timeout=$3; }
@@ -1644,6 +1668,26 @@ assign_stm: LOGLEVEL EQUAL snumber { IFOR();
 		| AUTO_SCALING_PROFILE EQUAL error {
 				yyerror("bad auto-scaling profile definition");
 				}
+		| SHM_AUTO_SCALING_PROFILE EQUAL ID { IFOR();
+				hg_shm_profile_name=$3; }
+		| SHM_AUTO_SCALING_PROFILE EQUAL error {
+				yyerror("profile name expected");
+				}
+		| PKG_AUTO_SCALING_PROFILE EQUAL ID { IFOR();
+				hg_pkg_profile_name=$3; }
+		| PKG_AUTO_SCALING_PROFILE EQUAL error {
+				yyerror("profile name expected");
+				}
+		| HG_RAM_FLOOR_MB EQUAL NUMBER { IFOR();
+				hg_ram_floor_mb=$3; }
+		| HG_RAM_FLOOR_MB EQUAL error {
+				yyerror("integer value expected");
+				}
+		| HG_AUTOSCALE_DRY_RUN EQUAL NUMBER { IFOR();
+				hg_autoscale_dry_run=$3; }
+		| HG_AUTOSCALE_DRY_RUN EQUAL error {
+				yyerror("integer value expected");
+				}
 		| AUTO_SCALING_CYCLE EQUAL NUMBER { IFOR();
 				auto_scaling_cycle=$3; }
 		| AUTO_SCALING_CYCLE EQUAL error {
@@ -2797,6 +2841,7 @@ static void fill_socket_id(struct listen_param *param, struct socket_id *s)
 	while (s) {
 		s->flags |= param->flags;
 		s->workers = param->workers;
+		s->pin_cpus = param->pin_cpus;
 		s->tos = param->tos;
 		s->auto_scaling_profile = param->auto_scaling_profile;
 		s->tag = param->tag;