mem: HG_MALLOC - large tier: size-bucketed free lists instead of one first-fit walk

The large tier kept every free fragment on one unsorted list and walked
it first-fit under the arena lock. With a million records loaded the
lock histograms put that at 1.5 ms per large allocation on average
(max 1.7 ms, 11 stalls over a millisecond on one sweep) - after the
two-phase commit the single largest holder of the lock left.

Free fragments now live in buckets by floor(log2(size)) with a bitmap
of the non-empty buckets: a request scans only its own bucket and
otherwise takes the head of the first non-empty bucket above it, where
every fragment fits by construction. Chunks are doubly linked so an
emptied chunk unlinks in O(1) instead of a walk of the chunk list.

README 12.7: the receive-queue note now says plainly that the socket
queue is the only buffer (one recvfrom() per datagram, no internal
queue), that maxbuffer is what sizes it and net.core.rmem_max what
bounds it, and that both have to be raised together.

Author: Lt-Flash

mem/README.hg_malloc_v3.md

@@ -852,10 +852,26 @@ A worker parked for tens of milliseconds cannot drain its UDP socket;
 with the default 256 KB receive buffer a burst of ~1,500 INVITE-sized
 datagrams overflows it and the kernel drops (`UdpRcvbufErrors`) — the
 lost-request counts in every elastic run above came from exactly that.
-Raise the core `maxbuffer` (the `SO_RCVBUF` applied to SIP sockets) on
-nodes with large arenas, e.g. `maxbuffer = 4194304`, so a stall the
-allocator has not yet removed becomes latency rather than loss
-(`net.core.rmem_max` must allow it).
+The socket queue is the only buffer between the NIC and the workers
+(OpenSIPS reads one datagram at a time with `recvfrom()`; there is no
+second, internal queue), and its size is what OpenSIPS asks for at
+startup — the core parameter `maxbuffer`, 256 KB by default — bounded by
+what the kernel allows, `net.core.rmem_max`. Both have to move, and
+neither is code:
+
+```
+sysctl -w net.core.rmem_max=8388608      # the kernel's ceiling
+maxbuffer = 8388608                       # opensips.cfg: what the SIP sockets ask for
+```
+
+Raising only the sysctl changes nothing (the sockets still ask for
+256 KB); raising only `maxbuffer` is clamped at the old ceiling. With
+both, a burst that a stall leaves unread parks in the queue and is
+drained afterwards, one datagram at a time — a stall the allocator has
+not yet removed becomes latency rather than loss. All UDP workers of a
+socket share that queue, so a single parked worker never drops anything;
+drops need every worker parked at once, which is what the two-phase
+commit ended.
 
 ### 12.5 The first soak, charted
 

mem/hg_large.c

@@ -22,6 +22,8 @@
 
 #ifdef HG_MALLOC
 
+#include <string.h>
+
 #include "hg_malloc.h"
 #include "hg_large.h"
 #include "hg_arena.h"
@@ -33,6 +35,7 @@
 
 struct hg_large_chunk {
 	struct hg_large_chunk *next;
+	struct hg_large_chunk *prev;   /* T21: O(1) unlink when a chunk empties */
 	unsigned long size;
 	/* the buddy block/run this chunk occupies, INCLUDING this header and
 	 * any rounding slack - what the grid actually lost, as opposed to
@@ -54,21 +57,39 @@ struct hg_large_chunk {
 #define HG_LFRAG_NEXT(f) \
 	((struct hg_lfrag *)(void *)((char *)(f) + HG_LFRAG_HDR + (f)->size))
 
+/* bucket = floor(log2(size)): every fragment in bucket k is >= 2^k, so a
+ * request of need bytes fits ANY fragment in a bucket above its own and
+ * only needs a scan inside its own bucket */
+static inline unsigned int lfrag_bucket(unsigned long size)
+{
+	unsigned int b = size ? (unsigned int)(63 - __builtin_clzl(size)) : 0;
+
+	return b >= HG_LARGE_BUCKETS ? HG_LARGE_BUCKETS - 1 : b;
+}
+
 static inline void lfrag_insert_free(struct hg_block *hb, struct hg_lfrag *frag)
 {
-	frag->prev = &hb->large_free;
-	frag->nxt_free = hb->large_free;
-	if (hb->large_free)
-		hb->large_free->prev = &frag->nxt_free;
-	hb->large_free = frag;
+	unsigned int b = lfrag_bucket(frag->size);
+
+	frag->prev = &hb->large_free[b];
+	frag->nxt_free = hb->large_free[b];
+	if (hb->large_free[b])
+		hb->large_free[b]->prev = &frag->nxt_free;
+	hb->large_free[b] = frag;
+	hb->large_free_map |= 1UL << b;
 }
 
-static inline void lfrag_remove_free(struct hg_lfrag *frag)
+/* call BEFORE changing frag->size: the bucket is derived from it */
+static inline void lfrag_remove_free(struct hg_block *hb, struct hg_lfrag *frag)
 {
+	unsigned int b = lfrag_bucket(frag->size);
+
 	*(frag->prev) = frag->nxt_free;
 	if (frag->nxt_free)
 		frag->nxt_free->prev = frag->prev;
 	frag->prev = NULL;
+	if (!hb->large_free[b])
+		hb->large_free_map &= ~(1UL << b);
 }
 
 #ifdef DBG_MALLOC
@@ -98,15 +119,25 @@ void *hg_large_alloc(struct hg_block *hb, unsigned long size)
 
 	hg_lock_enter(hb, HG_LK_LARGE);
 
-	/* linear first-fit across ONE shared, unsorted free list (large
-	 * allocations are inherently rare and already slow-path - see the
-	 * hg_large.h file header for why this doesn't need f_malloc's
-	 * size-hashed buckets) */
-	f = NULL;
-	for (n = hb->large_free; n; n = n->nxt_free) {
-		if (n->size >= need) {
-			f = n;
-			break;
+	/* T21: first fit inside the request's own size bucket, else the head
+	 * of the first non-empty bucket above it (every fragment there fits
+	 * by construction) - the former single unsorted list cost 1.5 ms per
+	 * allocation under the arena lock with a million records loaded */
+	{
+		unsigned int b = lfrag_bucket(need);
+		unsigned long higher;
+
+		f = NULL;
+		for (n = hb->large_free[b]; n; n = n->nxt_free) {
+			if (n->size >= need) {
+				f = n;
+				break;
+			}
+		}
+		if (!f && b + 1 < HG_LARGE_BUCKETS) {
+			higher = hb->large_free_map & ~((2UL << b) - 1);
+			if (higher)
+				f = hb->large_free[__builtin_ctzl(higher)];
 		}
 	}
 
@@ -194,6 +225,9 @@ void *hg_large_alloc(struct hg_block *hb, unsigned long size)
 		ch->last_frag->prev = NULL;
 
 		ch->next = hb->large_chunks;
+		ch->prev = NULL;
+		if (ch->next)
+			ch->next->prev = ch;
 		hb->large_chunks = ch;
 		ch->backing = backing;
 		ch->ord = chunk_ord;
@@ -218,7 +252,7 @@ void *hg_large_alloc(struct hg_block *hb, unsigned long size)
 		/* f->size is guaranteed >= need: the chunk was sized for it */
 	}
 
-	lfrag_remove_free(f);
+	lfrag_remove_free(hb, f);
 
 	/* split off the remainder if it's big enough to be worth keeping as
 	 * its own free frag (mirrors f_malloc's fm_split_frag threshold) */
@@ -283,15 +317,15 @@ void hg_large_free(struct hg_block *hb, struct hg_lfrag *frag)
 	 * for a chunk's sentinel, so this naturally stops at the boundary */
 	neigh = HG_LFRAG_NEXT(frag);
 	if (neigh->prev) {
-		lfrag_remove_free(neigh);
+		lfrag_remove_free(hb, neigh);
 		frag->size += HG_LFRAG_HDR + neigh->size;
 		HG_LFRAG_NEXT(neigh)->pf = frag;
 	}
 
 	/* backward coalesce */
 	neigh = frag->pf;
 	if (neigh && neigh->prev) {
-		lfrag_remove_free(neigh);
+		lfrag_remove_free(hb, neigh);
 		neigh->size += HG_LFRAG_HDR + frag->size;
 		HG_LFRAG_NEXT(frag)->pf = neigh;
 		frag = neigh;
@@ -319,16 +353,17 @@ void hg_large_free(struct hg_block *hb, struct hg_lfrag *frag)
 			((char *)frag - sizeof(struct hg_large_chunk));
 
 		if (ch->first_frag == frag) {
-			struct hg_large_chunk **pp;
 			void *base = ch;
 			unsigned long backing = ch->backing;
 			int ord = ch->ord;
 
-			for (pp = &hb->large_chunks; *pp; pp = &(*pp)->next)
-				if (*pp == ch) {
-					*pp = ch->next;
-					break;
-				}
+			/* T21: O(1) unlink */
+			if (ch->prev)
+				ch->prev->next = ch->next;
+			else
+				hb->large_chunks = ch->next;
+			if (ch->next)
+				ch->next->prev = ch->prev;
 
 			hb->large_backing -= backing;
 			hb->real_used     -= backing;
@@ -384,7 +419,8 @@ void hg_large_destroy(struct hg_block *hb)
 	 * so there is still nothing to release one chunk at a time - only the
 	 * bookkeeping to drop. */
 	hb->large_chunks = NULL;
-	hb->large_free = NULL;
+	memset(hb->large_free, 0, sizeof hb->large_free);
+	hb->large_free_map = 0;
 	hb->large_backing = 0;
 	hb->large_live = 0;
 }

mem/hg_malloc.h

@@ -432,7 +432,13 @@ struct hg_block {
 	 * list across all of them; hb->lock guards all of it (slow path
 	 * already, no separate lock needed) */
 	struct hg_large_chunk *large_chunks;
-	struct hg_lfrag *large_free;
+	/* T21: free fragments bucketed by floor(log2(size)) with a bitmap of
+	 * the non-empty buckets - a fit is found in O(bucket) instead of a
+	 * first-fit walk of every free fragment, which the lock histograms put
+	 * at 1.5 ms mean per large allocation with a million records loaded */
+#define HG_LARGE_BUCKETS 40
+	struct hg_lfrag *large_free[HG_LARGE_BUCKETS];
+	unsigned long large_free_map;
 	/*
 	 * Backing the large tier holds from the buddy grid, and how much of it
 	 * is handed out right now.  Both are needed because the two answer