2828#include "../../mem/mem.h"
2929#include "../../mem/shm_mem.h"
3030
31+ #include <strings.h>
3132#include "pcache_arena.h"
3233#include "pcache_mem.h"
3334
35+ /* the core's module-arena facade (HG_MALLOC v3 trees); older cores have no
36+ * such header - then only the own backing exists */
37+ #if defined(__has_include )
38+ # if __has_include ("../../mem/mem_arena.h" )
39+ # include "../../mem/mem_arena.h"
40+ # define PCACHE_HAVE_MEM_ARENA 1
41+ # endif
42+ #endif
43+ #ifndef PCACHE_HAVE_MEM_ARENA
44+ typedef void mem_arena_t ;
45+ static inline mem_arena_t * shm_arena_create (char * n , unsigned long i ,
46+ unsigned long c ) { return NULL ; }
47+ static inline int shm_arena_set_profile (mem_arena_t * a , const char * p )
48+ { return -1 ; }
49+ static inline int shm_allocator_is_hg (void ) { return 0 ; }
50+ static inline mem_arena_t * shm_arena_core (void ) { return NULL ; }
51+ static inline void mem_arena_extents (const mem_arena_t * a , unsigned long * lo ,
52+ unsigned long * hi ) { * lo = 0 ; * hi = 0 ; }
53+ static inline int mem_arena_tier (const mem_arena_t * a ) { return 0 ; }
54+ static inline void mem_arena_usage (mem_arena_t * a , unsigned long * c ,
55+ unsigned long * k , unsigned long * l ) { * c = 0 ; * k = 0 ; * l = 0 ; }
56+ #define mem_arena_malloc (a , s ) ((void *)0)
57+ #define mem_arena_free (a , p ) do { } while (0)
58+ #endif
59+
60+ char * pcache_backing_policy = "auto" ;
61+ int pcache_arena_hugepage_cap_mb = 0 ;
62+ char * pcache_arena_profile = NULL ;
63+ static int backing = PCACHE_BACKING_OWN ;
64+ static mem_arena_t * hg_handle ; /* CORE: the shm block; OWN_HG: ours */
65+
3466/* CP-20: MB to reserve for the huge-page arena; 0 = disabled (shm_malloc).
3567 * Set by the cachedb_perf "arena_hugepage_mb" modparam. */
3668int pcache_arena_hugepage_mb = 0 ;
@@ -236,6 +268,66 @@ int pcache_arena_init(void)
236268 size2class [idx ] = (unsigned char )c ; /* NCLASSES = impossible */
237269 }
238270
271+ /*
272+ * Which backing? An HG_MALLOC build can hand the whole job to HG: a
273+ * dedicated arena becomes an HG arena of our own (own-hg), and with
274+ * no arena asked for, cells go straight into the core shm arena when
275+ * that allocator is HG (core). Only without HG does this file's
276+ * chunked allocator run (own). The policy modparam can force any.
277+ */
278+ {
279+ const char * pol = pcache_backing_policy ? pcache_backing_policy : "auto" ;
280+ int want_arena = pcache_arena_hugepage_mb > 0 ;
281+ int try_own_hg = 0 , try_core = 0 ;
282+
283+ if (!strcasecmp (pol , "auto" )) {
284+ try_own_hg = want_arena ;
285+ try_core = !want_arena ;
286+ } else if (!strcasecmp (pol , "own-hg" )) {
287+ try_own_hg = 1 ;
288+ } else if (!strcasecmp (pol , "core" )) {
289+ try_core = 1 ;
290+ } else if (strcasecmp (pol , "own" )) {
291+ LM_ERR ("memory_backing '%s' is not auto|core|own-hg|own\n" , pol );
292+ return -1 ;
293+ }
294+ if (try_own_hg ) {
295+ unsigned long init = (unsigned long )(pcache_arena_hugepage_mb > 0 ?
296+ pcache_arena_hugepage_mb : 64 ) << 20 ;
297+ unsigned long cap = (unsigned long )pcache_arena_hugepage_cap_mb << 20 ;
298+
299+ hg_handle = shm_arena_create ("cachedb_perf" , init ,
300+ cap > init ? cap : init );
301+ if (hg_handle ) {
302+ backing = PCACHE_BACKING_OWN_HG ;
303+ if (pcache_arena_profile &&
304+ shm_arena_set_profile (hg_handle , pcache_arena_profile ) < 0 ) {
305+ LM_ERR ("arena_profile '%s' could not be attached\n" ,
306+ pcache_arena_profile );
307+ return -1 ;
308+ }
309+ } else if (strcasecmp (pol , "auto" )) {
310+ LM_ERR ("memory_backing=own-hg: no HG_MALLOC arena available "
311+ "(not an HG_MALLOC build, or the reservation failed)\n" );
312+ return -1 ;
313+ }
314+ }
315+ if (backing == PCACHE_BACKING_OWN && try_core ) {
316+ hg_handle = shm_arena_core ();
317+ if (hg_handle ) {
318+ backing = PCACHE_BACKING_CORE ;
319+ } else if (strcasecmp (pol , "auto" )) {
320+ LM_ERR ("memory_backing=core: the shm allocator is not "
321+ "HG_MALLOC\n" );
322+ return -1 ;
323+ }
324+ }
325+ }
326+ if (backing != PCACHE_BACKING_OWN ) {
327+ LM_DBG ("arena ready: %s\n" , pcache_arena_backing_str ());
328+ return 0 ;
329+ }
330+
239331 /* CP-20: reserve the huge-page arena, pre-fork, if requested */
240332 if (pcache_arena_hugepage_mb > 0 ) {
241333 arena -> hsize = (unsigned long )pcache_arena_hugepage_mb << 20 ;
@@ -266,6 +358,19 @@ void *pcache_region_alloc(size_t size)
266358 unsigned long need = size + sizeof (pcache_region_t ) + 64 ;
267359 char * aligned ;
268360
361+ if (backing != PCACHE_BACKING_OWN ) {
362+ /* HG hands out whole regions too; nothing to register - the
363+ * arena's extents and accounting are HG's */
364+ rg = mem_arena_malloc (hg_handle , need );
365+ if (!rg ) {
366+ LM_ERR ("no more arena memory for a %lu byte region\n" , need );
367+ return NULL ;
368+ }
369+ __atomic_fetch_add (& arena -> bytes , need , __ATOMIC_RELAXED );
370+ return (char * )(((unsigned long )rg + sizeof (pcache_region_t ) + 63 )
371+ & ~63UL );
372+ }
373+
269374 rg = pcache_chunk_backing (need );
270375 if (!rg ) {
271376 LM_ERR ("no more shm memory for a %lu byte region\n" , need );
@@ -296,6 +401,15 @@ void pcache_arena_destroy(void)
296401 if (!arena )
297402 return ;
298403
404+ if (backing != PCACHE_BACKING_OWN ) {
405+ /* the cells and regions are HG's; the arena mapping (ours or the
406+ * core's) goes with the process */
407+ lock_destroy (& arena -> lock );
408+ shm_free (arena );
409+ arena = NULL ;
410+ return ;
411+ }
412+
299413 /* blocks carved from the huge reservation are part of one mmap - they
300414 * must be munmap'd as a whole (below), never shm_free'd individually */
301415#define IN_HARENA (_p ) (arena->hbase && (char *)(_p) >= arena->hbase && \
@@ -328,7 +442,7 @@ void pcache_arena_child_init(void)
328442{
329443 struct pcache_palloc * pl = my_palloc ;
330444
331- if (!pl )
445+ if (backing != PCACHE_BACKING_OWN || !pl )
332446 return ;
333447
334448 /*
@@ -378,6 +492,18 @@ void *pcache_cell_alloc(unsigned int size)
378492 }
379493 c = size2class [(size + 31 ) >> 5 ];
380494
495+ if (backing != PCACHE_BACKING_OWN ) {
496+ /* an HG slab cell, class-rounded so pcache_cell_bound() stays
497+ * exact; byte 0 carries our class id exactly as in own chunks
498+ * (HG's own header sits in front of the pointer it returns) */
499+ cell = mem_arena_malloc (hg_handle , cell_sizes [c ]);
500+ if (!cell )
501+ return NULL ;
502+ * (unsigned char * )cell = (unsigned char )c ;
503+ __atomic_fetch_add (& arena -> bytes , cell_sizes [c ], __ATOMIC_RELAXED );
504+ return cell ;
505+ }
506+
381507 pl = get_palloc ();
382508 if (!pl )
383509 return NULL ;
@@ -438,6 +564,12 @@ void pcache_cell_free(void *cell)
438564 return ;
439565 }
440566
567+ if (backing != PCACHE_BACKING_OWN ) {
568+ __atomic_fetch_sub (& arena -> bytes , cell_sizes [c ], __ATOMIC_RELAXED );
569+ mem_arena_free (hg_handle , cell );
570+ return ;
571+ }
572+
441573 pl = get_palloc ();
442574 if (!pl ) {
443575 /* cannot even track it privately - hand it to the pool */
@@ -471,6 +603,11 @@ void pcache_cell_free_global(void *cell)
471603 cell , c );
472604 return ;
473605 }
606+ if (backing != PCACHE_BACKING_OWN ) {
607+ __atomic_fetch_sub (& arena -> bytes , cell_sizes [c ], __ATOMIC_RELAXED );
608+ mem_arena_free (hg_handle , cell );
609+ return ;
610+ }
474611 lock_get (& arena -> lock );
475612 gpool_push (c , cell );
476613 lock_release (& arena -> lock );
@@ -487,6 +624,12 @@ unsigned int pcache_cell_bound(const void *cell)
487624
488625void pcache_arena_extents (unsigned long * lo , unsigned long * hi )
489626{
627+ if (backing != PCACHE_BACKING_OWN ) {
628+ /* the HG reservation (the whole cap): every pointer HG ever
629+ * hands out from this arena lies inside it */
630+ mem_arena_extents (hg_handle , lo , hi );
631+ return ;
632+ }
490633 /* unlocked on purpose: both only ever grow outward. A reader mixing
491634 * a fresh lo with a stale hi sees a subset - the check then fails
492635 * closed (treated as invalid) for a moment during a chunk carve */
@@ -496,6 +639,11 @@ void pcache_arena_extents(unsigned long *lo, unsigned long *hi)
496639
497640void pcache_arena_stats (unsigned int * nchunks , unsigned long * bytes )
498641{
642+ if (backing != PCACHE_BACKING_OWN ) {
643+ * nchunks = 0 ; /* no chunks of ours: HG's cells */
644+ * bytes = __atomic_load_n (& arena -> bytes , __ATOMIC_RELAXED );
645+ return ;
646+ }
499647 lock_get (& arena -> lock );
500648 * nchunks = arena -> nchunks ;
501649 * bytes = arena -> bytes ;
@@ -514,9 +662,65 @@ int pcache_arena_tier(void)
514662 * HG_MALLOC. Returning PCACHE_MEM_4K here reported a property of an
515663 * arena that does not exist, and was read live as "the cache is on
516664 * small pages" while it was actually on hugepages. */
665+ if (backing == PCACHE_BACKING_OWN_HG )
666+ return mem_arena_tier (hg_handle ); /* same enum values */
667+ if (backing == PCACHE_BACKING_CORE )
668+ return PCACHE_MEM_NO_ARENA ; /* the core arena's tier */
517669 return arena -> hbase ? (int )arena -> htier : PCACHE_MEM_NO_ARENA ;
518670}
519671
672+ int pcache_arena_backing (void )
673+ {
674+ return backing ;
675+ }
676+
677+ const char * pcache_arena_backing_str (void )
678+ {
679+ switch (backing ) {
680+ case PCACHE_BACKING_CORE :
681+ return "core (HG_MALLOC shm arena cells)" ;
682+ case PCACHE_BACKING_OWN_HG :
683+ return "own arena managed by HG_MALLOC" ;
684+ default :
685+ return arena && arena -> hbase ? "own chunks in a dedicated reservation"
686+ : "own chunks in shm" ;
687+ }
688+ }
689+
690+ void pcache_arena_backing_notice (void )
691+ {
692+ unsigned long committed , cap , live ;
693+
694+ switch (backing ) {
695+ case PCACHE_BACKING_CORE :
696+ LM_NOTICE ("memory backing IN USE: the core HG_MALLOC shm arena - "
697+ "every cache cell is an HG slab cell (classes, per-process "
698+ "caches, GC and re-typing, growth and maintenance are HG's); "
699+ "counted in core's shmem: stats%s\n" ,
700+ pcache_arena_hugepage_mb > 0 ? " (arena_hugepage_mb ignored "
701+ "under memory_backing=core)" : "" );
702+ break ;
703+ case PCACHE_BACKING_OWN_HG :
704+ mem_arena_usage (hg_handle , & committed , & cap , & live );
705+ LM_NOTICE ("memory backing IN USE: a dedicated arena managed by "
706+ "HG_MALLOC - %lu MB committed, can grow to %lu MB%s; see "
707+ "hg_stats 'cachedb_perf'\n" , committed >> 20 , cap >> 20 ,
708+ pcache_arena_profile ? " under its auto-scaling profile" : "" );
709+ break ;
710+ default :
711+ if (pcache_arena_hugepage_mb > 0 )
712+ LM_NOTICE ("memory backing IN USE: a separate %d MB reservation, "
713+ "OUTSIDE OpenSIPS shared memory (arena_hugepage_mb), "
714+ "cachedb_perf's own chunk allocator\n" ,
715+ pcache_arena_hugepage_mb );
716+ else
717+ LM_NOTICE ("memory backing IN USE: OpenSIPS shared memory "
718+ "(shm_malloc), cachedb_perf's own chunk allocator - NOT a "
719+ "separate reservation; counted in core's own shmem: stats. "
720+ "Set arena_hugepage_mb to reserve a dedicated arena.\n" );
721+ }
722+ }
723+
520724/*
521725 * Capacity of the DEDICATED arena_hugepage_mb reservation specifically -
522726 * distinct from pcache_arena_stats()'s bytes/nchunks, which count total
@@ -528,9 +732,25 @@ int pcache_arena_tier(void)
528732 * 0/0/0 alone cannot distinguish "no dedicated reservation exists" from
529733 * "a reservation exists and happens to be still empty".
530734 */
735+ static void pcache_arena_hg_capacity (int * active , unsigned long * total ,
736+ unsigned long * used , unsigned long * free )
737+ {
738+ unsigned long committed , cap , live ;
739+
740+ mem_arena_usage (hg_handle , & committed , & cap , & live );
741+ * active = 1 ;
742+ * total = cap ;
743+ * used = live ;
744+ * free = cap > live ? cap - live : 0 ;
745+ }
746+
531747void pcache_arena_hugepage_capacity (int * active , unsigned long * total ,
532748 unsigned long * used , unsigned long * free )
533749{
750+ if (backing == PCACHE_BACKING_OWN_HG ) {
751+ pcache_arena_hg_capacity (active , total , used , free );
752+ return ;
753+ }
534754 if (!arena -> hbase ) {
535755 * active = 0 ;
536756 * total = 0 ;
0 commit comments