Skip to content

Commit 767c9ab

Browse files
committed
cachedb_perf: hand the memory to HG_MALLOC when the core has it
At startup the module now looks at what allocator the core runs and chooses its memory backing instead of always managing chunks itself: memory_backing = auto | core | own-hg | own core the shm allocator is HG_MALLOC: every cell is an HG slab cell, so classes, per-process caches, block GC with re-typing, elastic growth and the maintenance process are HG's; nothing of that is duplicated here own-hg arena_hugepage_mb on an HG_MALLOC build: the dedicated arena is created through the core's module-arena facade as an HG arena of its own (arena_hugepage_mb:arena_hugepage_cap_mb, optional arena_profile), fully managed by HG and listed by name in hg_stats own any other allocator: this module's chunk allocator as before, in shm or in its own reservation auto picks own-hg when an arena is asked for, core when the shm allocator is HG, own otherwise; the explicit values refuse to start when the backing is not available. The readers' safety contract is unchanged: a cell HG frees and re-types fails the extents/class/hash/ key/version checks the same way a stale own-chunk cell does. The perf_stats MI reports the backing in use ("arena.backing") and the startup NOTICE names it.
1 parent b64fae6 commit 767c9ab

3 files changed

Lines changed: 250 additions & 12 deletions

File tree

‎modules/cachedb_perf/cachedb_perf.c‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -635,6 +635,9 @@ static const param_export_t params[] = {
635635
{ "arena_selftest", INT_PARAM, &arena_selftest },
636636
{ "htable_selftest", INT_PARAM, &htable_selftest },
637637
{ "arena_hugepage_mb", INT_PARAM, &pcache_arena_hugepage_mb },
638+
{ "arena_hugepage_cap_mb", INT_PARAM, &pcache_arena_hugepage_cap_mb },
639+
{ "arena_profile", STR_PARAM, &pcache_arena_profile },
640+
{ "memory_backing", STR_PARAM, &pcache_backing_policy },
638641
{ "expiry_sweep_period", INT_PARAM, &expiry_sweep_period },
639642
{ "growth_load_factor", INT_PARAM, &growth_load_factor },
640643
{ "growth_budget", INT_PARAM, &growth_budget },
@@ -1197,7 +1200,10 @@ static mi_response_t *mi_perf_stats(str *col_s)
11971200
goto err;
11981201
pcache_arena_stats(&nchunks, &bytes);
11991202
if (add_mi_number(aobj, MI_SSTR("chunks"), nchunks) < 0 ||
1200-
add_mi_number(aobj, MI_SSTR("bytes"), bytes) < 0)
1203+
add_mi_number(aobj, MI_SSTR("bytes"), bytes) < 0 ||
1204+
add_mi_string(aobj, MI_SSTR("backing"),
1205+
(char *)pcache_arena_backing_str(),
1206+
strlen(pcache_arena_backing_str())) < 0)
12011207
goto err;
12021208

12031209
/* _probe = what this host is CAPABLE of (startup capability check,
@@ -4945,15 +4951,6 @@ static int mod_init(void)
49454951
"tier %d/4 - %s\n",
49464952
pcache_mem.tier, pcache_mem_tier_str(pcache_mem.tier));
49474953

4948-
if (pcache_arena_hugepage_mb > 0)
4949-
LM_NOTICE("memory backing IN USE: a separate %d MB reservation, "
4950-
"OUTSIDE OpenSIPS shared memory (arena_hugepage_mb)\n",
4951-
pcache_arena_hugepage_mb);
4952-
else
4953-
LM_NOTICE("memory backing IN USE: OpenSIPS shared memory "
4954-
"(shm_malloc) - NOT a separate reservation; counted in core's "
4955-
"own shmem: stats, not a cachedb_perf-specific total. Set "
4956-
"arena_hugepage_mb to reserve a dedicated arena instead.\n");
49574954

49584955
switch (pcache_mem.tier) {
49594956
case PCACHE_MEM_HUGETLB:
@@ -4976,11 +4973,14 @@ static int mod_init(void)
49764973
"returned on exit, so nothing is held while unused)\n");
49774974
}
49784975

4979-
/* the slab arena (DESIGN 3.3) - shm globals, pre-fork */
4976+
/* the slab arena (DESIGN 3.3) - shm globals, pre-fork; decides the
4977+
* memory backing (own chunks / the core HG arena / an HG arena of our
4978+
* own) and says which one is in use */
49804979
if (pcache_arena_init() < 0) {
49814980
LM_ERR("failed to init the arena\n");
49824981
return -1;
49834982
}
4983+
pcache_arena_backing_notice();
49844984

49854985
/* CP-11: huge pages were asked for but the arena settled on a lesser
49864986
* tier - flagged now, raised from the first timer tick (EVI has no

‎modules/cachedb_perf/pcache_arena.c‎

Lines changed: 221 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,41 @@
2828
#include "../../mem/mem.h"
2929
#include "../../mem/shm_mem.h"
3030

31+
#include <strings.h>
3132
#include "pcache_arena.h"
3233
#include "pcache_mem.h"
3334

35+
/* the core's module-arena facade (HG_MALLOC v3 trees); older cores have no
36+
* such header - then only the own backing exists */
37+
#if defined(__has_include)
38+
# if __has_include("../../mem/mem_arena.h")
39+
# include "../../mem/mem_arena.h"
40+
# define PCACHE_HAVE_MEM_ARENA 1
41+
# endif
42+
#endif
43+
#ifndef PCACHE_HAVE_MEM_ARENA
44+
typedef void mem_arena_t;
45+
static inline mem_arena_t *shm_arena_create(char *n, unsigned long i,
46+
unsigned long c) { return NULL; }
47+
static inline int shm_arena_set_profile(mem_arena_t *a, const char *p)
48+
{ return -1; }
49+
static inline int shm_allocator_is_hg(void) { return 0; }
50+
static inline mem_arena_t *shm_arena_core(void) { return NULL; }
51+
static inline void mem_arena_extents(const mem_arena_t *a, unsigned long *lo,
52+
unsigned long *hi) { *lo = 0; *hi = 0; }
53+
static inline int mem_arena_tier(const mem_arena_t *a) { return 0; }
54+
static inline void mem_arena_usage(mem_arena_t *a, unsigned long *c,
55+
unsigned long *k, unsigned long *l) { *c = 0; *k = 0; *l = 0; }
56+
#define mem_arena_malloc(a, s) ((void *)0)
57+
#define mem_arena_free(a, p) do { } while (0)
58+
#endif
59+
60+
char *pcache_backing_policy = "auto";
61+
int pcache_arena_hugepage_cap_mb = 0;
62+
char *pcache_arena_profile = NULL;
63+
static int backing = PCACHE_BACKING_OWN;
64+
static mem_arena_t *hg_handle; /* CORE: the shm block; OWN_HG: ours */
65+
3466
/* CP-20: MB to reserve for the huge-page arena; 0 = disabled (shm_malloc).
3567
* Set by the cachedb_perf "arena_hugepage_mb" modparam. */
3668
int pcache_arena_hugepage_mb = 0;
@@ -236,6 +268,66 @@ int pcache_arena_init(void)
236268
size2class[idx] = (unsigned char)c; /* NCLASSES = impossible */
237269
}
238270

271+
/*
272+
* Which backing? An HG_MALLOC build can hand the whole job to HG: a
273+
* dedicated arena becomes an HG arena of our own (own-hg), and with
274+
* no arena asked for, cells go straight into the core shm arena when
275+
* that allocator is HG (core). Only without HG does this file's
276+
* chunked allocator run (own). The policy modparam can force any.
277+
*/
278+
{
279+
const char *pol = pcache_backing_policy ? pcache_backing_policy : "auto";
280+
int want_arena = pcache_arena_hugepage_mb > 0;
281+
int try_own_hg = 0, try_core = 0;
282+
283+
if (!strcasecmp(pol, "auto")) {
284+
try_own_hg = want_arena;
285+
try_core = !want_arena;
286+
} else if (!strcasecmp(pol, "own-hg")) {
287+
try_own_hg = 1;
288+
} else if (!strcasecmp(pol, "core")) {
289+
try_core = 1;
290+
} else if (strcasecmp(pol, "own")) {
291+
LM_ERR("memory_backing '%s' is not auto|core|own-hg|own\n", pol);
292+
return -1;
293+
}
294+
if (try_own_hg) {
295+
unsigned long init = (unsigned long)(pcache_arena_hugepage_mb > 0 ?
296+
pcache_arena_hugepage_mb : 64) << 20;
297+
unsigned long cap = (unsigned long)pcache_arena_hugepage_cap_mb << 20;
298+
299+
hg_handle = shm_arena_create("cachedb_perf", init,
300+
cap > init ? cap : init);
301+
if (hg_handle) {
302+
backing = PCACHE_BACKING_OWN_HG;
303+
if (pcache_arena_profile &&
304+
shm_arena_set_profile(hg_handle, pcache_arena_profile) < 0) {
305+
LM_ERR("arena_profile '%s' could not be attached\n",
306+
pcache_arena_profile);
307+
return -1;
308+
}
309+
} else if (strcasecmp(pol, "auto")) {
310+
LM_ERR("memory_backing=own-hg: no HG_MALLOC arena available "
311+
"(not an HG_MALLOC build, or the reservation failed)\n");
312+
return -1;
313+
}
314+
}
315+
if (backing == PCACHE_BACKING_OWN && try_core) {
316+
hg_handle = shm_arena_core();
317+
if (hg_handle) {
318+
backing = PCACHE_BACKING_CORE;
319+
} else if (strcasecmp(pol, "auto")) {
320+
LM_ERR("memory_backing=core: the shm allocator is not "
321+
"HG_MALLOC\n");
322+
return -1;
323+
}
324+
}
325+
}
326+
if (backing != PCACHE_BACKING_OWN) {
327+
LM_DBG("arena ready: %s\n", pcache_arena_backing_str());
328+
return 0;
329+
}
330+
239331
/* CP-20: reserve the huge-page arena, pre-fork, if requested */
240332
if (pcache_arena_hugepage_mb > 0) {
241333
arena->hsize = (unsigned long)pcache_arena_hugepage_mb << 20;
@@ -266,6 +358,19 @@ void *pcache_region_alloc(size_t size)
266358
unsigned long need = size + sizeof(pcache_region_t) + 64;
267359
char *aligned;
268360

361+
if (backing != PCACHE_BACKING_OWN) {
362+
/* HG hands out whole regions too; nothing to register - the
363+
* arena's extents and accounting are HG's */
364+
rg = mem_arena_malloc(hg_handle, need);
365+
if (!rg) {
366+
LM_ERR("no more arena memory for a %lu byte region\n", need);
367+
return NULL;
368+
}
369+
__atomic_fetch_add(&arena->bytes, need, __ATOMIC_RELAXED);
370+
return (char *)(((unsigned long)rg + sizeof(pcache_region_t) + 63)
371+
& ~63UL);
372+
}
373+
269374
rg = pcache_chunk_backing(need);
270375
if (!rg) {
271376
LM_ERR("no more shm memory for a %lu byte region\n", need);
@@ -296,6 +401,15 @@ void pcache_arena_destroy(void)
296401
if (!arena)
297402
return;
298403

404+
if (backing != PCACHE_BACKING_OWN) {
405+
/* the cells and regions are HG's; the arena mapping (ours or the
406+
* core's) goes with the process */
407+
lock_destroy(&arena->lock);
408+
shm_free(arena);
409+
arena = NULL;
410+
return;
411+
}
412+
299413
/* blocks carved from the huge reservation are part of one mmap - they
300414
* must be munmap'd as a whole (below), never shm_free'd individually */
301415
#define IN_HARENA(_p) (arena->hbase && (char *)(_p) >= arena->hbase && \
@@ -328,7 +442,7 @@ void pcache_arena_child_init(void)
328442
{
329443
struct pcache_palloc *pl = my_palloc;
330444

331-
if (!pl)
445+
if (backing != PCACHE_BACKING_OWN || !pl)
332446
return;
333447

334448
/*
@@ -378,6 +492,18 @@ void *pcache_cell_alloc(unsigned int size)
378492
}
379493
c = size2class[(size + 31) >> 5];
380494

495+
if (backing != PCACHE_BACKING_OWN) {
496+
/* an HG slab cell, class-rounded so pcache_cell_bound() stays
497+
* exact; byte 0 carries our class id exactly as in own chunks
498+
* (HG's own header sits in front of the pointer it returns) */
499+
cell = mem_arena_malloc(hg_handle, cell_sizes[c]);
500+
if (!cell)
501+
return NULL;
502+
*(unsigned char *)cell = (unsigned char)c;
503+
__atomic_fetch_add(&arena->bytes, cell_sizes[c], __ATOMIC_RELAXED);
504+
return cell;
505+
}
506+
381507
pl = get_palloc();
382508
if (!pl)
383509
return NULL;
@@ -438,6 +564,12 @@ void pcache_cell_free(void *cell)
438564
return;
439565
}
440566

567+
if (backing != PCACHE_BACKING_OWN) {
568+
__atomic_fetch_sub(&arena->bytes, cell_sizes[c], __ATOMIC_RELAXED);
569+
mem_arena_free(hg_handle, cell);
570+
return;
571+
}
572+
441573
pl = get_palloc();
442574
if (!pl) {
443575
/* cannot even track it privately - hand it to the pool */
@@ -471,6 +603,11 @@ void pcache_cell_free_global(void *cell)
471603
cell, c);
472604
return;
473605
}
606+
if (backing != PCACHE_BACKING_OWN) {
607+
__atomic_fetch_sub(&arena->bytes, cell_sizes[c], __ATOMIC_RELAXED);
608+
mem_arena_free(hg_handle, cell);
609+
return;
610+
}
474611
lock_get(&arena->lock);
475612
gpool_push(c, cell);
476613
lock_release(&arena->lock);
@@ -487,6 +624,12 @@ unsigned int pcache_cell_bound(const void *cell)
487624

488625
void pcache_arena_extents(unsigned long *lo, unsigned long *hi)
489626
{
627+
if (backing != PCACHE_BACKING_OWN) {
628+
/* the HG reservation (the whole cap): every pointer HG ever
629+
* hands out from this arena lies inside it */
630+
mem_arena_extents(hg_handle, lo, hi);
631+
return;
632+
}
490633
/* unlocked on purpose: both only ever grow outward. A reader mixing
491634
* a fresh lo with a stale hi sees a subset - the check then fails
492635
* closed (treated as invalid) for a moment during a chunk carve */
@@ -496,6 +639,11 @@ void pcache_arena_extents(unsigned long *lo, unsigned long *hi)
496639

497640
void pcache_arena_stats(unsigned int *nchunks, unsigned long *bytes)
498641
{
642+
if (backing != PCACHE_BACKING_OWN) {
643+
*nchunks = 0; /* no chunks of ours: HG's cells */
644+
*bytes = __atomic_load_n(&arena->bytes, __ATOMIC_RELAXED);
645+
return;
646+
}
499647
lock_get(&arena->lock);
500648
*nchunks = arena->nchunks;
501649
*bytes = arena->bytes;
@@ -514,9 +662,65 @@ int pcache_arena_tier(void)
514662
* HG_MALLOC. Returning PCACHE_MEM_4K here reported a property of an
515663
* arena that does not exist, and was read live as "the cache is on
516664
* small pages" while it was actually on hugepages. */
665+
if (backing == PCACHE_BACKING_OWN_HG)
666+
return mem_arena_tier(hg_handle); /* same enum values */
667+
if (backing == PCACHE_BACKING_CORE)
668+
return PCACHE_MEM_NO_ARENA; /* the core arena's tier */
517669
return arena->hbase ? (int)arena->htier : PCACHE_MEM_NO_ARENA;
518670
}
519671

672+
int pcache_arena_backing(void)
673+
{
674+
return backing;
675+
}
676+
677+
const char *pcache_arena_backing_str(void)
678+
{
679+
switch (backing) {
680+
case PCACHE_BACKING_CORE:
681+
return "core (HG_MALLOC shm arena cells)";
682+
case PCACHE_BACKING_OWN_HG:
683+
return "own arena managed by HG_MALLOC";
684+
default:
685+
return arena && arena->hbase ? "own chunks in a dedicated reservation"
686+
: "own chunks in shm";
687+
}
688+
}
689+
690+
void pcache_arena_backing_notice(void)
691+
{
692+
unsigned long committed, cap, live;
693+
694+
switch (backing) {
695+
case PCACHE_BACKING_CORE:
696+
LM_NOTICE("memory backing IN USE: the core HG_MALLOC shm arena - "
697+
"every cache cell is an HG slab cell (classes, per-process "
698+
"caches, GC and re-typing, growth and maintenance are HG's); "
699+
"counted in core's shmem: stats%s\n",
700+
pcache_arena_hugepage_mb > 0 ? " (arena_hugepage_mb ignored "
701+
"under memory_backing=core)" : "");
702+
break;
703+
case PCACHE_BACKING_OWN_HG:
704+
mem_arena_usage(hg_handle, &committed, &cap, &live);
705+
LM_NOTICE("memory backing IN USE: a dedicated arena managed by "
706+
"HG_MALLOC - %lu MB committed, can grow to %lu MB%s; see "
707+
"hg_stats 'cachedb_perf'\n", committed >> 20, cap >> 20,
708+
pcache_arena_profile ? " under its auto-scaling profile" : "");
709+
break;
710+
default:
711+
if (pcache_arena_hugepage_mb > 0)
712+
LM_NOTICE("memory backing IN USE: a separate %d MB reservation, "
713+
"OUTSIDE OpenSIPS shared memory (arena_hugepage_mb), "
714+
"cachedb_perf's own chunk allocator\n",
715+
pcache_arena_hugepage_mb);
716+
else
717+
LM_NOTICE("memory backing IN USE: OpenSIPS shared memory "
718+
"(shm_malloc), cachedb_perf's own chunk allocator - NOT a "
719+
"separate reservation; counted in core's own shmem: stats. "
720+
"Set arena_hugepage_mb to reserve a dedicated arena.\n");
721+
}
722+
}
723+
520724
/*
521725
* Capacity of the DEDICATED arena_hugepage_mb reservation specifically -
522726
* distinct from pcache_arena_stats()'s bytes/nchunks, which count total
@@ -528,9 +732,25 @@ int pcache_arena_tier(void)
528732
* 0/0/0 alone cannot distinguish "no dedicated reservation exists" from
529733
* "a reservation exists and happens to be still empty".
530734
*/
735+
static void pcache_arena_hg_capacity(int *active, unsigned long *total,
736+
unsigned long *used, unsigned long *free)
737+
{
738+
unsigned long committed, cap, live;
739+
740+
mem_arena_usage(hg_handle, &committed, &cap, &live);
741+
*active = 1;
742+
*total = cap;
743+
*used = live;
744+
*free = cap > live ? cap - live : 0;
745+
}
746+
531747
void pcache_arena_hugepage_capacity(int *active, unsigned long *total,
532748
unsigned long *used, unsigned long *free)
533749
{
750+
if (backing == PCACHE_BACKING_OWN_HG) {
751+
pcache_arena_hg_capacity(active, total, used, free);
752+
return;
753+
}
534754
if (!arena->hbase) {
535755
*active = 0;
536756
*total = 0;

‎modules/cachedb_perf/pcache_arena.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,24 @@
4848
#define PCACHE_CELL_MAX 65536 /* largest cell; bigger allocs fail (v1) */
4949
#define PCACHE_NCLASSES 21
5050

51+
/* Memory backing, decided in pcache_arena_init() (mod_init, pre-fork):
52+
* PCACHE_BACKING_OWN - this file's chunked allocator (chunks from
53+
* shm_malloc or the dedicated reservation)
54+
* PCACHE_BACKING_CORE - the core shm allocator is HG_MALLOC: every cell
55+
* is an HG slab cell in the shm arena
56+
* PCACHE_BACKING_OWN_HG - arena_hugepage_mb set on an HG_MALLOC build: the
57+
* arena is an HG arena of its own, fully managed
58+
* by HG (classes, GC, growth/shrink, maintenance)
59+
* Policy: the "memory_backing" modparam (auto|core|own-hg|own). */
60+
enum pcache_backing { PCACHE_BACKING_OWN = 0, PCACHE_BACKING_CORE,
61+
PCACHE_BACKING_OWN_HG };
62+
extern char *pcache_backing_policy; /* modparam memory_backing */
63+
extern int pcache_arena_hugepage_cap_mb; /* modparam, 0 = fixed */
64+
extern char *pcache_arena_profile; /* modparam arena_profile */
65+
int pcache_arena_backing(void);
66+
const char *pcache_arena_backing_str(void);
67+
void pcache_arena_backing_notice(void);
68+
5169
int pcache_arena_init(void);
5270
void pcache_arena_destroy(void);
5371

0 commit comments

Comments
 (0)