From 55c9b07d030066a9001c890ad487190584818d3c Mon Sep 17 00:00:00 2001 From: Roy Babayov Date: Thu, 6 Feb 2025 11:52:28 +0000 Subject: [PATCH] Adding an option for monitoring infra in libntirpc The infra added is extracted based on the existing infra in Ganesha: https://github.com/nfs-ganesha/nfs-ganesha/tree/10260ba/src/monitoring The infra added includes several improvements over the existing code: * Better separation of modules - only extracted the metrics infra needed for adding metrics in libntirpc. * No longer requires the prometheus-cpp-lite submodule to exist during build when building with USE_MONITORING=OFF. * Improved the noop implementation for USE_MONITORING=OFF. Consumers of the infra no longer need to check the USE_MONITORING flag (will run noop calls when flag is off) Change-Id: Ie3c68f58439935f3fcfce2cba291b5c1a217d642 Signed-off-by: Roy Babayov --- .gitmodules | 3 + CMakeLists.txt | 5 +- config-h.in.cmake | 1 + src/CMakeLists.txt | 12 +- src/monitoring/CMakeLists.txt | 15 ++ src/monitoring/include/monitoring.h | 241 ++++++++++++++++++++++++++++ src/monitoring/monitoring.cc | 191 ++++++++++++++++++++++ src/monitoring/prometheus-cpp-lite | 1 + 8 files changed, 467 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 100644 src/monitoring/CMakeLists.txt create mode 100644 src/monitoring/include/monitoring.h create mode 100644 src/monitoring/monitoring.cc create mode 160000 src/monitoring/prometheus-cpp-lite diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..f1a35d38a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/monitoring/prometheus-cpp-lite"] + path = src/monitoring/prometheus-cpp-lite + url = https://github.com/biaks/prometheus-cpp-lite.git diff --git a/CMakeLists.txt b/CMakeLists.txt index c3d784374..e6b154df2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/") include(${CMAKE_SOURCE_DIR}/cmake/maintainer_mode.cmake) -project(NTIRPC C) +project(NTIRPC C CXX) # version numbers set(NTIRPC_MAJOR_VERSION 6) @@ -77,6 +77,8 @@ if (USE_GSS) endif(KRB5_FOUND) endif (USE_GSS) +option(USE_MONITORING "Enable monitoring stack" ON) + option(USE_RPC_RDMA "platform supports RDMA" OFF) if (USE_RPC_RDMA) find_package(RDMA REQUIRED) @@ -260,6 +262,7 @@ message(STATUS "USE_RPC_RDMA = ${USE_RPC_RDMA}") message(STATUS "USE_GSS = ${USE_GSS}") message(STATUS "USE_PROFILE = ${USE_PROFILE}") message(STATUS "USE_LTTNG_NTIRPC = ${USE_LTTNG_NTIRPC}") +message(STATUS "USE_MONITORING = ${USE_MONITORING}") #force command line options to be stored in cache set(_MSPAC_SUPPORT ${_MSPAC_SUPPORT} diff --git a/config-h.in.cmake b/config-h.in.cmake index 69bf5bf08..ddf57fef9 100644 --- a/config-h.in.cmake +++ b/config-h.in.cmake @@ -20,6 +20,7 @@ #cmakedefine TIRPC_EPOLL 1 #cmakedefine USE_RPC_RDMA 1 #cmakedefine USE_LTTNG_NTIRPC 1 +#cmakedefine USE_MONITORING 1 /* Package stuff */ #define PACKAGE "libntirpc" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fd3afc594..0cff36b2a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -92,6 +92,12 @@ if(USE_LTTNG_NTIRPC) add_subdirectory(lttng) endif(USE_LTTNG_NTIRPC) +# Monitoring stack. +LIST(APPEND CMAKE_LIBRARY_PATH "${CMAKE_CURRENT_SOURCE_DIR}/monitoring/") +LINK_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/monitoring/") +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/monitoring/include") +add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/monitoring/") + # declares the library add_library(ntirpc SHARED ${ntirpc_common_SRCS} @@ -101,7 +107,11 @@ add_library(ntirpc SHARED # add required libraries--for Ganesha build, it's ok for them to # propagate (i.e., omit PRIVATE) -target_link_libraries(ntirpc ${SYSTEM_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(ntirpc + ${SYSTEM_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} + ntirpcmonitoring +) if(USE_LTTNG) target_link_libraries(ntirpc ntirpc_lttng) diff --git a/src/monitoring/CMakeLists.txt b/src/monitoring/CMakeLists.txt new file mode 100644 index 000000000..6550bb199 --- /dev/null +++ b/src/monitoring/CMakeLists.txt @@ -0,0 +1,15 @@ +########### next target ############### + +set(CMAKE_CXX_STANDARD 17) + +SET(ntirpcmonitoring_SRCS + monitoring.cc +) + +add_library(ntirpcmonitoring ${ntirpcmonitoring_SRCS}) +add_sanitizers(ntirpcmonitoring) +set_target_properties(ntirpcmonitoring PROPERTIES COMPILE_FLAGS "-fPIC") +target_include_directories(ntirpcmonitoring PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/prometheus-cpp-lite/core/include) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic-errors -Werror -Wall -Wextra") + +########### install files ############### diff --git a/src/monitoring/include/monitoring.h b/src/monitoring/include/monitoring.h new file mode 100644 index 000000000..8ea3014cd --- /dev/null +++ b/src/monitoring/include/monitoring.h @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * vim:noexpandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) Google Inc., 2025 + * Author: Roy Babayov roybabayov@google.com + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See + * the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/** + * @brief Monitoring infra for libntirpc. + * + * This file contains C wrappers for prometheus-cpp-lite to enable metrics + * handling. + * + * We avoid using float/double values since updating them *atomically* also + * affects performance. + * + * Usage: + * - Create a static metric: + * my_handle = monitoring__register_counter(...); + * - Update the metric during running time: + * monitoring__counter_inc(my_handle, 1); + * + * Naming convention: + * For new metrics, please use "__", for example: + * "clients__lease_expire_count" + * + * See more: + * - https://prometheus.io/docs/concepts/data_model/ + * - https://prometheus.io/docs/concepts/metric_types/ + */ + +#ifndef NTIRPC_MONITORING_H +#define NTIRPC_MONITORING_H + +#include +#include +#include +#include + +#include "config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Metric help description */ +typedef struct metric_metadata { + const char *description; /* Helper message */ + const char *unit; /* Units like: second, byte */ +} metric_metadata_t; + +/* Label is a dimension in the metric family, for example "operation=GETATTR" */ +typedef struct metric_label { + const char *key; + const char *value; +} metric_label_t; + +/* Buckets of (a,b,c) mean boundaries of: (-INF,a) [a,b) [b,c) [c, INF) */ +typedef struct histogram_buckets { + const int64_t *buckets; + uint16_t count; +} histogram_buckets_t; + +/* C wrapper for prometheus::Counter pointer */ +typedef struct counter_metric_handle { + void *metric; +} counter_metric_handle_t; + +/* C wrapper for prometheus::Gauge pointer */ +typedef struct gauge_metric_handle { + void *metric; +} gauge_metric_handle_t; + +/* C wrapper for prometheus::Histogram pointer */ +typedef struct histogram_metric_handle { + void *metric; +} histogram_metric_handle_t; + +/* C wrapper for prometheus::Registry pointer */ +typedef struct prometheus_registry_handle { + void *registry; +} prometheus_registry_handle_t; + +/* Metric value units. */ +#define METRIC_UNIT_NONE (NULL) +#define METRIC_UNIT_MINUTE ("minute") +#define METRIC_UNIT_SECOND ("sec") +#define METRIC_UNIT_MILLISECOND ("ms") +#define METRIC_UNIT_MICROSECOND ("us") +#define METRIC_UNIT_NANOSECOND ("ns") + +#define METRIC_METADATA(DESCRIPTION, UNIT) \ + ((metric_metadata_t){ .description = (DESCRIPTION), .unit = (UNIT) }) + +#define METRIC_LABEL(KEY, VALUE) \ + ((metric_label_t){ .key = (KEY), .value = (VALUE) }) + +#ifdef USE_MONITORING + +/* Registers and initializes a new static counter metric. */ +counter_metric_handle_t +monitoring__register_counter(const char *name, metric_metadata_t metadata, + const metric_label_t *labels, uint16_t num_labels); + +/* Registers and initializes a new static gauge metric. */ +gauge_metric_handle_t monitoring__register_gauge(const char *name, + metric_metadata_t metadata, + const metric_label_t *labels, + uint16_t num_labels); + +/* Registers and initializes a new static histogram metric. */ +histogram_metric_handle_t +monitoring__register_histogram(const char *name, metric_metadata_t metadata, + const metric_label_t *labels, + uint16_t num_labels, + histogram_buckets_t buckets); + +/* Increments counter metric by value. */ +void monitoring__counter_inc(counter_metric_handle_t, int64_t val); + +/* Increments gauge metric by value. */ +void monitoring__gauge_inc(gauge_metric_handle_t, int64_t val); + +/* Decrements gauge metric by value. */ +void monitoring__gauge_dec(gauge_metric_handle_t, int64_t val); + +/* Sets gauge metric value. */ +void monitoring__gauge_set(gauge_metric_handle_t, int64_t val); + +/* Observes a histogram metric value */ +void monitoring__histogram_observe(histogram_metric_handle_t, int64_t val); + +/* Returns default exp2 histogram buckets. */ +histogram_buckets_t monitoring__buckets_exp2(void); + +/* Returns compact exp2 histogram buckets (fewer compared to default). */ +histogram_buckets_t monitoring__buckets_exp2_compact(void); + +/* Returns handle for metrics registry */ +prometheus_registry_handle_t monitoring__get_registry_handle(void); + +#else /* USE_MONITORING */ + +#ifndef UNUSED +#define UNUSED_ATTR __attribute__((unused)) +#define UNUSED(...) UNUSED_(__VA_ARGS__) +#define UNUSED_(arg) NOT_USED_##arg UNUSED_ATTR +#endif + +static inline histogram_buckets_t monitoring__buckets_exp2(void) +{ + histogram_buckets_t dummy_ret = {}; + return dummy_ret; +} + +static inline histogram_buckets_t monitoring__buckets_exp2_compact(void) +{ + histogram_buckets_t dummy_ret = {}; + return dummy_ret; +} + +static inline counter_metric_handle_t monitoring__register_counter( + const char *UNUSED(name), metric_metadata_t UNUSED(metadata), + const metric_label_t *UNUSED(labels), uint16_t UNUSED(num_labels)) +{ + counter_metric_handle_t dummy_ret = {}; + return dummy_ret; +} + +static inline gauge_metric_handle_t monitoring__register_gauge( + const char *UNUSED(name), metric_metadata_t UNUSED(metadata), + const metric_label_t *UNUSED(labels), uint16_t UNUSED(num_labels)) +{ + gauge_metric_handle_t dummy_ret = {}; + return dummy_ret; +} + +static inline histogram_metric_handle_t monitoring__register_histogram( + const char *UNUSED(name), metric_metadata_t UNUSED(metadata), + const metric_label_t *UNUSED(labels), uint16_t UNUSED(num_labels), + histogram_buckets_t UNUSED(buckets)) +{ + histogram_metric_handle_t dummy_ret = {}; + return dummy_ret; +} + +static inline void +monitoring__counter_inc(counter_metric_handle_t UNUSED(handle), + int64_t UNUSED(value)) +{ +} + +static inline void monitoring__gauge_inc(gauge_metric_handle_t UNUSED(handle), + int64_t UNUSED(value)) +{ +} + +static inline void monitoring__gauge_dec(gauge_metric_handle_t UNUSED(handle), + int64_t UNUSED(value)) +{ +} + +static inline void monitoring__gauge_set(gauge_metric_handle_t UNUSED(handle), + int64_t UNUSED(value)) +{ +} + +static inline void +monitoring__histogram_observe(histogram_metric_handle_t UNUSED(handle), + int64_t UNUSED(value)) +{ +} + +static inline prometheus_registry_handle_t monitoring__get_registry_handle(void) +{ + prometheus_registry_handle_t dummy_ret = {}; + return dummy_ret; +} + +#endif /* USE_MONITORING */ + +#ifdef __cplusplus +} +#endif + +#endif /* NTIRPC_MONITORING_H */ diff --git a/src/monitoring/monitoring.cc b/src/monitoring/monitoring.cc new file mode 100644 index 000000000..3b2ab1922 --- /dev/null +++ b/src/monitoring/monitoring.cc @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * vim:noexpandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) Google Inc., 2025 + * Author: Roy Babayov roybabayov@google.com + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See + * the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "monitoring.h" + +#ifdef USE_MONITORING + +#include "prometheus/counter.h" +#include "prometheus/gauge.h" +#include "prometheus/histogram.h" + +namespace ntirpc_monitoring +{ + +using CounterInt = prometheus::Counter; +using GaugeInt = prometheus::Gauge; +using HistogramInt = prometheus::Histogram; +using HistogramDouble = prometheus::Histogram; +using LabelsMap = std::map; + +static prometheus::Registry registry; + +/** + * @brief Formats full description from metadata into output buffer. + * + * @note description character array needs to be provided by the caller, + * the function populates a string into the character array. + */ +static const std::string get_description(const metric_metadata_t &metadata) +{ + std::ostringstream description; + description << metadata.description; + if (metadata.unit != METRIC_UNIT_NONE) { + description << " [" << metadata.unit << "]"; + } + + return description.str(); +} + +static const LabelsMap get_labels(const metric_label_t *labels, + uint16_t num_labels) +{ + LabelsMap labels_map; + for (uint16_t i = 0; i < num_labels; i++) { + labels_map.emplace(labels[i].key, labels[i].value); + } + return labels_map; +} + +template static X convert_to_handle(Y *metric) +{ + void *const ptr = static_cast(metric); + return { ptr }; +} + +template static X *convert_from_handle(Y handle) +{ + void *const ptr = handle.metric; + return static_cast(ptr); +} + +extern "C" { + +histogram_buckets_t monitoring__buckets_exp2(void) +{ + static const int64_t buckets[] = { + 1, 2, 4, 8, 16, 32, + 64, 128, 256, 512, 1024, 2048, + 4096, 8192, 16384, 32768, 65536, 131072, + 262144, 524288, 1048576, 2097152, 4194304, 8388608, + 16777216, 33554432, 67108864, 134217728, 268435456, 536870912, + 1073741824 + }; + return { buckets, sizeof(buckets) / sizeof(*buckets) }; +} + +histogram_buckets_t monitoring__buckets_exp2_compact(void) +{ + static const int64_t buckets[] = { 10, 20, 40, 80, + 160, 320, 640, 1280, + 2560, 5120, 10240, 20480, + 40960, 81920, 163840, 327680 }; + return { buckets, sizeof(buckets) / sizeof(*buckets) }; +} + +counter_metric_handle_t +monitoring__register_counter(const char *name, metric_metadata_t metadata, + const metric_label_t *labels, uint16_t num_labels) +{ + auto &counter = prometheus::Builder() + .Name(name) + .Help(get_description(metadata)) + .Register(registry) + .Add(get_labels(labels, num_labels)); + return convert_to_handle(&counter); +} + +gauge_metric_handle_t monitoring__register_gauge(const char *name, + metric_metadata_t metadata, + const metric_label_t *labels, + uint16_t num_labels) +{ + auto &gauge = prometheus::Builder() + .Name(name) + .Help(get_description(metadata)) + .Register(registry) + .Add(get_labels(labels, num_labels)); + return convert_to_handle(&gauge); +} + +histogram_metric_handle_t +monitoring__register_histogram(const char *name, metric_metadata_t metadata, + const metric_label_t *labels, + uint16_t num_labels, histogram_buckets_t buckets) +{ + const auto &buckets_vector = HistogramInt::BucketBoundaries( + buckets.buckets, buckets.buckets + buckets.count); + auto &histogram = + prometheus::Builder() + .Name(name) + .Help(get_description(metadata)) + .Register(registry) + .Add(get_labels(labels, num_labels), buckets_vector); + return convert_to_handle(&histogram); +} + +void monitoring__counter_inc(counter_metric_handle_t handle, int64_t value) +{ + convert_from_handle(handle)->Increment(value); +} + +void monitoring__gauge_inc(gauge_metric_handle_t handle, int64_t value) +{ + convert_from_handle(handle)->Increment(value); +} + +void monitoring__gauge_dec(gauge_metric_handle_t handle, int64_t value) +{ + convert_from_handle(handle)->Decrement(value); +} + +void monitoring__gauge_set(gauge_metric_handle_t handle, int64_t value) +{ + convert_from_handle(handle)->Set(value); +} + +void monitoring__histogram_observe(histogram_metric_handle_t handle, + int64_t value) +{ + convert_from_handle(handle)->Observe(value); +} + +prometheus_registry_handle_t monitoring__get_registry_handle(void) +{ + void *const registry_ptr = static_cast(®istry); + return { registry_ptr }; +} + +} /* extern "C" */ + +} /* namespace ntirpc_monitoring */ + +#endif /* USE_MONITORING */ diff --git a/src/monitoring/prometheus-cpp-lite b/src/monitoring/prometheus-cpp-lite new file mode 160000 index 000000000..48d09c45e --- /dev/null +++ b/src/monitoring/prometheus-cpp-lite @@ -0,0 +1 @@ +Subproject commit 48d09c45ee6deb90e02579b03037740e1c01fd27