From 6a5567590f4be7bcf9a9f943b5b5bda36a0a8468 Mon Sep 17 00:00:00 2001 From: Hemant Sharma Date: Tue, 17 Mar 2026 16:23:25 +0530 Subject: [PATCH 1/2] Making MDBX Map Size Constants Runtime Configurable --- src/utils/settings.hpp | 73 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/src/utils/settings.hpp b/src/utils/settings.hpp index 1c13a3a45..6f3fe2f79 100644 --- a/src/utils/settings.hpp +++ b/src/utils/settings.hpp @@ -45,17 +45,17 @@ namespace settings { // MDBX default map sizes. Growth step and initial size are the same for all databases. // System tables - constexpr size_t INDEX_META_MAP_SIZE_BITS = 21; // 2 MiB - constexpr size_t INDEX_META_MAP_SIZE_MAX_BITS = 27; // 128 MiB + constexpr size_t DEFAULT_INDEX_META_MAP_SIZE_BITS = 21; // 2 MiB + constexpr size_t DEFAULT_INDEX_META_MAP_SIZE_MAX_BITS = 27; // 128 MiB // Index-related tables - constexpr size_t ID_MAPPER_MAP_SIZE_BITS = 24; // 16 MiB - constexpr size_t ID_MAPPER_MAP_SIZE_MAX_BITS = 33; // 8 GiB - constexpr size_t FILTER_MAP_SIZE_BITS = 24; // 16 MiB - constexpr size_t FILTER_MAP_SIZE_MAX_BITS = 36; // 64 GiB - constexpr size_t METADATA_MAP_SIZE_BITS = 27; // 128 MiB - constexpr size_t METADATA_MAP_SIZE_MAX_BITS = 39; // 512 GiB - constexpr size_t VECTOR_MAP_SIZE_BITS = 30; // 1 GiB - constexpr size_t VECTOR_MAP_SIZE_MAX_BITS = 42; // 4 TiB + constexpr size_t DEFAULT_ID_MAPPER_MAP_SIZE_BITS = 24; // 16 MiB + constexpr size_t DEFAULT_ID_MAPPER_MAP_SIZE_MAX_BITS = 33; // 8 GiB + constexpr size_t DEFAULT_FILTER_MAP_SIZE_BITS = 24; // 16 MiB + constexpr size_t DEFAULT_FILTER_MAP_SIZE_MAX_BITS = 36; // 64 GiB + constexpr size_t DEFAULT_METADATA_MAP_SIZE_BITS = 27; // 128 MiB + constexpr size_t DEFAULT_METADATA_MAP_SIZE_MAX_BITS = 39; // 512 GiB + constexpr size_t DEFAULT_VECTOR_MAP_SIZE_BITS = 30; // 1 GiB + constexpr size_t DEFAULT_VECTOR_MAP_SIZE_MAX_BITS = 42; // 4 TiB constexpr size_t MAX_LINK_LIST_LOCKS = 65536; @@ -206,6 +206,48 @@ namespace settings { inline static bool AUTH_ENABLED = !AUTH_TOKEN.empty(); + // MDBX map sizes (bit counts, used as 1ULL << N) + inline static size_t INDEX_META_MAP_SIZE_BITS = [] { + const char* env = std::getenv("NDD_INDEX_META_MAP_SIZE_BITS"); + return env ? std::stoull(env) : DEFAULT_INDEX_META_MAP_SIZE_BITS; + }(); + inline static size_t INDEX_META_MAP_SIZE_MAX_BITS = [] { + const char* env = std::getenv("NDD_INDEX_META_MAP_SIZE_MAX_BITS"); + return env ? std::stoull(env) : DEFAULT_INDEX_META_MAP_SIZE_MAX_BITS; + }(); + inline static size_t ID_MAPPER_MAP_SIZE_BITS = [] { + const char* env = std::getenv("NDD_ID_MAPPER_MAP_SIZE_BITS"); + return env ? std::stoull(env) : DEFAULT_ID_MAPPER_MAP_SIZE_BITS; + }(); + inline static size_t ID_MAPPER_MAP_SIZE_MAX_BITS = [] { + const char* env = std::getenv("NDD_ID_MAPPER_MAP_SIZE_MAX_BITS"); + return env ? std::stoull(env) : DEFAULT_ID_MAPPER_MAP_SIZE_MAX_BITS; + }(); + inline static size_t FILTER_MAP_SIZE_BITS = [] { + const char* env = std::getenv("NDD_FILTER_MAP_SIZE_BITS"); + return env ? std::stoull(env) : DEFAULT_FILTER_MAP_SIZE_BITS; + }(); + inline static size_t FILTER_MAP_SIZE_MAX_BITS = [] { + const char* env = std::getenv("NDD_FILTER_MAP_SIZE_MAX_BITS"); + return env ? std::stoull(env) : DEFAULT_FILTER_MAP_SIZE_MAX_BITS; + }(); + inline static size_t METADATA_MAP_SIZE_BITS = [] { + const char* env = std::getenv("NDD_METADATA_MAP_SIZE_BITS"); + return env ? std::stoull(env) : DEFAULT_METADATA_MAP_SIZE_BITS; + }(); + inline static size_t METADATA_MAP_SIZE_MAX_BITS = [] { + const char* env = std::getenv("NDD_METADATA_MAP_SIZE_MAX_BITS"); + return env ? std::stoull(env) : DEFAULT_METADATA_MAP_SIZE_MAX_BITS; + }(); + inline static size_t VECTOR_MAP_SIZE_BITS = [] { + const char* env = std::getenv("NDD_VECTOR_MAP_SIZE_BITS"); + return env ? std::stoull(env) : DEFAULT_VECTOR_MAP_SIZE_BITS; + }(); + inline static size_t VECTOR_MAP_SIZE_MAX_BITS = [] { + const char* env = std::getenv("NDD_VECTOR_MAP_SIZE_MAX_BITS"); + return env ? std::stoull(env) : DEFAULT_VECTOR_MAP_SIZE_MAX_BITS; + }(); + // Function to get all settings values as a multiline string inline std::string getAllSettingsAsString() { std::ostringstream oss; @@ -224,6 +266,17 @@ namespace settings { oss << "ENABLE_DEBUG_LOG: " << (ENABLE_DEBUG_LOG ? "true" : "false") << "\n"; oss << "AUTH_ENABLED: " << (AUTH_ENABLED ? "true" : "false") << "\n"; oss << "DEFAULT_USERNAME: " << DEFAULT_USERNAME << "\n"; + oss << "\n=== MDBX Map Sizes (bit shifts) ===\n"; + oss << "INDEX_META_MAP_SIZE_BITS: " << INDEX_META_MAP_SIZE_BITS << "\n"; + oss << "INDEX_META_MAP_SIZE_MAX_BITS: " << INDEX_META_MAP_SIZE_MAX_BITS << "\n"; + oss << "ID_MAPPER_MAP_SIZE_BITS: " << ID_MAPPER_MAP_SIZE_BITS << "\n"; + oss << "ID_MAPPER_MAP_SIZE_MAX_BITS: " << ID_MAPPER_MAP_SIZE_MAX_BITS << "\n"; + oss << "FILTER_MAP_SIZE_BITS: " << FILTER_MAP_SIZE_BITS << "\n"; + oss << "FILTER_MAP_SIZE_MAX_BITS: " << FILTER_MAP_SIZE_MAX_BITS << "\n"; + oss << "METADATA_MAP_SIZE_BITS: " << METADATA_MAP_SIZE_BITS << "\n"; + oss << "METADATA_MAP_SIZE_MAX_BITS: " << METADATA_MAP_SIZE_MAX_BITS << "\n"; + oss << "VECTOR_MAP_SIZE_BITS: " << VECTOR_MAP_SIZE_BITS << "\n"; + oss << "VECTOR_MAP_SIZE_MAX_BITS: " << VECTOR_MAP_SIZE_MAX_BITS << "\n"; oss << "\n=== End Settings ===\n"; return oss.str(); } From a2e7b54187b57616f5f9ab4bcd5cec85ad324452 Mon Sep 17 00:00:00 2001 From: Hemant Sharma Date: Tue, 17 Mar 2026 16:53:58 +0530 Subject: [PATCH 2/2] adding DEFAULT_SPARSE_MAP_SIZE_MAX_BITS --- src/sparse/sparse_storage.hpp | 4 ++-- src/utils/settings.hpp | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/sparse/sparse_storage.hpp b/src/sparse/sparse_storage.hpp index e55c48e44..802d2049b 100644 --- a/src/sparse/sparse_storage.hpp +++ b/src/sparse/sparse_storage.hpp @@ -245,8 +245,8 @@ namespace ndd { return false; } - // Set geometry (max 1TB for now, can be configured) - rc = mdbx_env_set_geometry(env_, -1, -1, TB, -1, -1, -1); + // Set geometry - max size configurable via NDD_SPARSE_MAP_SIZE_MAX_BITS + rc = mdbx_env_set_geometry(env_, -1, -1, 1ULL << settings::SPARSE_MAP_SIZE_MAX_BITS, -1, -1, -1); if(rc != 0) { LOG_ERROR(2246, index_id_, "mdbx_env_set_geometry failed: " << mdbx_strerror(rc)); return false; diff --git a/src/utils/settings.hpp b/src/utils/settings.hpp index 6f3fe2f79..d7870348b 100644 --- a/src/utils/settings.hpp +++ b/src/utils/settings.hpp @@ -56,6 +56,8 @@ namespace settings { constexpr size_t DEFAULT_METADATA_MAP_SIZE_MAX_BITS = 39; // 512 GiB constexpr size_t DEFAULT_VECTOR_MAP_SIZE_BITS = 30; // 1 GiB constexpr size_t DEFAULT_VECTOR_MAP_SIZE_MAX_BITS = 42; // 4 TiB + // Sparse storage + constexpr size_t DEFAULT_SPARSE_MAP_SIZE_MAX_BITS = 40; // 1 TiB constexpr size_t MAX_LINK_LIST_LOCKS = 65536; @@ -247,6 +249,10 @@ namespace settings { const char* env = std::getenv("NDD_VECTOR_MAP_SIZE_MAX_BITS"); return env ? std::stoull(env) : DEFAULT_VECTOR_MAP_SIZE_MAX_BITS; }(); + inline static size_t SPARSE_MAP_SIZE_MAX_BITS = [] { + const char* env = std::getenv("NDD_SPARSE_MAP_SIZE_MAX_BITS"); + return env ? std::stoull(env) : DEFAULT_SPARSE_MAP_SIZE_MAX_BITS; + }(); // Function to get all settings values as a multiline string inline std::string getAllSettingsAsString() { @@ -277,6 +283,7 @@ namespace settings { oss << "METADATA_MAP_SIZE_MAX_BITS: " << METADATA_MAP_SIZE_MAX_BITS << "\n"; oss << "VECTOR_MAP_SIZE_BITS: " << VECTOR_MAP_SIZE_BITS << "\n"; oss << "VECTOR_MAP_SIZE_MAX_BITS: " << VECTOR_MAP_SIZE_MAX_BITS << "\n"; + oss << "SPARSE_MAP_SIZE_MAX_BITS: " << SPARSE_MAP_SIZE_MAX_BITS << "\n"; oss << "\n=== End Settings ===\n"; return oss.str(); }