diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d6fc30fa..4a1abd208 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -239,7 +239,7 @@ message(STATUS "Binary name: ${NDD_BINARY_NAME}") # Create the target -add_executable(${NDD_BINARY_NAME} src/main.cpp ${LMDB_SOURCES} third_party/roaring_bitmap/roaring.c) +add_executable(${NDD_BINARY_NAME} src/main.cpp src/utils/system_sanity/system_sanity.cpp ${LMDB_SOURCES} third_party/roaring_bitmap/roaring.c) # Set MDBX-specific compile flags set_source_files_properties(${LMDB_SOURCES} PROPERTIES diff --git a/src/main.cpp b/src/main.cpp index 40ae26b86..a0f0abe44 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -36,6 +36,7 @@ #include "quant/common.hpp" #include "cpu_compat_check/check_avx_compat.hpp" #include "cpu_compat_check/check_arm_compat.hpp" +#include "system_sanity/system_sanity.hpp" using ndd::quant::quantLevelToString; using ndd::quant::stringToQuantLevel; @@ -185,6 +186,7 @@ int main(int argc, char** argv) { printf("CPU is not compatible. Can't run Endee\n"); return 0; } + LOG_DEBUG("SERVER_ID: " << settings::SERVER_ID); LOG_DEBUG("SERVER_PORT: " << settings::SERVER_PORT); LOG_DEBUG("DATA_DIR: " << settings::DATA_DIR); @@ -762,6 +764,10 @@ int main(int argc, char** argv) { // Verify content type is application/msgpack or application/json auto content_type = req.get_header_value("Content-Type"); + if(is_disk_full()){ + return json_error(400, "Batch insertion aborted: Not enough storage space"); + } + if(content_type == "application/json") { auto body = crow::json::load(req.body); if(!body) { diff --git a/src/utils/settings.hpp b/src/utils/settings.hpp index c84ad89a1..08b0bbe63 100644 --- a/src/utils/settings.hpp +++ b/src/utils/settings.hpp @@ -71,6 +71,10 @@ namespace settings { // Maximum number of elements in the index constexpr size_t MAX_VECTORS_ADMIN = 1'000'000'000; + + //minimum bytes in filesystem before triggering out of storage sequence + constexpr size_t MINIMUM_REQUIRED_FS_BYTES = (1 * GB); + // Buffer for early exit in search base layer constexpr int EARLY_EXIT_BUFFER_INSERT = 16; constexpr int EARLY_EXIT_BUFFER_QUERY = 8; diff --git a/src/utils/system_sanity/system_sanity.cpp b/src/utils/system_sanity/system_sanity.cpp new file mode 100644 index 000000000..c0b7230ff --- /dev/null +++ b/src/utils/system_sanity/system_sanity.cpp @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "utils/settings.hpp" +#include "utils/log.hpp" + +size_t get_remaining_storage(const char *folder_path) { + struct statvfs vfs; + + if (!folder_path || statvfs(folder_path, &vfs) != 0) { + perror("get_remaining_storage: statvfs"); + return SIZE_MAX; // error sentinel + } + + // printf("%s: remaining space in %s is %zu bytes\n", __func__, folder_path, (size_t)vfs.f_bavail * (size_t)vfs.f_frsize); + + return (size_t)vfs.f_bavail * (size_t)vfs.f_frsize; +} + + +/** + * This returns true if the disk is considered full. + */ +bool is_disk_full(){ + size_t remaining_size = get_remaining_storage(settings::DATA_DIR.c_str()); + + if(remaining_size < settings::MINIMUM_REQUIRED_FS_BYTES){ + LOG_INFO("Remining storage in " + settings::DATA_DIR + " is : " + std::to_string(remaining_size/MB) + " MB"); + return true; + } + return false; +} \ No newline at end of file diff --git a/src/utils/system_sanity/system_sanity.hpp b/src/utils/system_sanity/system_sanity.hpp new file mode 100644 index 000000000..9354ad8bb --- /dev/null +++ b/src/utils/system_sanity/system_sanity.hpp @@ -0,0 +1,2 @@ + +bool is_disk_full(); \ No newline at end of file