Sanity Checks for DB

CMakeLists.txt

@@ -239,7 +239,7 @@ message(STATUS "Binary name: ${NDD_BINARY_NAME}")
 
 
 # Create the target
-add_executable(${NDD_BINARY_NAME} src/main.cpp ${LMDB_SOURCES} third_party/roaring_bitmap/roaring.c)
+add_executable(${NDD_BINARY_NAME} src/main.cpp src/utils/system_sanity/system_sanity.cpp ${LMDB_SOURCES} third_party/roaring_bitmap/roaring.c)
 
 # Set MDBX-specific compile flags
 set_source_files_properties(${LMDB_SOURCES} PROPERTIES

src/main.cpp

@@ -36,6 +36,7 @@
 #include "quant/common.hpp"
 #include "cpu_compat_check/check_avx_compat.hpp"
 #include "cpu_compat_check/check_arm_compat.hpp"
+#include "system_sanity/system_sanity.hpp"
 
 using ndd::quant::quantLevelToString;
 using ndd::quant::stringToQuantLevel;
@@ -185,6 +186,7 @@ int main(int argc, char** argv) {
         printf("CPU is not compatible. Can't run Endee\n");
         return 0;
     }
+
     LOG_DEBUG("SERVER_ID: " << settings::SERVER_ID);
     LOG_DEBUG("SERVER_PORT: " << settings::SERVER_PORT);
     LOG_DEBUG("DATA_DIR: " << settings::DATA_DIR);
@@ -762,6 +764,10 @@ int main(int argc, char** argv) {
                 // Verify content type is application/msgpack or application/json
                 auto content_type = req.get_header_value("Content-Type");
 
+                if(is_disk_full()){
+                    return json_error(400, "Batch insertion aborted: Not enough storage space");
+                }
+
                 if(content_type == "application/json") {
                     auto body = crow::json::load(req.body);
                     if(!body) {

src/utils/settings.hpp

@@ -71,6 +71,10 @@ namespace settings {
     // Maximum number of elements in the index
     constexpr size_t MAX_VECTORS_ADMIN = 1'000'000'000;
 
+
+    //minimum bytes in filesystem before triggering out of storage sequence
+    constexpr size_t MINIMUM_REQUIRED_FS_BYTES = (1 * GB);
+
     // Buffer for early exit in search base layer
     constexpr int EARLY_EXIT_BUFFER_INSERT = 16;
     constexpr int EARLY_EXIT_BUFFER_QUERY = 8;

src/utils/system_sanity/system_sanity.cpp

@@ -0,0 +1,39 @@
+#include <iostream>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/statvfs.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <limits.h>
+#include "utils/settings.hpp"
+#include "utils/log.hpp"
+
+size_t get_remaining_storage(const char *folder_path) {
+    struct statvfs vfs;
+
+    if (!folder_path || statvfs(folder_path, &vfs) != 0) {
+        perror("get_remaining_storage: statvfs");
+        return SIZE_MAX;  // error sentinel
+    }
+
+    // printf("%s: remaining space in %s is %zu bytes\n", __func__, folder_path, (size_t)vfs.f_bavail * (size_t)vfs.f_frsize);
+
+    return (size_t)vfs.f_bavail * (size_t)vfs.f_frsize;
+}
+
+
+/**
+ * This returns true if the disk is considered full.
+ */
+bool is_disk_full(){
+    size_t remaining_size = get_remaining_storage(settings::DATA_DIR.c_str());
+
+    if(remaining_size < settings::MINIMUM_REQUIRED_FS_BYTES){
+        LOG_INFO("Remining storage in " + settings::DATA_DIR + " is : " + std::to_string(remaining_size/MB) + " MB");
+        return true;
+    }
+    return false;
+}

src/utils/system_sanity/system_sanity.hpp

@@ -0,0 +1,2 @@
+
+bool is_disk_full();