feat(clp-s): Add end to end search tests for clp-s. (#668)

Author: gibber9809

components/core/CMakeLists.txt

@@ -299,18 +299,30 @@ set(SOURCE_FILES_clp_s_unitTest
     src/clp_s/SchemaTree.hpp
     src/clp_s/SchemaWriter.cpp
     src/clp_s/SchemaWriter.hpp
+    src/clp_s/search/AddTimestampConditions.cpp
+    src/clp_s/search/AddTimestampConditions.hpp
     src/clp_s/search/AndExpr.cpp
     src/clp_s/search/AndExpr.hpp
     src/clp_s/search/BooleanLiteral.cpp
     src/clp_s/search/BooleanLiteral.hpp
+    src/clp_s/search/clp_search/EncodedVariableInterpreter.cpp
+    src/clp_s/search/clp_search/EncodedVariableInterpreter.hpp
+    src/clp_s/search/clp_search/Grep.cpp
+    src/clp_s/search/clp_search/Grep.hpp
     src/clp_s/search/clp_search/Query.cpp
     src/clp_s/search/clp_search/Query.hpp
     src/clp_s/search/ColumnDescriptor.cpp
     src/clp_s/search/ColumnDescriptor.hpp
+    src/clp_s/search/ConstantProp.cpp
+    src/clp_s/search/ConstantProp.hpp
+    src/clp_s/search/ConvertToExists.cpp
+    src/clp_s/search/ConvertToExists.hpp
     src/clp_s/search/DateLiteral.cpp
     src/clp_s/search/DateLiteral.hpp
     src/clp_s/search/EmptyExpr.cpp
     src/clp_s/search/EmptyExpr.hpp
+    src/clp_s/search/EvaluateTimestampIndex.cpp
+    src/clp_s/search/EvaluateTimestampIndex.hpp
     src/clp_s/search/Expression.cpp
     src/clp_s/search/Expression.hpp
     src/clp_s/search/FilterExpr.cpp
@@ -319,12 +331,22 @@ set(SOURCE_FILES_clp_s_unitTest
     src/clp_s/search/Integral.cpp
     src/clp_s/search/Integral.hpp
     src/clp_s/search/Literal.hpp
+    src/clp_s/search/NarrowTypes.cpp
+    src/clp_s/search/NarrowTypes.hpp
     src/clp_s/search/NullLiteral.cpp
     src/clp_s/search/NullLiteral.hpp
     src/clp_s/search/OrExpr.cpp
     src/clp_s/search/OrExpr.hpp
     src/clp_s/search/OrOfAndForm.cpp
     src/clp_s/search/OrOfAndForm.hpp
+    src/clp_s/search/Output.cpp
+    src/clp_s/search/Output.hpp
+    src/clp_s/search/OutputHandler.cpp
+    src/clp_s/search/OutputHandler.hpp
+    src/clp_s/search/Projection.cpp
+    src/clp_s/search/Projection.hpp
+    src/clp_s/search/SchemaMatch.cpp
+    src/clp_s/search/SchemaMatch.hpp
     src/clp_s/search/SearchUtils.cpp
     src/clp_s/search/SearchUtils.hpp
     src/clp_s/search/StringLiteral.cpp
@@ -349,7 +371,29 @@ set(SOURCE_FILES_clp_s_unitTest
     src/clp_s/ZstdCompressor.hpp
     src/clp_s/ZstdDecompressor.cpp
     src/clp_s/ZstdDecompressor.hpp
-)
+    )
+
+set(SOURCE_FILES_reducer_unitTest
+    src/reducer/BufferedSocketWriter.cpp
+    src/reducer/BufferedSocketWriter.hpp
+    src/reducer/ConstRecordIterator.hpp
+    src/reducer/CountOperator.cpp
+    src/reducer/CountOperator.hpp
+    src/reducer/DeserializedRecordGroup.cpp
+    src/reducer/DeserializedRecordGroup.hpp
+    src/reducer/GroupTags.hpp
+    src/reducer/network_utils.cpp
+    src/reducer/network_utils.hpp
+    src/reducer/Operator.cpp
+    src/reducer/Operator.hpp
+    src/reducer/Pipeline.cpp
+    src/reducer/Pipeline.hpp
+    src/reducer/Record.hpp
+    src/reducer/RecordGroup.hpp
+    src/reducer/RecordGroupIterator.hpp
+    src/reducer/RecordTypedKeyIterator.hpp
+    src/reducer/types.hpp
+    )
 
 set(SOURCE_FILES_unitTest
         src/clp/Array.hpp
@@ -491,6 +535,8 @@ set(SOURCE_FILES_unitTest
         src/clp/MySQLParamBindings.hpp
         src/clp/MySQLPreparedStatement.cpp
         src/clp/MySQLPreparedStatement.hpp
+        src/clp/networking/socket_utils.cpp
+        src/clp/networking/socket_utils.hpp
         src/clp/NetworkReader.cpp
         src/clp/NetworkReader.hpp
         src/clp/PageAllocatedVector.hpp
@@ -578,10 +624,12 @@ set(SOURCE_FILES_unitTest
         submodules/sqlite3/sqlite3.h
         submodules/sqlite3/sqlite3ext.h
         tests/LogSuppressor.hpp
+        tests/TestOutputCleaner.hpp
         tests/test-Array.cpp
         tests/test-BoundedReader.cpp
         tests/test-BufferedFileReader.cpp
         tests/test-clp_s-end_to_end.cpp
+        tests/test-clp_s-search.cpp
         tests/test-EncodedVariableInterpreter.cpp
         tests/test-encoding_methods.cpp
         tests/test-error_handling.cpp
@@ -612,7 +660,11 @@ set(SOURCE_FILES_unitTest
         tests/test-utf8_utils.cpp
         tests/test-Utils.cpp
         )
-add_executable(unitTest ${SOURCE_FILES_unitTest} ${SOURCE_FILES_clp_s_unitTest})
+add_executable(unitTest
+        ${SOURCE_FILES_unitTest}
+        ${SOURCE_FILES_clp_s_unitTest}
+        ${SOURCE_FILES_reducer_unitTest}
+        )
 target_include_directories(unitTest
         PRIVATE
         ${CMAKE_SOURCE_DIR}/submodules

components/core/src/clp_s/search/OutputHandler.hpp

@@ -9,6 +9,7 @@
 #include <queue>
 #include <string>
 #include <string_view>
+#include <vector>
 
 #include <mongocxx/client.hpp>
 #include <mongocxx/collection.hpp>
@@ -294,6 +295,54 @@ class CountByTimeOutputHandler : public OutputHandler {
     std::map<int64_t, int64_t> m_bucket_counts;
     int64_t m_count_by_time_bucket_size;
 };
+
+/**
+ * Output handler that records all results in a provided vector.
+ */
+class VectorOutputHandler : public OutputHandler {
+public:
+    // Types
+    struct QueryResult {
+        // Constructors
+        QueryResult(
+                std::string_view message,
+                epochtime_t timestamp,
+                std::string_view archive_id,
+                int64_t log_event_idx
+        )
+                : message{message},
+                  timestamp{timestamp},
+                  archive_id{archive_id},
+                  log_event_idx{log_event_idx} {}
+
+        std::string message;
+        epochtime_t timestamp;
+        std::string archive_id;
+        int64_t log_event_idx;
+    };
+
+    // Constructors
+    VectorOutputHandler(std::vector<QueryResult>& output)
+            : OutputHandler{true, true},
+              m_output(output) {}
+
+    // Methods inherited from OutputHandler
+    void write(
+            std::string_view message,
+            epochtime_t timestamp,
+            std::string_view archive_id,
+            int64_t log_event_idx
+    ) override {
+        m_output.emplace_back(message, timestamp, archive_id, log_event_idx);
+    }
+
+    void write(std::string_view message) override {
+        m_output.emplace_back(message, epochtime_t{}, std::string_view{}, int64_t{});
+    }
+
+private:
+    std::vector<QueryResult>& m_output;
+};
 }  // namespace clp_s::search
 
 #endif  // CLP_S_SEARCH_OUTPUTHANDLER_HPP

components/core/tests/LogSuppressor.hpp

@@ -13,16 +13,17 @@ class LogSuppressor {
         spdlog::default_logger()->set_level(spdlog::level::off);
     }
 
-    LogSuppressor(LogSuppressor const& other) = default;
-    LogSuppressor(LogSuppressor&& other) noexcept = default;
-
-    LogSuppressor& operator=(LogSuppressor const& other) = default;
-    LogSuppressor& operator=(LogSuppressor&& other) noexcept = default;
-
     ~LogSuppressor() { spdlog::default_logger()->set_level(m_previous_logging_level); }
 
+    // Delete copy & move constructors and assignment operators
+    LogSuppressor(LogSuppressor const& other) = delete;
+    LogSuppressor(LogSuppressor&& other) = delete;
+    auto operator=(LogSuppressor const& other) -> LogSuppressor& = delete;
+    auto operator=(LogSuppressor&& other) -> LogSuppressor& = delete;
+
 private:
     spdlog::level::level_enum m_previous_logging_level;
 };
 
 #endif  // TESTS_LOGSUPPRESSOR_HPP
+

components/core/tests/TestOutputCleaner.hpp

@@ -0,0 +1,36 @@
+#ifndef TESTS_TESTOUTPUTCLEANER_HPP
+#define TESTS_TESTOUTPUTCLEANER_HPP
+
+#include <filesystem>
+#include <string>
+#include <vector>
+
+/**
+ * A class that deletes the directories and files created by test cases, both before and after each
+ * test case where the class is instantiated.
+ */
+class TestOutputCleaner {
+public:
+    explicit TestOutputCleaner(std::vector<std::string> const& paths) : m_paths(paths) {
+        delete_paths();
+    }
+
+    ~TestOutputCleaner() { delete_paths(); }
+
+    // Delete copy & move constructors and assignment operators
+    TestOutputCleaner(TestOutputCleaner const&) = delete;
+    TestOutputCleaner(TestOutputCleaner&&) = delete;
+    auto operator=(TestOutputCleaner const&) -> TestOutputCleaner& = delete;
+    auto operator=(TestOutputCleaner&&) -> TestOutputCleaner& = delete;
+
+private:
+    void delete_paths() const {
+        for (auto const& path : m_paths) {
+            std::filesystem::remove_all(path);
+        }
+    }
+
+    std::vector<std::string> m_paths;
+};
+
+#endif  // TESTS_TESTOUTPUTCLEANER_HPP

components/core/tests/test-clp_s-end_to_end.cpp

@@ -4,14 +4,14 @@
 #include <filesystem>
 #include <string>
 #include <string_view>
-#include <vector>
 
 #include <Catch2/single_include/catch2/catch.hpp>
 #include <fmt/format.h>
 
 #include "../src/clp_s/InputConfig.hpp"
 #include "../src/clp_s/JsonConstructor.hpp"
 #include "../src/clp_s/JsonParser.hpp"
+#include "TestOutputCleaner.hpp"
 
 constexpr std::string_view cTestEndToEndArchiveDirectory{"test-end-to-end-archive"};
 constexpr std::string_view cTestEndToEndOutputDirectory{"test-end-to-end-out"};
@@ -20,33 +20,9 @@ constexpr std::string_view cTestEndToEndInputFileDirectory{"test_log_files"};
 constexpr std::string_view cTestEndToEndInputFile{"test_no_floats_sorted.jsonl"};
 
 namespace {
-/**
- * A class that deletes the directories and files created by test cases, both before and after each
- * test case where the class is instantiated.
- */
-class TestOutputCleaner {
-public:
-    TestOutputCleaner() { delete_files(); }
-
-    ~TestOutputCleaner() { delete_files(); }
-
-    // Delete copy & move constructors and assignment operators
-    TestOutputCleaner(TestOutputCleaner const&) = delete;
-    TestOutputCleaner(TestOutputCleaner&&) = delete;
-    auto operator=(TestOutputCleaner const&) -> TestOutputCleaner& = delete;
-    auto operator=(TestOutputCleaner&&) -> TestOutputCleaner& = delete;
-
-private:
-    static void delete_files() {
-        std::filesystem::remove_all(cTestEndToEndArchiveDirectory);
-        std::filesystem::remove_all(cTestEndToEndOutputDirectory);
-        std::filesystem::remove(cTestEndToEndOutputSortedJson);
-    }
-};
-
 auto get_test_input_path_relative_to_tests_dir() -> std::filesystem::path;
 auto get_test_input_local_path() -> std::string;
-void compress(bool structurize_arrays);
+void compress(bool structurize_arrays, bool single_file_archive);
 auto extract() -> std::filesystem::path;
 void compare(std::filesystem::path const& extracted_json_path);
 
@@ -60,7 +36,7 @@ auto get_test_input_local_path() -> std::string {
     return (tests_dir / get_test_input_path_relative_to_tests_dir()).string();
 }
 
-void compress(bool structurize_arrays) {
+void compress(bool structurize_arrays, bool single_file_archive) {
     constexpr auto cDefaultTargetEncodedSize = 8ULL * 1024 * 1024 * 1024;  // 8 GiB
     constexpr auto cDefaultMaxDocumentSize = 512ULL * 1024 * 1024;  // 512 MiB
     constexpr auto cDefaultMinTableSize = 1ULL * 1024 * 1024;  // 1 MiB
@@ -82,6 +58,7 @@ void compress(bool structurize_arrays) {
     parser_option.compression_level = cDefaultCompressionLevel;
     parser_option.print_archive_stats = cDefaultPrintArchiveStats;
     parser_option.structurize_arrays = structurize_arrays;
+    parser_option.single_file_archive = single_file_archive;
 
     clp_s::JsonParser parser{parser_option};
     REQUIRE(parser.parse());
@@ -102,11 +79,6 @@ auto extract() -> std::filesystem::path {
     constructor_option.ordered = cDefaultOrdered;
     constructor_option.target_ordered_chunk_size = cDefaultTargetOrderedChunkSize;
     for (auto const& entry : std::filesystem::directory_iterator(cTestEndToEndArchiveDirectory)) {
-        if (false == entry.is_directory()) {
-            // Skip non-directories
-            continue;
-        }
-
         constructor_option.archive_path = clp_s::Path{
                 .source{clp_s::InputSource::Filesystem},
                 .path{entry.path().string()}
@@ -153,10 +125,15 @@ void compare(std::filesystem::path const& extracted_json_path) {
 
 TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") {
     auto structurize_arrays = GENERATE(true, false);
+    auto single_file_archive = GENERATE(true, false);
 
-    TestOutputCleaner const test_cleanup;
+    TestOutputCleaner const test_cleanup{
+            {std::string{cTestEndToEndArchiveDirectory},
+             std::string{cTestEndToEndOutputDirectory},
+             std::string{cTestEndToEndOutputSortedJson}}
+    };
 
-    compress(structurize_arrays);
+    compress(structurize_arrays, single_file_archive);
 
     auto extracted_json_path = extract();