Feature: read_parquet_mergetree (#13)

* parquet ordered scan / mergetree like feature

Co-authored-by: Lorenzo Mangani <[email protected]>

Author: akvlad

CMakeLists.txt

@@ -1,30 +1,13 @@
 cmake_minimum_required(VERSION 3.5)
-
 # Set extension name here
 set(TARGET_NAME chsql)
-
 # DuckDB's extension distribution supports vcpkg. As such, dependencies can be added in ./vcpkg.json and then
 # used in cmake with find_package. Feel free to remove or replace with other dependencies.
 # Note that it should also be removed from vcpkg.json to prevent needlessly installing it..
 find_package(OpenSSL REQUIRED)
-
 set(EXTENSION_NAME ${TARGET_NAME}_extension)
 set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
-
 project(${TARGET_NAME})
-include_directories(src/include)
-
-set(EXTENSION_SOURCES src/chsql_extension.cpp)
-
-build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})
-build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES})
-
-# Link OpenSSL in both the static library as the loadable extension
-target_link_libraries(${EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto)
-target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto)
-
-install(
-  TARGETS ${EXTENSION_NAME}
-  EXPORT "${DUCKDB_EXPORT_SET}"
-  LIBRARY DESTINATION "${INSTALL_LIB_DIR}"
-  ARCHIVE DESTINATION "${INSTALL_LIB_DIR}")
+set(EXT_NAME chsql)
+set(DUCKDB_EXTENSION_CONFIGS ../chsql/extension_config.cmake)
+add_subdirectory(./duckdb)

Makefile

@@ -1,4 +1,4 @@
-PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))chsql/
 
 # Configuration of extension
 EXT_NAME=chsql

chsql/CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 3.5)
+# Set extension name here
+set(TARGET_NAME chsql)
+# DuckDB's extension distribution supports vcpkg. As such, dependencies can be added in ./vcpkg.json and then
+# used in cmake with find_package. Feel free to remove or replace with other dependencies.
+# Note that it should also be removed from vcpkg.json to prevent needlessly installing it..
+find_package(OpenSSL REQUIRED)
+set(EXTENSION_NAME ${TARGET_NAME}_extension)
+set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension)
+project(${TARGET_NAME})
+
+include_directories(
+        ./src/include
+        ./src
+        ${CMAKE_CURRENT_SOURCE_DIR}/../duckdb/extension/parquet/include
+        ../duckdb/third_party/lz4
+        ../duckdb/third_party/parquet
+        ../duckdb/third_party/thrift
+        ../duckdb/third_party/snappy
+        ../duckdb/third_party/zstd/include
+        ../duckdb/third_party/mbedtls
+        ../duckdb/third_party/mbedtls/include
+        ../duckdb/third_party/brotli/include)
+set(EXTENSION_SOURCES src/chsql_extension.cpp)
+build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})
+build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES})
+# Link OpenSSL in both the static library as the loadable extension
+target_link_libraries(${EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto)
+target_link_libraries(${LOADABLE_EXTENSION_NAME} OpenSSL::SSL OpenSSL::Crypto)
+install(
+        TARGETS ${EXTENSION_NAME}
+        EXPORT "${DUCKDB_EXPORT_SET}"
+        LIBRARY DESTINATION "${INSTALL_LIB_DIR}"
+        ARCHIVE DESTINATION "${INSTALL_LIB_DIR}")

chsql/extension_config.cmake

@@ -0,0 +1,22 @@
+# This file is included by DuckDB's build system. It specifies which extension to load
+
+include_directories(
+        ./src/include
+        ${CMAKE_CURRENT_SOURCE_DIR}/../duckdb/extension/parquet/include
+        ../duckdb/third_party/lz4
+        ../duckdb/third_party/parquet
+        ../duckdb/third_party/thrift
+        ../duckdb/third_party/snappy
+        ../duckdb/third_party/zstd/include
+        ../duckdb/third_party/mbedtls
+        ../duckdb/third_party/mbedtls/include
+        ../duckdb/third_party/brotli/include)
+
+# Extension from this repo
+duckdb_extension_load(chsql
+    SOURCE_DIR  ${CMAKE_CURRENT_LIST_DIR}
+    LOAD_TESTS
+)
+
+# Any extra extensions that should be built
+# e.g.: duckdb_extension_load(json)

src/chsql_extension.cpp chsql/src/chsql_extension.cpp

@@ -12,7 +12,7 @@
 
 // OpenSSL linked through vcpkg
 #include <openssl/opensslv.h>
-
+#include "parquet_ordered_scan.cpp"
 namespace duckdb {
 
 // To add a new scalar SQL macro, add a new macro to this array!
@@ -188,6 +188,7 @@ static void LoadInternal(DatabaseInstance &instance) {
 		auto table_info = DefaultTableFunctionGenerator::CreateTableMacroInfo(chsql_table_macros[index]);
         ExtensionUtil::RegisterFunction(instance, *table_info);
 	}
+	ExtensionUtil::RegisterFunction(instance, ReadParquetOrderedFunction());
 }
 
 void ChsqlExtension::Load(DuckDB &db) {

src/include/chsql_extension.hpp chsql/src/include/chsql_extension.hpp

@@ -10,5 +10,6 @@ class ChsqlExtension : public Extension {
 	std::string Name() override;
         std::string Version() const override;
 };
-
+duckdb::TableFunction ReadParquetOrderedFunction();
+static void RegisterSillyBTreeStore(DatabaseInstance &instance);
 } // namespace duckdb