Merge branch 'main' into combine-actions-counters

Author: sebastiantia

README.md

@@ -1,4 +1,13 @@
-# delta-kernel-rs
+# Delta Kernel (rust)   [![build-status]][actions] [![latest-version]][crates.io] [![docs]][docs.rs] [![rustc-version-1.81+]][rustc]
+
+[build-status]: https://img.shields.io/github/actions/workflow/status/delta-io/delta-kernel-rs/build.yml?branch=main
+[actions]: https://github.com/delta-io/delta-kernel-rs/actions/workflows/build.yml?query=branch%3Amain
+[latest-version]: https://img.shields.io/crates/v/delta_kernel.svg
+[crates.io]: https://crates.io/crates/delta\_kernel
+[rustc-version-1.81+]: https://img.shields.io/badge/rustc-1.81+-lightgray.svg
+[rustc]: https://blog.rust-lang.org/2024/09/05/Rust-1.81.0/
+[docs]: https://img.shields.io/docsrs/delta_kernel
+[docs.rs]: https://docs.rs/delta_kernel/latest/delta_kernel/
 
 Delta-kernel-rs is an experimental [Delta][delta] implementation focused on interoperability with a
 wide range of query engines. It currently supports reads and (experimental) writes. Only blind

ffi/examples/visit-expression/expression.h

@@ -53,7 +53,8 @@ enum LitType {
   Decimal,
   Null,
   Struct,
-  Array
+  Array,
+  Map
 };
 enum ExpressionType { BinOp, Variadic, Literal, Unary, Column };
 enum VariadicType {
@@ -103,6 +104,10 @@ struct Struct {
 struct ArrayData {
   ExpressionItemList exprs;
 };
+struct MapData {
+  ExpressionItemList keys;
+  ExpressionItemList vals;
+};
 struct Literal {
   enum LitType type;
   union LiteralValue {
@@ -116,6 +121,7 @@ struct Literal {
     char* string_data;
     struct Struct struct_data;
     struct ArrayData array_data;
+    struct MapData map_data;
     struct BinaryData binary;
     struct Decimal decimal;
   } value;
@@ -275,6 +281,18 @@ void visit_expr_array_literal(void* data, uintptr_t sibling_list_id, uintptr_t c
   put_expr_item(data, sibling_list_id, literal, Literal);
 }
 
+void visit_expr_map_literal(void* data,
+                            uintptr_t sibling_list_id,
+                            uintptr_t key_list_id,
+                            uintptr_t value_list_id) {
+  struct Literal* literal = malloc(sizeof(struct Literal));
+  literal->type = Map;
+  struct MapData* map = &(literal->value.map_data);
+  map->keys = get_expr_list(data, key_list_id);
+  map->vals = get_expr_list(data, value_list_id);
+  put_expr_item(data, sibling_list_id, literal, Literal);
+}
+
 /*************************************************************
  * Unary Expressions
  ************************************************************/
@@ -339,6 +357,7 @@ ExpressionItemList construct_predicate(SharedExpression* predicate) {
     .visit_literal_string = visit_expr_string_literal,
     .visit_literal_struct = visit_expr_struct_literal,
     .visit_literal_array = visit_expr_array_literal,
+    .visit_literal_map = visit_expr_map_literal,
     .visit_and = visit_expr_and,
     .visit_or = visit_expr_or,
     .visit_not = visit_expr_not,
@@ -394,6 +413,12 @@ void free_expression_item(ExpressionItem ref) {
           free_expression_list(array->exprs);
           break;
         }
+        case Map: {
+          struct MapData* map = &lit->value.map_data;
+          free_expression_list(map->keys);
+          free_expression_list(map->vals);
+          break;
+        }
         case String: {
           free(lit->value.string_data);
           break;

ffi/examples/visit-expression/expression_print.h

@@ -175,6 +175,27 @@ void print_tree_helper(ExpressionItem ref, int depth) {
           struct ArrayData* array = &lit->value.array_data;
           print_expression_item_list(array->exprs, depth + 1);
           break;
+        case Map:
+          printf("Map\n");
+          struct MapData* map_data = &lit->value.map_data;
+          for (size_t i = 0; i < map_data->keys.len; i++) {
+            print_n_spaces(depth + 1);
+
+            // Extract key
+            ExpressionItem key = map_data->keys.list[i];
+            assert(key.type == Literal);
+            struct Literal* key_lit = key.ref;
+            assert(key_lit->type == String);
+            // Extract val
+            ExpressionItem val = map_data->vals.list[i];
+            assert(val.type == Literal);
+            struct Literal* val_lit = val.ref;
+            assert(val_lit->type == String);
+
+            // instead of recursing (which forces newlines) we just directly print strings here
+            printf("String(%s): String(%s)\n", key_lit->value.string_data, val_lit->value.string_data);
+          }
+          break;
       }
       break;
     }

ffi/src/error.rs

@@ -4,6 +4,7 @@ use crate::{kernel_string_slice, ExternEngine, KernelStringSlice};
 
 #[repr(C)]
 #[derive(Debug)]
+#[non_exhaustive]
 pub enum KernelError {
     UnknownError, // catch-all for unrecognized kernel Error types
     FFIError,     // errors encountered in the code layer that supports FFI
@@ -54,6 +55,7 @@ pub enum KernelError {
     InvalidCheckpoint,
     LiteralExpressionTransformError,
     CheckpointWriteError,
+    SchemaError,
 }
 
 impl From<Error> for KernelError {
@@ -116,6 +118,8 @@ impl From<Error> for KernelError {
             Error::LiteralExpressionTransformError(_) => {
                 KernelError::LiteralExpressionTransformError
             }
+            Error::Schema(_) => KernelError::SchemaError,
+            _ => KernelError::UnknownError,
         }
     }
 }

ffi/src/expressions/kernel.rs

@@ -6,7 +6,7 @@ use std::ffi::c_void;
 use crate::{handle::Handle, kernel_string_slice, KernelStringSlice};
 use delta_kernel::expressions::{
     ArrayData, BinaryExpression, BinaryOperator, Expression, JunctionExpression, JunctionOperator,
-    Scalar, StructData, UnaryExpression, UnaryOperator,
+    MapData, Scalar, StructData, UnaryExpression, UnaryOperator,
 };
 
 /// Free the memory the passed SharedExpression
@@ -113,6 +113,15 @@ pub struct EngineExpressionVisitor {
     /// The values of the array are in a list identified by `child_list_id`.
     pub visit_literal_array:
         extern "C" fn(data: *mut c_void, sibling_list_id: usize, child_list_id: usize),
+    /// Visit a map literal belonging to the list identified by `sibling_list_id`.
+    /// The keys of the map are in order in a list identified by `key_list_id`. The values of the
+    /// map are in order in a list identified by `value_list_id`.
+    pub visit_literal_map: extern "C" fn(
+        data: *mut c_void,
+        sibling_list_id: usize,
+        key_list_id: usize,
+        value_list_id: usize,
+    ),
     /// Visits a null value belonging to the list identified by `sibling_list_id.
     pub visit_literal_null: extern "C" fn(data: *mut c_void, sibling_list_id: usize),
     /// Visits an `and` expression belonging to the list identified by `sibling_list_id`.
@@ -228,6 +237,27 @@ fn visit_expression_array(
     call!(visitor, visit_literal_array, sibling_list_id, child_list_id);
 }
 
+fn visit_expression_map(
+    visitor: &mut EngineExpressionVisitor,
+    map_data: &MapData,
+    sibling_list_id: usize,
+) {
+    let pairs = map_data.pairs();
+    let key_list_id = call!(visitor, make_field_list, pairs.len());
+    let value_list_id = call!(visitor, make_field_list, pairs.len());
+    for (key, val) in pairs {
+        visit_expression_scalar(visitor, key, key_list_id);
+        visit_expression_scalar(visitor, val, value_list_id);
+    }
+    call!(
+        visitor,
+        visit_literal_map,
+        sibling_list_id,
+        key_list_id,
+        value_list_id
+    );
+}
+
 fn visit_expression_struct_literal(
     visitor: &mut EngineExpressionVisitor,
     struct_data: &StructData,
@@ -333,6 +363,7 @@ fn visit_expression_scalar(
             visit_expression_struct_literal(visitor, struct_data, sibling_list_id)
         }
         Scalar::Array(array) => visit_expression_array(visitor, array, sibling_list_id),
+        Scalar::Map(map_data) => visit_expression_map(visitor, map_data, sibling_list_id),
     }
 }
 

ffi/src/test_ffi.rs

@@ -4,8 +4,10 @@ use std::sync::Arc;
 
 use crate::{expressions::SharedExpression, handle::Handle};
 use delta_kernel::{
-    expressions::{column_expr, ArrayData, BinaryOperator, Expression as Expr, Scalar, StructData},
-    schema::{ArrayType, DataType, StructField, StructType},
+    expressions::{
+        column_expr, ArrayData, BinaryOperator, Expression as Expr, MapData, Scalar, StructData,
+    },
+    schema::{ArrayType, DataType, MapType, StructField, StructType},
 };
 
 /// Constructs a kernel expression that is passed back as a SharedExpression handle. The expected
@@ -20,7 +22,18 @@ pub unsafe extern "C" fn get_testing_kernel_expression() -> Handle<SharedExpress
         DataType::Primitive(delta_kernel::schema::PrimitiveType::Short),
         false,
     );
-    let array_data = ArrayData::new(array_type.clone(), vec![Scalar::Short(5), Scalar::Short(0)]);
+    let array_data =
+        ArrayData::try_new(array_type.clone(), vec![Scalar::Short(5), Scalar::Short(0)]).unwrap();
+
+    let map_type = MapType::new(DataType::STRING, DataType::STRING, false);
+    let map_data = MapData::try_new(
+        map_type.clone(),
+        [
+            ("key1".to_string(), "val1".to_string()),
+            ("key2".to_string(), "val2".to_string()),
+        ],
+    )
+    .unwrap();
 
     let nested_fields = vec![
         StructField::not_null("a", DataType::INTEGER),
@@ -62,6 +75,7 @@ pub unsafe extern "C" fn get_testing_kernel_expression() -> Handle<SharedExpress
         Expr::null_literal(DataType::SHORT),
         Scalar::Struct(top_level_struct).into(),
         Scalar::Array(array_data).into(),
+        Scalar::Map(map_data).into(),
         Expr::struct_from(vec![Expr::or_from(vec![
             Scalar::Integer(5).into(),
             Scalar::Long(20).into(),

ffi/tests/test-expression-visitor/expected.txt

@@ -30,6 +30,9 @@ And
   Array
     Short(5)
     Short(0)
+  Map
+    String(key1): String(val1)
+    String(key2): String(val2)
   StructExpression
     Or
       Integer(5)