include refine expr columns

Author: wkalt

rust/lance/src/dataset/scanner.rs

@@ -2987,6 +2987,12 @@ impl Scanner {
                     .with_row_id()
                     .union_column(&q.column, OnMissing::Error)?;
 
+                // Include columns referenced by refine expression
+                if let Some(ref refine_expr) = filter_plan.refine_expr {
+                    let refine_cols = Planner::column_names_in_expr(refine_expr);
+                    scan_projection = scan_projection.union_columns(refine_cols, OnMissing::Error)?;
+                }
+
                 scan_projection.with_row_addr =
                     self.projection_plan.physical_projection.with_row_addr;
 
@@ -3003,16 +3009,7 @@ impl Scanner {
 
                 // Apply refine filter if present
                 let plan = if let Some(ref refine_expr) = filter_plan.refine_expr {
-                    // Convert logical expression to physical expression
-                    let schema = plan.schema();
-                    let df_schema = DFSchema::try_from(schema.as_ref().clone())?;
-                    let physical_expr =
-                        create_physical_expr(refine_expr, &df_schema, &ExecutionProps::new())?;
-
-                    Arc::new(datafusion::physical_plan::filter::FilterExec::try_new(
-                        physical_expr,
-                        plan,
-                    )?)
+                    Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?)
                 } else {
                     plan
                 };