feat: late materialization of vectors in filtered vector search

python/python/tests/test_scalar_index.py

@@ -412,7 +412,7 @@ def make_fts_search(ds):
     assert "ScalarIndexQuery" in plan
     assert "MaterializeIndex" not in plan
     assert "KNNVectorDistance" in plan
-    assert "LanceScan" in plan
+    assert "LanceRead" in plan
     assert make_vec_search(ds).to_table().num_rows == 12
 
     plan = make_fts_search(ds).explain_plan()

rust/lance-datafusion/src/exec.rs

@@ -411,6 +411,21 @@ pub struct ExecutionSummaryCounts {
     pub all_counts: HashMap<String, usize>,
 }
 
+impl ExecutionSummaryCounts {
+    /// Create a new ExecutionSummaryCounts with all values initialized to zero
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Create a new ExecutionSummaryCounts with only custom counts
+    pub fn with_counts(counts: impl IntoIterator<Item = (impl Into<String>, usize)>) -> Self {
+        Self {
+            all_counts: counts.into_iter().map(|(k, v)| (k.into(), v)).collect(),
+            ..Default::default()
+        }
+    }
+}
+
 fn visit_node(node: &dyn ExecutionPlan, counts: &mut ExecutionSummaryCounts) {
     if let Some(metrics) = node.metrics() {
         for (metric_name, count) in metrics.iter_counts() {

rust/lance-tools/src/meta.rs

@@ -47,7 +47,7 @@ impl LanceToolFileMetadata {
             .open_file(&path, &CachedFileSize::unknown())
             .await?;
         let file_metadata = FileReader::read_all_metadata(&file_scheduler).await?;
-        let lance_tool_file_metadata = LanceToolFileMetadata { file_metadata };
+        let lance_tool_file_metadata = Self { file_metadata };
         Ok(lance_tool_file_metadata)
     }
 }

rust/lance/src/dataset.rs

@@ -8873,12 +8873,7 @@ mod tests {
         }
 
         fn make_tx(read_version: u64) -> Transaction {
-            Transaction::new(
-                read_version,
-                Operation::Append { fragments: vec![] },
-                None,
-                None,
-            )
+            Transaction::new(read_version, Operation::Append { fragments: vec![] }, None)
         }
 
         async fn delete_external_tx_file(ds: &Dataset) {
@@ -8939,7 +8934,6 @@ mod tests {
                 ds.load_indices().await.unwrap().as_ref().clone(),
                 &tx_file,
                 &ManifestWriteConfig::default(),
-                None,
             )
             .unwrap();
         let location = write_manifest_file(