cleanup

Author: shehabgamin

sail/benchmark.sh

@@ -38,8 +38,8 @@ echo "Install Python packages"
 python3 -m venv myenv
 source myenv/bin/activate
 pip install --upgrade setuptools wheel
-env RUSTFLAGS="-C target-cpu=native" pip install "pysail==0.3.3" -v --no-binary pysail
-pip install "pyspark[connect]==4.0.0" \
+env RUSTFLAGS="-C target-cpu=native" pip install --no-cache-dir "pysail==0.3.3" -v --no-binary pysail
+pip install "pyspark-client==4.0.0" \
   "protobuf==5.28.3" \
   "grpcio==1.71.2" \
   "grpcio-status==1.71.2" \

sail/query.py

@@ -16,8 +16,7 @@
 
 import os
 os.environ["SAIL_PARQUET__BINARY_AS_STRING"] = "true"
-# os.environ["SAIL_PARQUET__PUSHDOWN_FILTERS"] = "true"
-# os.environ["SAIL_PARQUET__REORDER_FILTERS"] = "true"
+os.environ["SAIL_PARQUET__REORDER_FILTERS"] = "true"
 os.environ["SAIL_RUNTIME__ENABLE_SECONDARY"] = "true"
 os.environ["SAIL_PARQUET__ALLOW_SINGLE_FILE_PARALLELISM"] = "true"
 
@@ -34,7 +33,7 @@
     try:
         start = timeit.default_timer()
         result = spark.sql(query)
-        res = result.collect()
+        res = result.toPandas()
         end = timeit.default_timer()
         if try_num == 0:
             print(res)