Try and make run more flex

high-performance-spark · Aug 29, 2023 · 6a29d69 · 6a29d69
1 parent 5224ba0
commit 6a29d69
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/python/examples/bad_pyspark.py b/python/examples/bad_pyspark.py
@@ -170,9 +170,12 @@ def _test():
     (failure_count, test_count) = doctest.testmod(
         globs=globs, optionflags=doctest.ELLIPSIS
     )
+    print("All tests done, stopping Spark context.")
     globs["sc"].stop()
     if failure_count:
         exit(-1)
+    else:
+        exit(0)
 
 
 if __name__ == "__main__":

diff --git a/run_pyspark_examples.sh b/run_pyspark_examples.sh
@@ -2,6 +2,8 @@
 
 source env_setup.sh
 
+pip install -r ./python/requirements.txt
+
 for ex in python/examples/*.py; do
   spark-submit \
 	       --master local[5] \
@@ -13,5 +15,5 @@ for ex in python/examples/*.py; do
 	       --conf "spark.sql.catalog.local.warehouse=$PWD/warehouse" \
 	       $(cat "${ex}.conf" || echo "") \
 	       --name "${ex}" \
-	       ${ex} | tee -a "${ex}.out"
+	       ${ex} 2>&1 | tee -a "${ex}.out"
 done