-
Notifications
You must be signed in to change notification settings - Fork 234
/
run_sql_examples.sh
executable file
·62 lines (56 loc) · 2.01 KB
/
run_sql_examples.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
#!/bin/bash
set -ex
set -o pipefail
source env_setup.sh
# You might want to set SPARK_EXTRA to do things like log more info
function run_example () {
local sql_file="$1"
local extra="$2"
EXTENSIONS=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
if [ -n "$EXTRA_EXTENSIONS" ]; then
EXTENSIONS="$EXTENSIONS,$EXTRA_EXTENSIONS"
fi
# shellcheck disable=SC2046,SC2086
${SPARK_HOME}/bin/spark-sql --master local[5] \
--conf spark.eventLog.enabled=true \
--conf spark.sql.extensions=$EXTENSIONS \
--conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog \
--conf spark.sql.catalog.spark_catalog.type=hive \
--conf spark.sql.catalog.local=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.local.type=hadoop \
--conf "spark.sql.catalog.local.warehouse=$PWD/warehouse" \
${extra} ${SPARK_EXTRA} \
$(cat "${sql_file}.conf" || echo "") \
--name "${sql_file}" \
-f "${sql_file}" 2>&1 | tee -a "${sql_file}.out" || ls "${sql_file}.expected_to_fail"
}
# If you want to look at them
# ${SPARK_PATH}/sbin/start-history-server.sh
if [ $# -eq 1 ]; then
if [[ "$1" != *"gluten_only"* ]]; then
run_example "sql/$1"
else
echo "Processing gluten ${sql_file}"
# shellcheck disable=SC2046
run_example "$sql_file"
fi
else
# For each SQL
for sql_file in sql/*.sql; do
if [[ "$sql_file" != *"_only"* ]]; then
echo "Processing ${sql_file}"
# shellcheck disable=SC2046
run_example "$sql_file"
elif [[ "$sql_file" != *"gluten_only"* && "$GLUTEN_EXISTS" == "true" ]]; then
echo "Processing gluten ${sql_file}"
# shellcheck disable=SC2046
run_example "$sql_file"
elif [[ "$sql_file" != *"gluten_udf_only"* && "$GLUTEN_UDF_EXISTS" == "true" ]]; then
echo "Processing gluten UDF ${sql_file}"
# shellcheck disable=SC2046
run_example "$sql_file"
else
echo "Skipping $sql_file since we did not find gluten and this is restricted example."
fi
done
fi