Skip to content

Commit 5508a8d

Browse files
tobixdevtobixdev
committed
Update slts that are affected by the nl-join-projection-push-down
1 parent 135a946 commit 5508a8d

File tree

5 files changed

+81
-65
lines changed

5 files changed

+81
-65
lines changed

‎datafusion/sqllogictest/test_files/explain.slt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@ physical_plan after OutputRequirements
230230
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/example.csv]]}, projection=[a, b, c], file_type=csv, has_header=true
231231
physical_plan after aggregate_statistics SAME TEXT AS ABOVE
232232
physical_plan after join_selection SAME TEXT AS ABOVE
233+
physical_plan after nl-join-projection-push-down SAME TEXT AS ABOVE
233234
physical_plan after LimitedDistinctAggregation SAME TEXT AS ABOVE
234235
physical_plan after FilterPushdown SAME TEXT AS ABOVE
235236
physical_plan after EnforceDistribution SAME TEXT AS ABOVE
@@ -309,6 +310,7 @@ physical_plan after OutputRequirements
309310
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]
310311
physical_plan after aggregate_statistics SAME TEXT AS ABOVE
311312
physical_plan after join_selection SAME TEXT AS ABOVE
313+
physical_plan after nl-join-projection-push-down SAME TEXT AS ABOVE
312314
physical_plan after LimitedDistinctAggregation SAME TEXT AS ABOVE
313315
physical_plan after FilterPushdown SAME TEXT AS ABOVE
314316
physical_plan after EnforceDistribution SAME TEXT AS ABOVE
@@ -354,6 +356,7 @@ physical_plan after OutputRequirements
354356
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet
355357
physical_plan after aggregate_statistics SAME TEXT AS ABOVE
356358
physical_plan after join_selection SAME TEXT AS ABOVE
359+
physical_plan after nl-join-projection-push-down SAME TEXT AS ABOVE
357360
physical_plan after LimitedDistinctAggregation SAME TEXT AS ABOVE
358361
physical_plan after FilterPushdown SAME TEXT AS ABOVE
359362
physical_plan after EnforceDistribution SAME TEXT AS ABOVE
@@ -594,6 +597,7 @@ physical_plan after OutputRequirements
594597
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/example.csv]]}, projection=[a, b, c], file_type=csv, has_header=true
595598
physical_plan after aggregate_statistics SAME TEXT AS ABOVE
596599
physical_plan after join_selection SAME TEXT AS ABOVE
600+
physical_plan after nl-join-projection-push-down SAME TEXT AS ABOVE
597601
physical_plan after LimitedDistinctAggregation SAME TEXT AS ABOVE
598602
physical_plan after FilterPushdown SAME TEXT AS ABOVE
599603
physical_plan after EnforceDistribution SAME TEXT AS ABOVE

‎datafusion/sqllogictest/test_files/join.slt.part‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -849,9 +849,11 @@ logical_plan
849849
05)----TableScan: department projection=[dept_name]
850850
physical_plan
851851
01)ProjectionExec: expr=[emp_id@1 as emp_id, name@2 as name, dept_name@0 as dept_name]
852-
02)--NestedLoopJoinExec: join_type=Right, filter=name@0 = Alice OR name@0 = Bob
852+
02)--NestedLoopJoinExec: join_type=Right, filter=join_proj_push_down_1@0, projection=[dept_name@0, emp_id@1, name@2]
853853
03)----DataSourceExec: partitions=1, partition_sizes=[1]
854-
04)----DataSourceExec: partitions=1, partition_sizes=[1]
854+
04)----ProjectionExec: expr=[emp_id@0 as emp_id, name@1 as name, name@1 = Alice OR name@1 = Bob as join_proj_push_down_1]
855+
05)------CoalesceBatchesExec: target_batch_size=8192
856+
06)--------DataSourceExec: partitions=1, partition_sizes=[1]
855857

856858
query ITT rowsort
857859
SELECT e.emp_id, e.name, d.dept_name

‎datafusion/sqllogictest/test_files/joins.slt‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3519,10 +3519,16 @@ logical_plan
35193519
04)--SubqueryAlias: t2
35203520
05)----TableScan: annotated_data projection=[a0, a, b, c, d]
35213521
physical_plan
3522-
01)NestedLoopJoinExec: join_type=Inner, filter=example(CAST(a@0 AS Float64), CAST(a@1 AS Float64)) > 3
3523-
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[a0, a, b, c, d], output_ordering=[a@1 ASC, b@2 ASC NULLS LAST, c@3 ASC NULLS LAST], file_type=csv, has_header=true
3524-
03)--RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
3525-
04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[a0, a, b, c, d], output_ordering=[a@1 ASC, b@2 ASC NULLS LAST, c@3 ASC NULLS LAST], file_type=csv, has_header=true
3522+
01)NestedLoopJoinExec: join_type=Inner, filter=example(join_proj_push_down_1@0, join_proj_push_down_2@1) > 3, projection=[a0@0, a@1, b@2, c@3, d@4, a0@6, a@7, b@8, c@9, d@10]
3523+
02)--CoalescePartitionsExec
3524+
03)----ProjectionExec: expr=[a0@0 as a0, a@1 as a, b@2 as b, c@3 as c, d@4 as d, CAST(a@1 AS Float64) as join_proj_push_down_1]
3525+
04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
3526+
05)--------CoalesceBatchesExec: target_batch_size=2
3527+
06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[a0, a, b, c, d], output_ordering=[a@1 ASC, b@2 ASC NULLS LAST, c@3 ASC NULLS LAST], file_type=csv, has_header=true
3528+
07)--ProjectionExec: expr=[a0@0 as a0, a@1 as a, b@2 as b, c@3 as c, d@4 as d, CAST(a@1 AS Float64) as join_proj_push_down_2]
3529+
08)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
3530+
09)------CoalesceBatchesExec: target_batch_size=2
3531+
10)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[a0, a, b, c, d], output_ordering=[a@1 ASC, b@2 ASC NULLS LAST, c@3 ASC NULLS LAST], file_type=csv, has_header=true
35263532

35273533
####
35283534
# Config teardown

‎datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part‎

Lines changed: 44 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -75,51 +75,53 @@ logical_plan
7575
physical_plan
7676
01)SortExec: TopK(fetch=10), expr=[value@1 DESC], preserve_partitioning=[false]
7777
02)--ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value]
78-
03)----NestedLoopJoinExec: join_type=Inner, filter=CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Decimal128(38, 15)) > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@1, projection=[ps_partkey@0, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1]
78+
03)----NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, projection=[ps_partkey@0, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@3]
7979
04)------CoalescePartitionsExec
80-
05)--------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
80+
05)--------ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as sum(partsupp.ps_supplycost * partsupp.ps_availqty), CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) as join_proj_push_down_1]
8181
06)----------CoalesceBatchesExec: target_batch_size=8192
82-
07)------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4
83-
08)--------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
84-
09)----------------CoalesceBatchesExec: target_batch_size=8192
85-
10)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]
82+
07)------------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
83+
08)--------------CoalesceBatchesExec: target_batch_size=8192
84+
09)----------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4
85+
10)------------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
8686
11)--------------------CoalesceBatchesExec: target_batch_size=8192
87-
12)----------------------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=4
87+
12)----------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]
8888
13)------------------------CoalesceBatchesExec: target_batch_size=8192
89-
14)--------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_availqty@2, ps_supplycost@3, s_nationkey@5]
89+
14)--------------------------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=4
9090
15)----------------------------CoalesceBatchesExec: target_batch_size=8192
91-
16)------------------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4
92-
17)--------------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false
93-
18)----------------------------CoalesceBatchesExec: target_batch_size=8192
94-
19)------------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=4
95-
20)--------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
96-
21)----------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false
97-
22)--------------------CoalesceBatchesExec: target_batch_size=8192
98-
23)----------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4
91+
16)------------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_availqty@2, ps_supplycost@3, s_nationkey@5]
92+
17)--------------------------------CoalesceBatchesExec: target_batch_size=8192
93+
18)----------------------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4
94+
19)------------------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false
95+
20)--------------------------------CoalesceBatchesExec: target_batch_size=8192
96+
21)----------------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=4
97+
22)------------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
98+
23)--------------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false
9999
24)------------------------CoalesceBatchesExec: target_batch_size=8192
100-
25)--------------------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]
101-
26)----------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
102-
27)------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false
103-
28)------ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)]
104-
29)--------AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
105-
30)----------CoalescePartitionsExec
106-
31)------------AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
107-
32)--------------CoalesceBatchesExec: target_batch_size=8192
108-
33)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]
109-
34)------------------CoalesceBatchesExec: target_batch_size=8192
110-
35)--------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4
111-
36)----------------------CoalesceBatchesExec: target_batch_size=8192
112-
37)------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)], projection=[ps_availqty@1, ps_supplycost@2, s_nationkey@4]
113-
38)--------------------------CoalesceBatchesExec: target_batch_size=8192
114-
39)----------------------------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4
115-
40)------------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false
116-
41)--------------------------CoalesceBatchesExec: target_batch_size=8192
117-
42)----------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=4
118-
43)------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
119-
44)--------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false
120-
45)------------------CoalesceBatchesExec: target_batch_size=8192
121-
46)--------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4
122-
47)----------------------CoalesceBatchesExec: target_batch_size=8192
123-
48)------------------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]
124-
49)--------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
125-
50)----------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false
100+
25)--------------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4
101+
26)----------------------------CoalesceBatchesExec: target_batch_size=8192
102+
27)------------------------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]
103+
28)--------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
104+
29)----------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false
105+
30)------ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)]
106+
31)--------AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
107+
32)----------CoalescePartitionsExec
108+
33)------------AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
109+
34)--------------CoalesceBatchesExec: target_batch_size=8192
110+
35)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]
111+
36)------------------CoalesceBatchesExec: target_batch_size=8192
112+
37)--------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4
113+
38)----------------------CoalesceBatchesExec: target_batch_size=8192
114+
39)------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)], projection=[ps_availqty@1, ps_supplycost@2, s_nationkey@4]
115+
40)--------------------------CoalesceBatchesExec: target_batch_size=8192
116+
41)----------------------------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4
117+
42)------------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false
118+
43)--------------------------CoalesceBatchesExec: target_batch_size=8192
119+
44)----------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=4
120+
45)------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
121+
46)--------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false
122+
47)------------------CoalesceBatchesExec: target_batch_size=8192
123+
48)--------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4
124+
49)----------------------CoalesceBatchesExec: target_batch_size=8192
125+
50)------------------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]
126+
51)--------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
127+
52)----------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false

0 commit comments

Comments
 (0)