Skip to content

Commit 8a41546

Browse files
Pajarajasummaryzb
Pajaraja
authored and
summaryzb
committed
[SPARK-51720][SQL] Add Cross Join as legal in recursion of Recursive CTE
### What changes were proposed in this pull request? Add Cross Joins as legal in recursion in Recursive CTEs. ### Why are the changes needed? Cross join is allowed in the recursion plan of Recursive CTEs in postgreSQL. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add test to golden file cte-recursion.sql. ### Was this patch authored or co-authored using generative AI tooling? No. Closes apache#50308 from Pajaraja/pavle-martinovic_data/CrossJoinRecursiveCTE2. Authored-by: pavle-martinovic_data <[email protected]> Signed-off-by: Peter Toth <[email protected]>
1 parent e72fd6b commit 8a41546

File tree

4 files changed

+246
-2
lines changed

4 files changed

+246
-2
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import scala.collection.mutable
2121

2222
import org.apache.spark.sql.AnalysisException
2323
import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
24-
import org.apache.spark.sql.catalyst.plans.{Inner, LeftAnti, LeftOuter, LeftSemi, RightOuter}
24+
import org.apache.spark.sql.catalyst.plans.{Cross, Inner, LeftAnti, LeftOuter, LeftSemi, RightOuter}
2525
import org.apache.spark.sql.catalyst.plans.logical._
2626
import org.apache.spark.sql.catalyst.rules.Rule
2727
import org.apache.spark.sql.catalyst.trees.TreePattern.{CTE, PLAN_EXPRESSION}
@@ -220,6 +220,9 @@ object ResolveWithCTE extends Rule[LogicalPlan] {
220220
case Join(left, right, Inner, _, _) =>
221221
checkIfSelfReferenceIsPlacedCorrectly(left, cteId, allowRecursiveRef)
222222
checkIfSelfReferenceIsPlacedCorrectly(right, cteId, allowRecursiveRef)
223+
case Join(left, right, Cross, _, _) =>
224+
checkIfSelfReferenceIsPlacedCorrectly(left, cteId, allowRecursiveRef)
225+
checkIfSelfReferenceIsPlacedCorrectly(right, cteId, allowRecursiveRef)
223226
case Join(left, right, LeftOuter, _, _) =>
224227
checkIfSelfReferenceIsPlacedCorrectly(left, cteId, allowRecursiveRef)
225228
checkIfSelfReferenceIsPlacedCorrectly(right, cteId, allowRecursiveRef = false)

‎sql/core/src/test/resources/sql-tests/analyzer-results/cte-recursion.sql.out‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1167,3 +1167,92 @@ WithCTE
11671167
+- Project [a#x]
11681168
+- SubqueryAlias t1
11691169
+- CTERelationRef xxxx, true, [a#x, b#x, c#x], false, false
1170+
1171+
1172+
-- !query
1173+
CREATE TABLE tb (next INT)
1174+
-- !query analysis
1175+
CreateDataSourceTableCommand `spark_catalog`.`default`.`tb`, false
1176+
1177+
1178+
-- !query
1179+
INSERT INTO tb VALUES (0), (1)
1180+
-- !query analysis
1181+
InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_dir}/tb, false, Parquet, [path=file:[not included in comparison]/{warehouse_dir}/tb], Append, `spark_catalog`.`default`.`tb`, org.apache.spark.sql.execution.datasources.InMemoryFileIndex(file:[not included in comparison]/{warehouse_dir}/tb), [next]
1182+
+- Project [cast(col1#x as int) AS next#x]
1183+
+- LocalRelation [col1#x]
1184+
1185+
1186+
-- !query
1187+
WITH RECURSIVE t(n) AS (
1188+
SELECT 1
1189+
UNION ALL
1190+
SELECT next FROM t CROSS JOIN tb
1191+
)
1192+
SELECT * FROM t LIMIT 63
1193+
-- !query analysis
1194+
WithCTE
1195+
:- CTERelationDef xxxx, false
1196+
: +- SubqueryAlias t
1197+
: +- Project [1#x AS n#x]
1198+
: +- UnionLoop xxxx
1199+
: :- Project [1 AS 1#x]
1200+
: : +- OneRowRelation
1201+
: +- Project [next#x]
1202+
: +- Join Cross
1203+
: :- SubqueryAlias t
1204+
: : +- Project [1#x AS n#x]
1205+
: : +- UnionLoopRef xxxx, [1#x], false
1206+
: +- SubqueryAlias spark_catalog.default.tb
1207+
: +- Relation spark_catalog.default.tb[next#x] parquet
1208+
+- GlobalLimit 63
1209+
+- LocalLimit 63
1210+
+- Project [n#x]
1211+
+- SubqueryAlias t
1212+
+- CTERelationRef xxxx, true, [n#x], false, false
1213+
1214+
1215+
-- !query
1216+
DROP TABLE tb
1217+
-- !query analysis
1218+
DropTable false, false
1219+
+- ResolvedIdentifier V2SessionCatalog(spark_catalog), default.tb
1220+
1221+
1222+
-- !query
1223+
WITH RECURSIVE
1224+
x(id) AS (SELECT 1 UNION SELECT 2),
1225+
t(id, xid) AS (
1226+
SELECT 0 AS id, 0 AS xid
1227+
UNION ALL
1228+
SELECT t.id + 1, xid * 10 + x.id FROM t CROSS JOIN x WHERE t.id < 3
1229+
)
1230+
SELECT * FROM t
1231+
-- !query analysis
1232+
WithCTE
1233+
:- CTERelationDef xxxx, false
1234+
: +- SubqueryAlias x
1235+
: +- Project [1#x AS id#x]
1236+
: +- Distinct
1237+
: +- Union false, false
1238+
: :- Project [1 AS 1#x]
1239+
: : +- OneRowRelation
1240+
: +- Project [2 AS 2#x]
1241+
: +- OneRowRelation
1242+
:- CTERelationDef xxxx, false
1243+
: +- SubqueryAlias t
1244+
: +- Project [id#x AS id#x, xid#x AS xid#x]
1245+
: +- UnionLoop xxxx
1246+
: :- Project [0 AS id#x, 0 AS xid#x]
1247+
: : +- OneRowRelation
1248+
: +- Project [(id#x + 1) AS (id + 1)#x, ((xid#x * 10) + id#x) AS ((xid * 10) + id)#x]
1249+
: +- Filter (id#x < 3)
1250+
: +- Join Cross
1251+
: :- SubqueryAlias t
1252+
: : +- Project [id#x AS id#x, xid#x AS xid#x]
1253+
: : +- UnionLoopRef xxxx, [id#x, xid#x], false
1254+
: +- SubqueryAlias x
1255+
: +- CTERelationRef xxxx, true, [id#x], false, false, 2
1256+
+- Project [id#x, xid#x]
1257+
+- SubqueryAlias t
1258+
+- CTERelationRef xxxx, true, [id#x, xid#x], false, false

‎sql/core/src/test/resources/sql-tests/inputs/cte-recursion.sql‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -462,4 +462,28 @@ WITH RECURSIVE t1(a,b,c) AS (
462462
SELECT 1,1,1
463463
UNION ALL
464464
SELECT a+1,a+1,a+1 FROM t1)
465-
SELECT a FROM t1 LIMIT 5;
465+
SELECT a FROM t1 LIMIT 5;
466+
467+
-- CROSS JOIN example
468+
CREATE TABLE tb (next INT);
469+
470+
INSERT INTO tb VALUES (0), (1);
471+
472+
WITH RECURSIVE t(n) AS (
473+
SELECT 1
474+
UNION ALL
475+
SELECT next FROM t CROSS JOIN tb
476+
)
477+
SELECT * FROM t LIMIT 63;
478+
479+
DROP TABLE tb;
480+
-- CROSS JOIN example 2
481+
482+
WITH RECURSIVE
483+
x(id) AS (SELECT 1 UNION SELECT 2),
484+
t(id, xid) AS (
485+
SELECT 0 AS id, 0 AS xid
486+
UNION ALL
487+
SELECT t.id + 1, xid * 10 + x.id FROM t CROSS JOIN x WHERE t.id < 3
488+
)
489+
SELECT * FROM t

‎sql/core/src/test/resources/sql-tests/results/cte-recursion.sql.out‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1050,3 +1050,131 @@ struct<a:int>
10501050
3
10511051
4
10521052
5
1053+
1054+
1055+
-- !query
1056+
CREATE TABLE tb (next INT)
1057+
-- !query schema
1058+
struct<>
1059+
-- !query output
1060+
1061+
1062+
1063+
-- !query
1064+
INSERT INTO tb VALUES (0), (1)
1065+
-- !query schema
1066+
struct<>
1067+
-- !query output
1068+
1069+
1070+
1071+
-- !query
1072+
WITH RECURSIVE t(n) AS (
1073+
SELECT 1
1074+
UNION ALL
1075+
SELECT next FROM t CROSS JOIN tb
1076+
)
1077+
SELECT * FROM t LIMIT 63
1078+
-- !query schema
1079+
struct<n:int>
1080+
-- !query output
1081+
0
1082+
0
1083+
0
1084+
0
1085+
0
1086+
0
1087+
0
1088+
0
1089+
0
1090+
0
1091+
0
1092+
0
1093+
0
1094+
0
1095+
0
1096+
0
1097+
0
1098+
0
1099+
0
1100+
0
1101+
0
1102+
0
1103+
0
1104+
0
1105+
0
1106+
0
1107+
0
1108+
0
1109+
0
1110+
0
1111+
0
1112+
1
1113+
1
1114+
1
1115+
1
1116+
1
1117+
1
1118+
1
1119+
1
1120+
1
1121+
1
1122+
1
1123+
1
1124+
1
1125+
1
1126+
1
1127+
1
1128+
1
1129+
1
1130+
1
1131+
1
1132+
1
1133+
1
1134+
1
1135+
1
1136+
1
1137+
1
1138+
1
1139+
1
1140+
1
1141+
1
1142+
1
1143+
1
1144+
1145+
1146+
-- !query
1147+
DROP TABLE tb
1148+
-- !query schema
1149+
struct<>
1150+
-- !query output
1151+
1152+
1153+
1154+
-- !query
1155+
WITH RECURSIVE
1156+
x(id) AS (SELECT 1 UNION SELECT 2),
1157+
t(id, xid) AS (
1158+
SELECT 0 AS id, 0 AS xid
1159+
UNION ALL
1160+
SELECT t.id + 1, xid * 10 + x.id FROM t CROSS JOIN x WHERE t.id < 3
1161+
)
1162+
SELECT * FROM t
1163+
-- !query schema
1164+
struct<id:int,xid:int>
1165+
-- !query output
1166+
0 0
1167+
1 1
1168+
1 2
1169+
2 11
1170+
2 12
1171+
2 21
1172+
2 22
1173+
3 111
1174+
3 112
1175+
3 121
1176+
3 122
1177+
3 211
1178+
3 212
1179+
3 221
1180+
3 222

0 commit comments

Comments
 (0)