diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala index b47231948dc98..204da1d05f1ca 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala @@ -225,16 +225,19 @@ class ClientE2ETestSuite test("spark deep recursion") { var df = spark.range(1) - for (a <- 1 to 500) { + assert(spark.conf.get("spark.connect.grpc.marshallerRecursionLimit").toInt == 4096) + // spark.connect.grpc.marshallerRecursionLimit must be at least 2048, to handle certain + // deep recursion cases. + for (a <- 1 to 600) { df = df.union(spark.range(a, a + 1)) } - assert(df.collect().length == 501) + assert(df.collect().length == 601) } test("handle unknown exception") { var df = spark.range(1) - val limit = spark.conf.get("spark.connect.grpc.marshallerRecursionLimit").toInt + 1 - for (a <- 1 to limit) { + val limit = spark.conf.get("spark.connect.grpc.marshallerRecursionLimit").toInt + for (a <- 1 to 605) { df = df.union(spark.range(a, a + 1)) } val ex = intercept[SparkException] { diff --git a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/common/config/ConnectCommon.scala b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/common/config/ConnectCommon.scala index e244fd13595b2..ed30219444d03 100644 --- a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/common/config/ConnectCommon.scala +++ b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/common/config/ConnectCommon.scala @@ -20,5 +20,5 @@ private[sql] object ConnectCommon { val CONNECT_GRPC_BINDING_PORT: Int = 15002 val CONNECT_GRPC_PORT_MAX_RETRIES: Int = 0 val CONNECT_GRPC_MAX_MESSAGE_SIZE: Int = 128 * 1024 * 1024 - val CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT: Int = 1024 + val CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT: Int = 4096 }