[SPARK-49685][CONNECT] Increase CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT from 1024 to 2048

connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/ClientE2ETestSuite.scala

@@ -225,16 +225,19 @@ class ClientE2ETestSuite
 
   test("spark deep recursion") {
     var df = spark.range(1)
-    for (a <- 1 to 500) {
+    assert(spark.conf.get("spark.connect.grpc.marshallerRecursionLimit").toInt == 4096)
+    // spark.connect.grpc.marshallerRecursionLimit must be at least 2048, to handle certain
+    // deep recursion cases.
+    for (a <- 1 to 600) {
       df = df.union(spark.range(a, a + 1))
     }
-    assert(df.collect().length == 501)
+    assert(df.collect().length == 601)
   }
 
   test("handle unknown exception") {
     var df = spark.range(1)
-    val limit = spark.conf.get("spark.connect.grpc.marshallerRecursionLimit").toInt + 1
-    for (a <- 1 to limit) {
+    val limit = spark.conf.get("spark.connect.grpc.marshallerRecursionLimit").toInt
+    for (a <- 1 to 605) {
       df = df.union(spark.range(a, a + 1))
     }
     val ex = intercept[SparkException] {

sql/connect/common/src/main/scala/org/apache/spark/sql/connect/common/config/ConnectCommon.scala

@@ -20,5 +20,5 @@ private[sql] object ConnectCommon {
   val CONNECT_GRPC_BINDING_PORT: Int = 15002
   val CONNECT_GRPC_PORT_MAX_RETRIES: Int = 0
   val CONNECT_GRPC_MAX_MESSAGE_SIZE: Int = 128 * 1024 * 1024
-  val CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT: Int = 1024
+  val CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT: Int = 4096
 }