[SPARK-51657] UTF8_BINARY default table collation shown by default in Desc As JSON (v1)

### What changes were proposed in this pull request?

Display `UTF8_BINARY` default table collation by default in Desc As JSON for v1 table.

Note this only affects the `desc as json` output and not `desc table extended` -- desc table does NOT always show the top-level table collation (if omitted in the`desc table` output, it is `UTF8_BINARY`)

### Why are the changes needed?

If the CatalogTable `collation` property is None, then it is by default `UTF8_BINARY`. This update is for completeness for the `desc as json` output.

### Does this PR introduce _any_ user-facing change?

Yes, affects the JSON result

### How was this patch tested?

Added tests in `v1/DescribeTableSuite.scala`

### Was this patch authored or co-authored using generative AI tooling?

No

Closes #50451 from asl3/asl3/descasjson-defaultcollation.

Authored-by: Amanda Liu <[email protected]>
Signed-off-by: Gengliang Wang <[email protected]>
(cherry picked from commit 13945c8)
Signed-off-by: Gengliang Wang <[email protected]>

Author: asl3

sql/core/src/main/scala/org/apache/spark/sql/execution/command/DescribeRelationJsonCommand.scala

@@ -99,6 +99,10 @@ case class DescribeRelationJsonCommand(
       case _ => throw QueryCompilationErrors.describeAsJsonNotSupportedForV2TablesError()
     }
 
+    // Add default collation if not yet added (addKeyValueToMap only adds unique keys).
+    // Add here to only affect `DESC AS JSON` and not the `DESC TABLE` output.
+    addKeyValueToMap("collation", JString("UTF8_BINARY"), jsonMap)
+
     Seq(Row(compact(render(JObject(jsonMap.toList)))))
   }
 

sql/core/src/test/resources/sql-tests/results/describe.sql.out

@@ -76,7 +76,7 @@ DESCRIBE EXTENDED t AS JSON
 -- !query schema
 struct<json_metadata:string>
 -- !query output
-{"table_name":"t","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"a","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"b","type":{"name":"int"},"nullable":true},{"name":"c","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"d","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true}],"num_buckets":2,"bucket_columns":["a"],"sort_columns":["b"],"location":"file:[not included in comparison]/{warehouse_dir}/t","storage_properties":{"a":"1","b":"2","password":"*********(redacted)"},"created_time [not included in comparison]":"None","last_access [not included in comparison]":"None","created_by [not included in comparison]":"None","type":"MANAGED","provider":"parquet","comment":"table_comment","table_properties":{"e":"3","password":"*********(redacted)","t":"test"},"partition_provider":"Catalog","partition_columns":["c","d"]}
+{"table_name":"t","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"a","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"b","type":{"name":"int"},"nullable":true},{"name":"c","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"d","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true}],"num_buckets":2,"bucket_columns":["a"],"sort_columns":["b"],"location":"file:[not included in comparison]/{warehouse_dir}/t","storage_properties":{"a":"1","b":"2","password":"*********(redacted)"},"created_time [not included in comparison]":"None","last_access [not included in comparison]":"None","created_by [not included in comparison]":"None","type":"MANAGED","provider":"parquet","comment":"table_comment","table_properties":{"e":"3","password":"*********(redacted)","t":"test"},"partition_provider":"Catalog","partition_columns":["c","d"],"collation":"UTF8_BINARY"}
 
 
 -- !query
@@ -303,7 +303,7 @@ DESC EXTENDED t PARTITION (c='Us', d=1) AS JSON
 -- !query schema
 struct<json_metadata:string>
 -- !query output
-{"table_name":"t","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"a","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"b","type":{"name":"int"},"nullable":true},{"name":"c","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"d","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true}],"partition_values":{"c":"Us","d":"1"},"location":"file:[not included in comparison]/{warehouse_dir}/t/c=Us/d=1","storage_properties":{"a":"1","b":"2","password":"*********(redacted)"},"created_time [not included in comparison]":"None","last_access [not included in comparison]":"None","created_by [not included in comparison]":"None","type":"MANAGED","provider":"parquet","num_buckets":2,"bucket_columns":["a"],"sort_columns":["b"],"table_properties":{"password":"*********(redacted)","t":"test"},"partition_provider":"Catalog","partition_columns":["c","d"]}
+{"table_name":"t","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"a","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"b","type":{"name":"int"},"nullable":true},{"name":"c","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"d","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true}],"partition_values":{"c":"Us","d":"1"},"location":"file:[not included in comparison]/{warehouse_dir}/t/c=Us/d=1","storage_properties":{"a":"1","b":"2","password":"*********(redacted)"},"created_time [not included in comparison]":"None","last_access [not included in comparison]":"None","created_by [not included in comparison]":"None","type":"MANAGED","provider":"parquet","num_buckets":2,"bucket_columns":["a"],"sort_columns":["b"],"table_properties":{"password":"*********(redacted)","t":"test"},"partition_provider":"Catalog","partition_columns":["c","d"],"collation":"UTF8_BINARY"}
 
 
 -- !query
@@ -929,7 +929,7 @@ DESC TABLE EXTENDED f PARTITION (B='SPARK', C=TIMESTAMP'2018-11-17 13:33:33') AS
 -- !query schema
 struct<json_metadata:string>
 -- !query output
-{"table_name":"f","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"A","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"B","type":{"name":"binary"},"nullable":true},{"name":"C","type":{"name":"timestamp_ltz"},"nullable":true}],"partition_values":{"B":"SPARK","C":"2018-11-17 13:33:33"},"location":"file:[not included in comparison]/{warehouse_dir}/f/B=SPARK/C=2018-11-17 13%3A33%3A33","partition_parameters":{"numFiles":"1","totalSize":"15","transient_lastDdlTime [not included in comparison]":"None"},"created_time [not included in comparison]":"None","last_access [not included in comparison]":"None","created_by [not included in comparison]":"None","type":"MANAGED","provider":"json","partition_provider":"Catalog","partition_columns":["B","C"]}
+{"table_name":"f","catalog_name":"spark_catalog","namespace":["default"],"schema_name":"default","columns":[{"name":"A","type":{"name":"string","collation":"UTF8_BINARY"},"nullable":true},{"name":"B","type":{"name":"binary"},"nullable":true},{"name":"C","type":{"name":"timestamp_ltz"},"nullable":true}],"partition_values":{"B":"SPARK","C":"2018-11-17 13:33:33"},"location":"file:[not included in comparison]/{warehouse_dir}/f/B=SPARK/C=2018-11-17 13%3A33%3A33","partition_parameters":{"numFiles":"1","totalSize":"15","transient_lastDdlTime [not included in comparison]":"None"},"created_time [not included in comparison]":"None","last_access [not included in comparison]":"None","created_by [not included in comparison]":"None","type":"MANAGED","provider":"json","partition_provider":"Catalog","partition_columns":["B","C"],"collation":"UTF8_BINARY"}
 
 
 -- !query

sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeTableSuite.scala

@@ -248,6 +248,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
         last_access = Some("UNKNOWN"),
         created_by = Some(s"Spark $SPARK_VERSION"),
         `type` = Some("MANAGED"),
+        collation = Some("UTF8_BINARY"),
         provider = Some("parquet"),
         bucket_columns = Some(List("employee_id")),
         sort_columns = Some(List("employee_name")),
@@ -313,6 +314,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
         last_access = Some("UNKNOWN"),
         created_by = Some(s"Spark $SPARK_VERSION"),
         `type` = Some("MANAGED"),
+        collation = Some("UTF8_BINARY"),
         provider = Some("parquet"),
         bucket_columns = Some(Nil),
         sort_columns = Some(Nil),
@@ -377,6 +379,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
         last_access = Some("UNKNOWN"),
         created_by = Some(s"Spark $SPARK_VERSION"),
         `type` = Some("MANAGED"),
+        collation = Some("UTF8_BINARY"),
         provider = Some("parquet"),
         bucket_columns = Some(Nil),
         sort_columns = Some(Nil),
@@ -412,7 +415,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
            |  id INT
            |)
            |USING parquet COMMENT 'table_comment'
-           |DEFAULT COLLATION UTF8_BINARY
+           |DEFAULT COLLATION DE
            |""".stripMargin
       spark.sql(tableCreationStr)
 
@@ -430,12 +433,12 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
           TableColumn("c1", Type("string", collation = Some("UNICODE_CI"))),
           TableColumn("c2", Type("string", collation = Some("UNICODE_RTRIM"))),
           TableColumn("c3", Type("string", collation = Some("fr"))),
-          TableColumn("c4", Type("string", collation = Some("UTF8_BINARY"))),
+          TableColumn("c4", Type("string", collation = Some("de"))),
           TableColumn("id", Type("int")))),
         last_access = Some("UNKNOWN"),
         created_by = Some(s"Spark $SPARK_VERSION"),
         `type` = Some("MANAGED"),
-        collation = Some("UTF8_BINARY"),
+        collation = Some("de"),
         storage_properties = None,
         provider = Some("parquet"),
         bucket_columns = Some(Nil),
@@ -489,6 +492,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
         last_access = Some("UNKNOWN"),
         created_by = Some(s"Spark $SPARK_VERSION"),
         `type` = Some("MANAGED"),
+        collation = Some("UTF8_BINARY"),
         storage_properties = None,
         provider = Some("parquet"),
         bucket_columns = Some(Nil),
@@ -532,27 +536,27 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
               val jsonValue = firstRow.getString(0)
               val parsedOutput = parse(jsonValue).extract[DescribeTableJson]
 
-              val expectedOutput = DescribeTableJson(
-                table_name = Some("view"),
-                catalog_name = if (isTemp) Some("system") else Some("spark_catalog"),
-                namespace = if (isTemp) Some(List("session")) else Some(List("default")),
-                schema_name = if (isTemp) Some("session") else Some("default"),
-                columns = Some(List(
-                  TableColumn("id", Type("int")),
-                  TableColumn("name", Type("string", collation = Some("UTF8_BINARY"))),
-                  TableColumn("created_at", Type("timestamp_ltz"))
-                )),
-                last_access = Some("UNKNOWN"),
-                created_by = Some(s"Spark $SPARK_VERSION"),
-                `type` = Some("VIEW"),
-                view_text = Some("SELECT * FROM spark_catalog.ns.table"),
-                view_original_text =
-                  if (isTemp) None else Some("SELECT * FROM spark_catalog.ns.table"),
-                // TODO: this is unexpected and temp view should also use COMPENSATION mode.
-                view_schema_mode = if (isTemp) Some("BINDING") else Some("COMPENSATION"),
-                view_catalog_and_namespace = Some("spark_catalog.default"),
-                view_query_output_columns = Some(List("id", "name", "created_at"))
-              )
+          val expectedOutput = DescribeTableJson(
+            table_name = Some("view"),
+            catalog_name = if (isTemp) Some("system") else Some("spark_catalog"),
+            namespace = if (isTemp) Some(List("session")) else Some(List("default")),
+            schema_name = if (isTemp) Some("session") else Some("default"),
+            columns = Some(List(
+              TableColumn("id", Type("int")),
+              TableColumn("name", Type("string", collation = Some("UTF8_BINARY"))),
+              TableColumn("created_at", Type("timestamp_ltz"))
+            )),
+            last_access = Some("UNKNOWN"),
+            created_by = Some(s"Spark $SPARK_VERSION"),
+            `type` = Some("VIEW"),
+            collation = Some("UTF8_BINARY"),
+            view_text = Some("SELECT * FROM spark_catalog.ns.table"),
+            view_original_text = if (isTemp) None else Some("SELECT * FROM spark_catalog.ns.table"),
+            // TODO: this is unexpected and temp view should also use COMPENSATION mode.
+            view_schema_mode = if (isTemp) Some("BINDING") else Some("COMPENSATION"),
+            view_catalog_and_namespace = Some("spark_catalog.default"),
+            view_query_output_columns = Some(List("id", "name", "created_at"))
+          )
 
               assert(iso8601Regex.matches(parsedOutput.created_time.get))
               assert(expectedOutput == parsedOutput.copy(
@@ -738,6 +742,7 @@ trait DescribeTableSuiteBase extends command.DescribeTableSuiteBase
         last_access = Some("UNKNOWN"),
         created_by = Some(s"Spark $SPARK_VERSION"),
         `type` = Some("MANAGED"),
+        collation = Some("UTF8_BINARY"),
         provider = Some("parquet"),
         comment = Some("A table with nested complex types"),
         table_properties = Some(Map(