[SPARK-58730][SQL] Support the ANSI SQL/JSON JSON_ARRAY constructor function

common/utils/src/main/resources/error/error-conditions.json

@@ -1953,6 +1953,11 @@
           "The <inputName> value must to be a <requireType> literal of <validValues>, but got <inputValue>."
         ]
       },
+      "INVALID_JSON_FORMAT_JSON_INPUT" : {
+        "message" : [
+          "<functionName> FORMAT JSON requires a string argument containing JSON text, but the argument at position <position> has type <inputType>."
+        ]
+      },
       "INVALID_JSON_MAP_KEY_TYPE" : {
         "message" : [
           "Input schema <schema> can only contain STRING as a key type for a MAP."
@@ -1963,6 +1968,11 @@
           "<functionName> has an invalid or unsupported JSON path <path>. Only simple, wildcard-free paths are supported."
         ]
       },
+      "INVALID_JSON_RETURNING_TYPE" : {
+        "message" : [
+          "<functionName> RETURNING clause must be STRING (CHAR/VARCHAR are normalized to STRING), but got <returningType>."
+        ]
+      },
       "INVALID_JSON_SCALAR_RETURNING_TYPE" : {
         "message" : [
           "<functionName> cannot return a value of type <returningType>. The RETURNING type must be a scalar (string, numeric, boolean, or datetime) type."
@@ -4440,6 +4450,12 @@
     ],
     "sqlState" : "2203G"
   },
+  "INVALID_JSON_FORMAT_JSON_VALUE" : {
+    "message" : [
+      "<functionName> FORMAT JSON argument at position <position> must contain exactly one well-formed JSON value, but got <value>."
+    ],
+    "sqlState" : "22032"
+  },
   "INVALID_JSON_RECORD_TYPE" : {
     "message" : [
       "Detected an invalid type of a JSON record while inferring a common schema in the mode <failFastMode>. Expected a STRUCT type, but found <invalidType>."

docs/sql-ref-ansi-compliance.md

@@ -411,6 +411,7 @@ Below is a list of all the keywords in Spark SQL.
 
 |Keyword|Spark SQL<br/>ANSI Mode|Spark SQL<br/>NonANSI Mode|SQL-2016|
 |--|----------------------|-------------------------|--------|
+|ABSENT|non-reserved|non-reserved|non-reserved|
 |ADD|non-reserved|non-reserved|non-reserved|
 |AFTER|non-reserved|non-reserved|non-reserved|
 |AGGREGATE|non-reserved|non-reserved|non-reserved|
@@ -618,6 +619,7 @@ Below is a list of all the keywords in Spark SQL.
 |ITERATE|non-reserved|non-reserved|non-reserved|
 |JOIN|reserved|strict-non-reserved|reserved|
 |JSON|non-reserved|non-reserved|non-reserved|
+|JSON_ARRAY|non-reserved|non-reserved|reserved|
 |JSON_TABLE|non-reserved|non-reserved|reserved|
 |JSON_VALUE|non-reserved|non-reserved|reserved|
 |KEY|non-reserved|non-reserved|non-reserved|

docs/sql-ref-syntax-qry-select-json-array.md

@@ -0,0 +1,125 @@
+---
+layout: global
+title: JSON_ARRAY
+displayTitle: JSON_ARRAY
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `JSON_ARRAY` constructor function builds a JSON array from a list of argument values and
+returns it as JSON text. This is the SQL-standard way (SQL:2016) to assemble a JSON array inline,
+and is commonly used to migrate queries from other systems such as Oracle, DB2, and MySQL.
+`JSON_ARRAY` is an expression that can appear anywhere a value is allowed.
+
+Each argument is serialized with the same JSON writer as the built-in `to_json` function, so
+numbers, decimals, booleans, dates, timestamps, and nested structs/arrays/maps render the same way.
+Null-field handling inside a struct argument therefore follows
+`spark.sql.jsonGenerator.ignoreNullFields`, exactly as `to_json` does; the `ON NULL` clause below
+controls only the top-level array elements.
+
+### Syntax
+
+```sql
+JSON_ARRAY ( [ value [ FORMAT JSON ] [, ...] ]
+             [ { NULL | ABSENT } ON NULL ]
+             [ RETURNING data_type ] )
+```
+
+### Parameters
+
+* **value**
+
+    An expression producing an element of the array. Arguments may have different types and may be
+    nested `JSON_ARRAY` constructors. `JSON_ARRAY()` with no arguments produces the empty array
+    `[]`.
+
+* **FORMAT JSON**
+
+    Marks a string `value` as already-JSON text, so it is spliced into the array verbatim instead
+    of being quoted as a JSON string. For example, `JSON_ARRAY('[1,2]')` produces `["[1,2]"]`, while
+    `JSON_ARRAY('[1,2]' FORMAT JSON)` produces `[[1,2]]`. A nested `JSON_ARRAY` constructor carries
+    `FORMAT JSON` implicitly, so `JSON_ARRAY(JSON_ARRAY(1))` produces `[[1]]`. `FORMAT JSON` requires
+    a string argument (an untyped `NULL` literal is also accepted and follows the `ON NULL`
+    behavior, exactly as a non-`FORMAT JSON` `NULL` would). At runtime, a non-null `FORMAT JSON`
+    value must contain exactly one well-formed JSON value; malformed text or multiple top-level
+    values raise an error. This decision is fixed from the query text and does not depend on query
+    optimization: a `JSON_ARRAY` result that reaches an argument through a column reference is a
+    plain `STRING` and is quoted, whether or not the optimizer inlines it.
+
+* **{ NULL | ABSENT } ON NULL**
+
+    How to handle a `NULL` element:
+    * `ABSENT ON NULL` (the default) omits `NULL` elements from the array.
+    * `NULL ON NULL` keeps them as JSON `null` values.
+
+* **RETURNING data_type**
+
+    The type of the result. It must be a string type; the result is JSON text. If `RETURNING` is
+    omitted, the result type is `STRING`. `CHAR` / `VARCHAR` are normalized to `STRING` (the length
+    is not enforced, because the fragment is serialized directly).
+
+### Examples
+
+```sql
+-- Construct an array from a mixed value list
+SELECT json_array(1, 'x', true);
++---------------------------+
+|json_array(1, x, true)     |
++---------------------------+
+|[1,"x",true]               |
++---------------------------+
+
+-- ABSENT ON NULL (the default) drops NULL elements
+SELECT json_array(1, NULL, 3);
++------------------------+
+|json_array(1, NULL, 3)  |
++------------------------+
+|[1,3]                   |
++------------------------+
+
+-- NULL ON NULL keeps them as JSON null
+SELECT json_array(1, NULL, 3 NULL ON NULL);
++--------------------------------------+
+|json_array(1, NULL, 3 NULL ON NULL)   |
++--------------------------------------+
+|[1,null,3]                            |
++--------------------------------------+
+
+-- A nested JSON_ARRAY is spliced in raw (implicit FORMAT JSON)
+SELECT json_array(json_array(1, 2), 3);
++---------------------------------+
+|json_array(json_array(1, 2), 3)  |
++---------------------------------+
+|[[1,2],3]                        |
++---------------------------------+
+
+-- FORMAT JSON splices an already-JSON string verbatim
+SELECT json_array('[1,2]' FORMAT JSON);
++----------------------------------+
+|json_array([1,2] FORMAT JSON)     |
++----------------------------------+
+|[[1,2]]                           |
++----------------------------------+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
+* [JSON_VALUE](sql-ref-syntax-qry-select-json-value.html)
+* [JSON_TABLE](sql-ref-syntax-qry-select-json-table.html)
+* [Built-in Functions](sql-ref-functions-builtin.html)

docs/sql-ref-syntax-qry-select.md

@@ -213,6 +213,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { [ [ named_expression | regex_column_
 * [Set Operators](sql-ref-syntax-qry-select-setops.html)
 * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
 * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+* [JSON_ARRAY](sql-ref-syntax-qry-select-json-array.html)
 * [JSON_VALUE](sql-ref-syntax-qry-select-json-value.html)
 * [Window Function](sql-ref-syntax-qry-select-window.html)
 * [CASE Clause](sql-ref-syntax-qry-select-case.html)

docs/sql-ref-syntax.md

@@ -83,6 +83,7 @@ ability to generate logical and physical plan for a given query using
    * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
    * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
    * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+   * [JSON_ARRAY](sql-ref-syntax-qry-select-json-array.html)
    * [JSON_TABLE](sql-ref-syntax-qry-select-json-table.html)
    * [JSON_VALUE](sql-ref-syntax-qry-select-json-value.html)
    * [WHERE Clause](sql-ref-syntax-qry-select-where.html)

sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4

@@ -122,12 +122,13 @@ BANG: '!';
 
 // NOTE: If you add a new token in the list below, you should update the list of keywords
 // and reserved tag in `docs/sql-ref-ansi-compliance.md#sql-keywords`, and
-// modify `ParserUtils.toExprAlias()` which assumes all keywords are between `ADD` and `ZONE`.
+// modify `ParserUtils.toExprAlias()` which assumes all keywords are between `ABSENT` and `ZONE`.
 
 //============================
 // Start of the keywords list
 //============================
 //--SPARK-KEYWORD-LIST-START
+ABSENT: 'ABSENT';
 ADD: 'ADD';
 AFTER: 'AFTER';
 AGGREGATE: 'AGGREGATE';
@@ -335,6 +336,7 @@ ITEMS: 'ITEMS';
 ITERATE: 'ITERATE';
 JOIN: 'JOIN';
 JSON: 'JSON';
+JSON_ARRAY: 'JSON_ARRAY';
 JSON_TABLE: 'JSON_TABLE';
 JSON_VALUE: 'JSON_VALUE';
 KEY: 'KEY';

sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4

@@ -1466,6 +1466,11 @@ primaryExpression
       (RETURNING returning=dataType)?
       (emptyBehavior=jsonValueBehavior ON EMPTY)?
       (errorBehavior=jsonValueBehavior ON ERROR)? RIGHT_PAREN                                  #jsonValue
+    | JSON_ARRAY LEFT_PAREN
+      (values+=jsonArrayValue (COMMA values+=jsonArrayValue)*)?
+      (nullBehavior=jsonConstructorNullBehavior ON NULL)?
+      (RETURNING returning=dataType)?
+      RIGHT_PAREN                                  #jsonArray
     | constant                                                                                 #constantDefault
     | ASTERISK exceptClause?                                                                   #star
     | qualifiedName DOT ASTERISK exceptClause?                                                 #star
@@ -1500,6 +1505,20 @@ jsonValueBehavior
     | DEFAULT defaultExpr=expression                                                            #jsonValueBehaviorDefault
     ;
 
+// The behavior selected by JSON_ARRAY/JSON_OBJECT `... ON NULL` clause: NULL keeps nulls, ABSENT
+// drops null elements/pairs.
+jsonConstructorNullBehavior
+    : NULL                                                                                     #jsonConstructorNullBehaviorNull
+    | ABSENT                                                                                   #jsonConstructorNullBehaviorAbsent
+    ;
+
+// A JSON_ARRAY element. The optional `FORMAT JSON` clause marks a string argument as already-JSON
+// text, so it is spliced into the array verbatim instead of being quoted as a JSON string. A
+// lexically-nested JSON constructor (e.g. JSON_ARRAY(JSON_ARRAY(1))) carries this implicitly.
+jsonArrayValue
+    : value=expression (FORMAT JSON)?
+    ;
+
 semiStructuredExtractionPath
     : jsonPathFirstPart (jsonPathParts)*
     ;
@@ -2081,7 +2100,8 @@ operatorPipeSetAssignmentSeq
 // The non-reserved keywords are listed below. Keywords not in this list are reserved keywords.
 ansiNonReserved
 //--ANSI-NON-RESERVED-START
-    : ADD
+    : ABSENT
+    | ADD
     | AFTER
     | AGGREGATE
     | ALIGN
@@ -2244,6 +2264,7 @@ ansiNonReserved
     | ITEMS
     | ITERATE
     | JSON
+    | JSON_ARRAY
     | JSON_TABLE
     | JSON_VALUE
     | KEY
@@ -2482,7 +2503,8 @@ strictNonReserved
 
 nonReserved
 //--DEFAULT-NON-RESERVED-START
-    : ADD
+    : ABSENT
+    | ADD
     | AFTER
     | AGGREGATE
     | ALIGN
@@ -2683,6 +2705,7 @@ nonReserved
     | ITEMS
     | ITERATE
     | JSON
+    | JSON_ARRAY
     | JSON_TABLE
     | JSON_VALUE
     | KEY

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyDefaultCollation.scala

@@ -424,8 +424,11 @@ object ApplyDefaultCollation extends Rule[LogicalPlan] {
       case cast @ Cast(e: DefaultStringProducingExpression, dt, _, _) if newType == dt =>
         cast.copy(child = e.withNewChildren(e.children.map(inner)))
 
-      // Add cast on top of [[DefaultStringProducingExpression]].
-      case e: DefaultStringProducingExpression =>
+      // Add cast on top of [[DefaultStringProducingExpression]], unless it already carries an
+      // explicit (non-default) string collation -- e.g. `JSON_ARRAY(... RETURNING STRING COLLATE
+      // UTF8_BINARY)` -- which the user chose deliberately and the object/view default must not
+      // override.
+      case e: DefaultStringProducingExpression if !hasExplicitStringCollation(e.dataType) =>
         Cast(e.withNewChildren(e.children.map(inner)), newType)
 
       case other =>
@@ -440,6 +443,17 @@ object ApplyDefaultCollation extends Rule[LogicalPlan] {
   private def hasDefaultStringCharOrVarcharType(dataType: DataType): Boolean =
     dataType.existsRecursively(isDefaultStringCharOrVarcharType)
 
+  /**
+   * A [[StringType]] carrying an explicit, non-default collation (distinguished from the default
+   * `StringType` companion by reference identity, matching the convention documented in the
+   * single-pass resolver's `DefaultCollationTypeCoercion`). Such a type reflects a collation the
+   * user chose explicitly, so the object/view default collation must not overwrite it.
+   */
+  private def hasExplicitStringCollation(dataType: DataType): Boolean = dataType match {
+    case st: StringType => !st.eq(StringType)
+    case _ => false
+  }
+
   private def replaceColumnTypes(
       colTypes: Seq[QualifiedColType],
       collation: String): Seq[QualifiedColType] = {

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/DefaultCollationTypeCoercion.scala

@@ -45,7 +45,12 @@ object DefaultCollationTypeCoercion {
   def apply(expression: Expression, collation: String): Expression = {
     val collatedStringType = StringType(collation)
     expression match {
-      case _: DefaultStringProducingExpression if collatedStringType != StringType =>
+      // Skip an expression that already carries an explicit, non-default string collation (e.g.
+      // `JSON_ARRAY(... RETURNING STRING COLLATE UTF8_BINARY)`): the user chose it deliberately, so
+      // the object/view default must not overwrite it. `isDefaultStringType` distinguishes the two
+      // by reference identity.
+      case e: DefaultStringProducingExpression
+          if collatedStringType != StringType && !isExplicitlyCollatedStringType(e.dataType) =>
         Cast(child = expression, dataType = collatedStringType)
       case literal: Literal if hasDefaultStringType(literal.dataType) =>
         literal.copy(dataType = replaceDefaultStringType(literal.dataType, collatedStringType))
@@ -86,6 +91,10 @@ object DefaultCollationTypeCoercion {
       case _ => false
     }
 
+  /** A [[StringType]] with an explicit, non-default collation (see [[isDefaultStringType]]). */
+  private def isExplicitlyCollatedStringType(dataType: DataType): Boolean =
+    dataType.isInstanceOf[StringType] && !isDefaultStringType(dataType)
+
   /**
    * When a default collation is specified for a View,
    * and the cast's dataType contains companion object [[StringType]],