diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 6df35deef807..2922eb02787c 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1953,6 +1953,11 @@ "The value must to be a literal of , but got ." ] }, + "INVALID_JSON_FORMAT_JSON_INPUT" : { + "message" : [ + " FORMAT JSON requires a string argument containing JSON text, but the argument at position has type ." + ] + }, "INVALID_JSON_MAP_KEY_TYPE" : { "message" : [ "Input schema can only contain STRING as a key type for a MAP." @@ -1963,6 +1968,11 @@ " has an invalid or unsupported JSON path . Only simple, wildcard-free paths are supported." ] }, + "INVALID_JSON_RETURNING_TYPE" : { + "message" : [ + " RETURNING clause must be STRING (CHAR/VARCHAR are normalized to STRING), but got ." + ] + }, "INVALID_JSON_SCALAR_RETURNING_TYPE" : { "message" : [ " cannot return a value of type . The RETURNING type must be a scalar (string, numeric, boolean, or datetime) type." @@ -4440,6 +4450,12 @@ ], "sqlState" : "2203G" }, + "INVALID_JSON_FORMAT_JSON_VALUE" : { + "message" : [ + " FORMAT JSON argument at position must contain exactly one well-formed JSON value, but got ." + ], + "sqlState" : "22032" + }, "INVALID_JSON_RECORD_TYPE" : { "message" : [ "Detected an invalid type of a JSON record while inferring a common schema in the mode . Expected a STRUCT type, but found ." diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md index 77c4d96da1ea..74a769b82011 100644 --- a/docs/sql-ref-ansi-compliance.md +++ b/docs/sql-ref-ansi-compliance.md @@ -411,6 +411,7 @@ Below is a list of all the keywords in Spark SQL. |Keyword|Spark SQL
ANSI Mode|Spark SQL
NonANSI Mode|SQL-2016| |--|----------------------|-------------------------|--------| +|ABSENT|non-reserved|non-reserved|non-reserved| |ADD|non-reserved|non-reserved|non-reserved| |AFTER|non-reserved|non-reserved|non-reserved| |AGGREGATE|non-reserved|non-reserved|non-reserved| @@ -618,6 +619,7 @@ Below is a list of all the keywords in Spark SQL. |ITERATE|non-reserved|non-reserved|non-reserved| |JOIN|reserved|strict-non-reserved|reserved| |JSON|non-reserved|non-reserved|non-reserved| +|JSON_ARRAY|non-reserved|non-reserved|reserved| |JSON_TABLE|non-reserved|non-reserved|reserved| |JSON_VALUE|non-reserved|non-reserved|reserved| |KEY|non-reserved|non-reserved|non-reserved| diff --git a/docs/sql-ref-syntax-qry-select-json-array.md b/docs/sql-ref-syntax-qry-select-json-array.md new file mode 100644 index 000000000000..5220d3e5508c --- /dev/null +++ b/docs/sql-ref-syntax-qry-select-json-array.md @@ -0,0 +1,125 @@ +--- +layout: global +title: JSON_ARRAY +displayTitle: JSON_ARRAY +license: | + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--- + +### Description + +The `JSON_ARRAY` constructor function builds a JSON array from a list of argument values and +returns it as JSON text. This is the SQL-standard way (SQL:2016) to assemble a JSON array inline, +and is commonly used to migrate queries from other systems such as Oracle, DB2, and MySQL. +`JSON_ARRAY` is an expression that can appear anywhere a value is allowed. + +Each argument is serialized with the same JSON writer as the built-in `to_json` function, so +numbers, decimals, booleans, dates, timestamps, and nested structs/arrays/maps render the same way. +Null-field handling inside a struct argument therefore follows +`spark.sql.jsonGenerator.ignoreNullFields`, exactly as `to_json` does; the `ON NULL` clause below +controls only the top-level array elements. + +### Syntax + +```sql +JSON_ARRAY ( [ value [ FORMAT JSON ] [, ...] ] + [ { NULL | ABSENT } ON NULL ] + [ RETURNING data_type ] ) +``` + +### Parameters + +* **value** + + An expression producing an element of the array. Arguments may have different types and may be + nested `JSON_ARRAY` constructors. `JSON_ARRAY()` with no arguments produces the empty array + `[]`. + +* **FORMAT JSON** + + Marks a string `value` as already-JSON text, so it is spliced into the array verbatim instead + of being quoted as a JSON string. For example, `JSON_ARRAY('[1,2]')` produces `["[1,2]"]`, while + `JSON_ARRAY('[1,2]' FORMAT JSON)` produces `[[1,2]]`. A nested `JSON_ARRAY` constructor carries + `FORMAT JSON` implicitly, so `JSON_ARRAY(JSON_ARRAY(1))` produces `[[1]]`. `FORMAT JSON` requires + a string argument (an untyped `NULL` literal is also accepted and follows the `ON NULL` + behavior, exactly as a non-`FORMAT JSON` `NULL` would). At runtime, a non-null `FORMAT JSON` + value must contain exactly one well-formed JSON value; malformed text or multiple top-level + values raise an error. This decision is fixed from the query text and does not depend on query + optimization: a `JSON_ARRAY` result that reaches an argument through a column reference is a + plain `STRING` and is quoted, whether or not the optimizer inlines it. + +* **{ NULL | ABSENT } ON NULL** + + How to handle a `NULL` element: + * `ABSENT ON NULL` (the default) omits `NULL` elements from the array. + * `NULL ON NULL` keeps them as JSON `null` values. + +* **RETURNING data_type** + + The type of the result. It must be a string type; the result is JSON text. If `RETURNING` is + omitted, the result type is `STRING`. `CHAR` / `VARCHAR` are normalized to `STRING` (the length + is not enforced, because the fragment is serialized directly). + +### Examples + +```sql +-- Construct an array from a mixed value list +SELECT json_array(1, 'x', true); ++---------------------------+ +|json_array(1, x, true) | ++---------------------------+ +|[1,"x",true] | ++---------------------------+ + +-- ABSENT ON NULL (the default) drops NULL elements +SELECT json_array(1, NULL, 3); ++------------------------+ +|json_array(1, NULL, 3) | ++------------------------+ +|[1,3] | ++------------------------+ + +-- NULL ON NULL keeps them as JSON null +SELECT json_array(1, NULL, 3 NULL ON NULL); ++--------------------------------------+ +|json_array(1, NULL, 3 NULL ON NULL) | ++--------------------------------------+ +|[1,null,3] | ++--------------------------------------+ + +-- A nested JSON_ARRAY is spliced in raw (implicit FORMAT JSON) +SELECT json_array(json_array(1, 2), 3); ++---------------------------------+ +|json_array(json_array(1, 2), 3) | ++---------------------------------+ +|[[1,2],3] | ++---------------------------------+ + +-- FORMAT JSON splices an already-JSON string verbatim +SELECT json_array('[1,2]' FORMAT JSON); ++----------------------------------+ +|json_array([1,2] FORMAT JSON) | ++----------------------------------+ +|[[1,2]] | ++----------------------------------+ +``` + +### Related Statements + +* [SELECT](sql-ref-syntax-qry-select.html) +* [JSON_VALUE](sql-ref-syntax-qry-select-json-value.html) +* [JSON_TABLE](sql-ref-syntax-qry-select-json-table.html) +* [Built-in Functions](sql-ref-functions-builtin.html) diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md index b2afd160c5a3..253c3e686a82 100644 --- a/docs/sql-ref-syntax-qry-select.md +++ b/docs/sql-ref-syntax-qry-select.md @@ -213,6 +213,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { [ [ named_expression | regex_column_ * [Set Operators](sql-ref-syntax-qry-select-setops.html) * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html) * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html) +* [JSON_ARRAY](sql-ref-syntax-qry-select-json-array.html) * [JSON_VALUE](sql-ref-syntax-qry-select-json-value.html) * [Window Function](sql-ref-syntax-qry-select-window.html) * [CASE Clause](sql-ref-syntax-qry-select-case.html) diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md index c8ddecaf6bdb..31424531223d 100644 --- a/docs/sql-ref-syntax.md +++ b/docs/sql-ref-syntax.md @@ -83,6 +83,7 @@ ability to generate logical and physical plan for a given query using * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html) * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html) * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html) + * [JSON_ARRAY](sql-ref-syntax-qry-select-json-array.html) * [JSON_TABLE](sql-ref-syntax-qry-select-json-table.html) * [JSON_VALUE](sql-ref-syntax-qry-select-json-value.html) * [WHERE Clause](sql-ref-syntax-qry-select-where.html) diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 index c042bca5e8ce..7283fce8f7f3 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 @@ -122,12 +122,13 @@ BANG: '!'; // NOTE: If you add a new token in the list below, you should update the list of keywords // and reserved tag in `docs/sql-ref-ansi-compliance.md#sql-keywords`, and -// modify `ParserUtils.toExprAlias()` which assumes all keywords are between `ADD` and `ZONE`. +// modify `ParserUtils.toExprAlias()` which assumes all keywords are between `ABSENT` and `ZONE`. //============================ // Start of the keywords list //============================ //--SPARK-KEYWORD-LIST-START +ABSENT: 'ABSENT'; ADD: 'ADD'; AFTER: 'AFTER'; AGGREGATE: 'AGGREGATE'; @@ -335,6 +336,7 @@ ITEMS: 'ITEMS'; ITERATE: 'ITERATE'; JOIN: 'JOIN'; JSON: 'JSON'; +JSON_ARRAY: 'JSON_ARRAY'; JSON_TABLE: 'JSON_TABLE'; JSON_VALUE: 'JSON_VALUE'; KEY: 'KEY'; diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index ee7de7eac2bf..27efe6c28860 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -1466,6 +1466,11 @@ primaryExpression (RETURNING returning=dataType)? (emptyBehavior=jsonValueBehavior ON EMPTY)? (errorBehavior=jsonValueBehavior ON ERROR)? RIGHT_PAREN #jsonValue + | JSON_ARRAY LEFT_PAREN + (values+=jsonArrayValue (COMMA values+=jsonArrayValue)*)? + (nullBehavior=jsonConstructorNullBehavior ON NULL)? + (RETURNING returning=dataType)? + RIGHT_PAREN #jsonArray | constant #constantDefault | ASTERISK exceptClause? #star | qualifiedName DOT ASTERISK exceptClause? #star @@ -1500,6 +1505,20 @@ jsonValueBehavior | DEFAULT defaultExpr=expression #jsonValueBehaviorDefault ; +// The behavior selected by JSON_ARRAY/JSON_OBJECT `... ON NULL` clause: NULL keeps nulls, ABSENT +// drops null elements/pairs. +jsonConstructorNullBehavior + : NULL #jsonConstructorNullBehaviorNull + | ABSENT #jsonConstructorNullBehaviorAbsent + ; + +// A JSON_ARRAY element. The optional `FORMAT JSON` clause marks a string argument as already-JSON +// text, so it is spliced into the array verbatim instead of being quoted as a JSON string. A +// lexically-nested JSON constructor (e.g. JSON_ARRAY(JSON_ARRAY(1))) carries this implicitly. +jsonArrayValue + : value=expression (FORMAT JSON)? + ; + semiStructuredExtractionPath : jsonPathFirstPart (jsonPathParts)* ; @@ -2081,7 +2100,8 @@ operatorPipeSetAssignmentSeq // The non-reserved keywords are listed below. Keywords not in this list are reserved keywords. ansiNonReserved //--ANSI-NON-RESERVED-START - : ADD + : ABSENT + | ADD | AFTER | AGGREGATE | ALIGN @@ -2244,6 +2264,7 @@ ansiNonReserved | ITEMS | ITERATE | JSON + | JSON_ARRAY | JSON_TABLE | JSON_VALUE | KEY @@ -2482,7 +2503,8 @@ strictNonReserved nonReserved //--DEFAULT-NON-RESERVED-START - : ADD + : ABSENT + | ADD | AFTER | AGGREGATE | ALIGN @@ -2683,6 +2705,7 @@ nonReserved | ITEMS | ITERATE | JSON + | JSON_ARRAY | JSON_TABLE | JSON_VALUE | KEY diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyDefaultCollation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyDefaultCollation.scala index ff43b3668839..bdde930edb17 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyDefaultCollation.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyDefaultCollation.scala @@ -424,8 +424,11 @@ object ApplyDefaultCollation extends Rule[LogicalPlan] { case cast @ Cast(e: DefaultStringProducingExpression, dt, _, _) if newType == dt => cast.copy(child = e.withNewChildren(e.children.map(inner))) - // Add cast on top of [[DefaultStringProducingExpression]]. - case e: DefaultStringProducingExpression => + // Add cast on top of [[DefaultStringProducingExpression]], unless it already carries an + // explicit (non-default) string collation -- e.g. `JSON_ARRAY(... RETURNING STRING COLLATE + // UTF8_BINARY)` -- which the user chose deliberately and the object/view default must not + // override. + case e: DefaultStringProducingExpression if !hasExplicitStringCollation(e.dataType) => Cast(e.withNewChildren(e.children.map(inner)), newType) case other => @@ -440,6 +443,17 @@ object ApplyDefaultCollation extends Rule[LogicalPlan] { private def hasDefaultStringCharOrVarcharType(dataType: DataType): Boolean = dataType.existsRecursively(isDefaultStringCharOrVarcharType) + /** + * A [[StringType]] carrying an explicit, non-default collation (distinguished from the default + * `StringType` companion by reference identity, matching the convention documented in the + * single-pass resolver's `DefaultCollationTypeCoercion`). Such a type reflects a collation the + * user chose explicitly, so the object/view default collation must not overwrite it. + */ + private def hasExplicitStringCollation(dataType: DataType): Boolean = dataType match { + case st: StringType => !st.eq(StringType) + case _ => false + } + private def replaceColumnTypes( colTypes: Seq[QualifiedColType], collation: String): Seq[QualifiedColType] = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/DefaultCollationTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/DefaultCollationTypeCoercion.scala index bda40f71ecd1..dff4db5e584e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/DefaultCollationTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/DefaultCollationTypeCoercion.scala @@ -45,7 +45,12 @@ object DefaultCollationTypeCoercion { def apply(expression: Expression, collation: String): Expression = { val collatedStringType = StringType(collation) expression match { - case _: DefaultStringProducingExpression if collatedStringType != StringType => + // Skip an expression that already carries an explicit, non-default string collation (e.g. + // `JSON_ARRAY(... RETURNING STRING COLLATE UTF8_BINARY)`): the user chose it deliberately, so + // the object/view default must not overwrite it. `isDefaultStringType` distinguishes the two + // by reference identity. + case e: DefaultStringProducingExpression + if collatedStringType != StringType && !isExplicitlyCollatedStringType(e.dataType) => Cast(child = expression, dataType = collatedStringType) case literal: Literal if hasDefaultStringType(literal.dataType) => literal.copy(dataType = replaceDefaultStringType(literal.dataType, collatedStringType)) @@ -86,6 +91,10 @@ object DefaultCollationTypeCoercion { case _ => false } + /** A [[StringType]] with an explicit, non-default collation (see [[isDefaultStringType]]). */ + private def isExplicitlyCollatedStringType(dataType: DataType): Boolean = + dataType.isInstanceOf[StringType] && !isDefaultStringType(dataType) + /** * When a default collation is specified for a View, * and the cast's dataType contains companion object [[StringType]], diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index c91221690091..ccbbc1c3d884 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.catalyst.expressions +import com.fasterxml.jackson.core.{JsonFactory, JsonProcessingException} + import org.apache.spark.SparkException import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult @@ -31,12 +33,13 @@ import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke} import org.apache.spark.sql.catalyst.json._ import org.apache.spark.sql.catalyst.trees.TreePattern.{GET_JSON_OBJECT, JSON_TO_STRUCT, RUNTIME_REPLACEABLE, TreePattern} -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, GenericArrayData} import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryErrorsBase, QueryExecutionErrors} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.types.StringTypeWithCollation import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String +import org.apache.spark.util.Utils /** * Extracts json object from a json string based on json path specified, and returns json string @@ -860,7 +863,409 @@ object JsonValue { } /** - * Converts an json input string to a [[StructType]], [[ArrayType]] or [[MapType]] + * Behavior of `JSON_ARRAY`'s `ON NULL` clause: what to do with NULL elements in the array. + */ +sealed trait JsonConstructorNullBehavior +object JsonConstructorNullBehavior { + /** Include NULL elements as JSON `null` values. */ + case object Null extends JsonConstructorNullBehavior + /** Omit NULL elements from the array. */ + case object Absent extends JsonConstructorNullBehavior +} + +/** + * Marker for expressions whose result is JSON text and therefore carry an implicit SQL/JSON + * `FORMAT JSON`: when such an expression appears as an argument of a JSON constructor (e.g. + * `JSON_ARRAY`), its value is spliced in verbatim rather than quoted as a JSON string, so + * `JSON_ARRAY(JSON_ARRAY(1))` yields `[[1]]`, not `[["[1]"]]`. Crucially, the constructor freezes + * this decision from the *lexical* argument at parse time (see `AstBuilder.visitJsonArray`) rather + * than re-deriving it from the child expression during evaluation, so a later optimizer rewrite + * (e.g. `CollapseProject` inlining a `JSON_ARRAY` alias into an argument position) cannot change + * whether a value is spliced or quoted. `JSON_OBJECT` / `JSON_QUERY` should extend this as they are + * added. + */ +trait ImplicitlyFormattedAsJson extends Expression + +// scalastyle:off line.size.limit +/** + * The SQL:2016 `JSON_ARRAY` constructor function (feature T811): constructs a JSON array from + * a list of values, with optional `(NULL | ABSENT) ON NULL` control and `RETURNING` type clause. + * + * `NULL ON NULL` (non-default) keeps NULL elements as JSON `null` values. + * `ABSENT ON NULL` (default per the standard) omits NULL elements. + * RETURNING defaults to STRING. + * + * `formatJson(i)` marks element `i` as already-JSON text (SQL/JSON `FORMAT JSON`), so it is spliced + * in verbatim instead of quoted as a string. It is set once, at parse time, from the lexical + * argument -- implicitly for a nested JSON constructor and explicitly for a `... FORMAT JSON` + * clause -- and is a plain field (not a child), so optimizer rewrites that swap the child + * expression (e.g. `CollapseProject`) leave it unchanged. This keeps the output independent of plan + * shape: a value is spliced iff it was written as JSON in the source, never because a rewrite + * happened to substitute a JSON constructor into an argument position. + * + * {{{ + * JSON_ARRAY(1, 'x', true) -- '[1,"x",true]' + * JSON_ARRAY(1, NULL, 3) -- '[1,3]' (ABSENT ON NULL default) + * JSON_ARRAY(1, NULL, 3 NULL ON NULL) -- '[1,null,3]' + * JSON_ARRAY() -- '[]' + * JSON_ARRAY(JSON_ARRAY(1)) -- '[[1]]' (nested constructor: implicit FORMAT JSON) + * JSON_ARRAY('[1,2]') -- '["[1,2]"]' (plain string: quoted) + * JSON_ARRAY('[1,2]' FORMAT JSON) -- '[[1,2]]' (explicit FORMAT JSON: spliced raw) + * }}} + */ +// scalastyle:on line.size.limit +case class JsonArray( + values: Seq[Expression], + formatJson: Seq[Boolean], + needsValidation: Seq[Boolean], + nullBehavior: JsonConstructorNullBehavior, + returning: DataType, + timeZoneId: Option[String] = None) + extends Expression + with TimeZoneAwareExpression + with CodegenFallback + with ExpectsInputTypes + with QueryErrorsBase + with DefaultStringProducingExpression + with ImplicitlyFormattedAsJson { + + // `formatJson(i)` freezes whether element `i` is spliced raw (vs quoted); `needsValidation(i)` + // freezes whether its raw text is arbitrary user input that must be JSON-validated at eval (an + // explicit `FORMAT JSON` on a non-constructor). Both are decided from the lexical argument at + // parse time (see `AstBuilder.visitJsonArray`) and never re-derived from the child expression, + // so an analyzer/optimizer rewrite that wraps a child (e.g. a default-collation `Cast` around a + // trusted nested constructor) cannot flip splicing or spuriously mark a trusted value as needing + // validation. A validated element implies a spliced one. + assert(values.length == formatJson.length && values.length == needsValidation.length, + "JsonArray requires one formatJson and one needsValidation flag per value") + assert(needsValidation.lazyZip(formatJson).forall((nv, fj) => !nv || fj), + "JsonArray needsValidation implies formatJson") + + // True iff some element carries an *explicit* `FORMAT JSON` on a non-constructor: such a value is + // arbitrary user text validated at eval, so it can throw and must not be constant-folded. A + // nested (implicit) constructor produces well-formed JSON by construction and never throws here. + // Read straight off the frozen `needsValidation` flags -- never re-derived from the (possibly + // rewritten) child expressions -- so it drives both `throwable` and the `foldable` exclusion. + private lazy val hasExplicitFormatJson: Boolean = needsValidation.contains(true) + + // Throwable only when the expression can actually throw at eval: when an element carries an + // explicit `FORMAT JSON` (validated, may throw on malformed text) or when a child is itself + // throwable. A plain `JSON_ARRAY(...)` with no explicit `FORMAT JSON` cannot throw (RETURNING is + // restricted to string types at analysis, so the result cast is STRING -> STRING), so leaving it + // non-throwable lets `PushPredicateThroughJoin` push safe filters like `JSON_ARRAY(k) = '[42]'` + // below a join. + override lazy val throwable: Boolean = children.exists(_.throwable) || hasExplicitFormatJson + + // A non-throwing JSON array constructor always yields a value: NULL elements are dropped or + // rendered as JSON `null` (never propagated), the empty argument list yields `[]`, and the + // STRING -> STRING RETURNING cast cannot null a non-null input. For throwable shapes, report + // nullable conservatively so `NullPropagation` does not fold `JSON_ARRAY(...) IS [NOT] NULL` and + // accidentally skip evaluation-time validation or child exceptions. + override def nullable: Boolean = throwable + + // The default RETURNING is a plain STRING, so mix in `DefaultStringProducingExpression` (above) + // to let `ApplyDefaultCollation` cast the result to a non-default object/session collation (e.g. + // `CREATE TABLE ... DEFAULT COLLATION UTF8_LCASE AS SELECT JSON_ARRAY(...)`). The `dataType` + // override below stays authoritative when RETURNING is given explicitly. + + // A constant argument list has no per-row state, so let `ConstantFolding` evaluate the whole + // constructor once instead of serializing JSON row by row. But only fold shapes that cannot throw + // at eval: an explicit `FORMAT JSON` value is validated and may throw on malformed text, and + // `ConstantFolding` evaluates foldables outside conditional branches eagerly -- folding such a + // shape would surface the error at optimization even for rows a later filter/join would drop. A + // nested (implicit FORMAT JSON) value is produced by a constructor and is never malformed, so it + // stays foldable, and its rawness round-trips through `.sql` via an explicit `FORMAT JSON`. + override def foldable: Boolean = children.forall(_.foldable) && !hasExplicitFormatJson + + override def children: Seq[Expression] = values + + override def inputTypes: Seq[AbstractDataType] = values.map(_ => AnyDataType) + + override def dataType: DataType = returning + + override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression = + copy(timeZoneId = Option(timeZoneId)) + + override def checkInputDataTypes(): TypeCheckResult = { + // A constructor emits JSON text, so RETURNING is restricted to string types (VARIANT is a + // deferred extension). CHAR/VARCHAR are normalized to STRING by the parser. + if (!JsonArray.isValidReturningType(returning)) { + DataTypeMismatch( + errorSubClass = "INVALID_JSON_RETURNING_TYPE", + messageParameters = Map( + "functionName" -> toSQLId(prettyName), "returningType" -> toSQLType(returning))) + } else { + // Validate each element up front rather than failing at runtime: a FORMAT JSON element must + // be a string carrying JSON text, and every other element must be serializable to JSON. The + // latter mirrors `to_json`'s analysis-time `JacksonUtils.verifyType` check. + var result: TypeCheckResult = TypeCheckResult.TypeCheckSuccess + var i = 0 + while (i < values.length && result == TypeCheckResult.TypeCheckSuccess) { + val dt = values(i).dataType + if (formatJson(i) && !(dt.isInstanceOf[StringType] || dt == NullType)) { + // A FORMAT JSON element must carry JSON text (string), but an untyped NULL literal is + // allowed: `eval` handles nulls (ABSENT/NULL ON NULL) before it would ever splice, so + // `JSON_ARRAY(NULL FORMAT JSON)` behaves like any other NULL element. + result = DataTypeMismatch( + errorSubClass = "INVALID_JSON_FORMAT_JSON_INPUT", + messageParameters = Map( + "functionName" -> toSQLId(prettyName), + "position" -> (i + 1).toString, + "inputType" -> toSQLType(dt))) + } else { + val elemCheck = JacksonUtils.verifyType(prettyName, dt) + // `verifyType` accepts every `AtomicType`, but `JacksonGenerator` (the writer this shares + // with `to_json`) has no serializer for the spatial atomics and would fail at runtime. + // Reject them here so a passing analysis implies a serializable element. The scan mirrors + // `verifyType`'s traversal (struct fields, array elements, map *values* -- map keys are + // written via `toString`, so a spatial key is fine). + if (elemCheck.isFailure) { + result = elemCheck + } else if (JsonArray.containsUnsupportedJsonType(dt)) { + result = DataTypeMismatch( + errorSubClass = "CANNOT_CONVERT_TO_JSON", + messageParameters = Map( + "name" -> toSQLId(prettyName), + "type" -> toSQLType(dt))) + } + } + i += 1 + } + result + } + } + + // Reuses the mutable `castInput` row and per-element JSON writers, so it holds evaluation state + // and must be fresh-copied before interpreted execution (matches the neighboring JSON + // expressions). + override def stateful: Boolean = true + + // A JSON array is heterogeneous, so each element is serialized with its own data type rather + // than a single shared element type. We build one serializer per child, each configured as a + // single-element `ArrayType(child.dataType)`; serializing `[value]` yields the text `[]`, + // whose outer brackets we strip to recover the element fragment ``. This reuses the same + // Jackson generation path as `to_json`, so numbers, decimals, datetimes, and nested structures + // are rendered correctly. + @transient private lazy val resolvedZoneId: String = + timeZoneId.getOrElse(SQLConf.get.sessionLocalTimeZone) + + @transient private lazy val elementEvaluators: Array[StructsToJsonEvaluator] = + new Array[StructsToJsonEvaluator](values.length) + + @transient private lazy val cachedValidatedFormatJsonTexts: Array[String] = + new Array[String](values.length) + + @transient private lazy val singleElem: Array[Any] = new Array[Any](1) + + // Wraps the reused `singleElem` array by reference (GenericArrayData does not copy), so a single + // instance is shared across elements and rows: `renderElement` mutates `singleElem` in place and + // the serializer reads it synchronously. + @transient private lazy val singleElemArray: GenericArrayData = new GenericArrayData(singleElem) + + @transient private lazy val castInput: GenericInternalRow = new GenericInternalRow(1) + + @transient private lazy val returningCast: Expression = + Cast(BoundReference(0, StringType, nullable = true), returning, timeZoneId, EvalMode.ANSI) + + @transient private lazy val formatJsonFactory: JsonFactory = new JsonFactory() + + // A FORMAT JSON element is spliced into the result verbatim, so it must itself be exactly one + // well-formed JSON value. `checkInputDataTypes` only guarantees the argument is string-typed; a + // string carrying `1,2` or `{bad` would otherwise corrupt the surrounding array into invalid or + // unintended JSON (e.g. `JSON_ARRAY('1,2' FORMAT JSON)` -> `[1,2]`). Validate the runtime value + // before appending: parse one value and require that nothing follows it. + private def validateJsonText(idx: Int, text: String): Unit = { + val valid = + try { + Utils.tryWithResource(formatJsonFactory.createParser(text)) { parser => + if (parser.nextToken() == null) { + false // empty / whitespace-only input carries no JSON value + } else { + // For a scalar this is a no-op; for an array/object it advances to the matching close. + parser.skipChildren() + parser.nextToken() == null // reject anything trailing the first value + } + } + } catch { + case _: JsonProcessingException => false + } + if (!valid) { + throw QueryExecutionErrors.invalidJsonFormatJsonValueError(prettyName, idx + 1, text) + } + } + + // Render a single non-null element to its JSON fragment via its own-typed serializer. + // TODO(SPARK-58730): this serializes each element as a one-element array and strips the brackets, + // so a row with N values does N Jackson flushes plus intermediate string/substring allocations. + // A JSON_ARRAY-specific evaluator that opens the top-level array once and writes each element + // into the shared generator (and codegen for the whole path) would avoid this per-element trip. + private def renderElement(idx: Int, value: Any): String = { + singleElem(0) = value + var evaluator = elementEvaluators(idx) + if (evaluator == null) { + evaluator = StructsToJsonEvaluator( + Map.empty, ArrayType(values(idx).dataType), Some(resolvedZoneId)) + elementEvaluators(idx) = evaluator + } + val arrJson = evaluator + .evaluate(singleElemArray).asInstanceOf[UTF8String].toString + // `arrJson` is "[]" (compact, no spaces); strip the outer brackets. + arrJson.substring(1, arrJson.length - 1) + } + + private def formatJsonText(idx: Int, value: Any): String = { + val text = value.asInstanceOf[UTF8String].toString + if (!needsValidation(idx)) { + text + } else if (values(idx).foldable) { + val cached = cachedValidatedFormatJsonTexts(idx) + if (cached != null) { + cached + } else { + validateJsonText(idx, text) + cachedValidatedFormatJsonTexts(idx) = text + text + } + } else { + validateJsonText(idx, text) + text + } + } + + override def eval(input: InternalRow): Any = { + val sb = new StringBuilder("[") + var first = true + var i = 0 + while (i < values.length) { + val v = values(i).eval(input) + // ABSENT ON NULL drops NULL elements; NULL ON NULL keeps them as JSON `null`. + if (v != null || nullBehavior == JsonConstructorNullBehavior.Null) { + if (!first) sb.append(",") + first = false + if (v == null) { + sb.append("null") + } else if (formatJson(i)) { + // Already-JSON text (a nested JSON constructor or an explicit FORMAT JSON): splice it in + // verbatim. `checkInputDataTypes` guarantees a FORMAT JSON element is string-typed. A + // nested JSON constructor emits well-formed JSON by construction, but an explicit FORMAT + // JSON string is arbitrary user input, so validate it is exactly one well-formed JSON + // value before splicing to avoid corrupting the surrounding array. Whether validation is + // needed is the frozen parse-time `needsValidation(i)`, not a re-derivation from the + // (possibly rewritten) child -- an analyzer cast around a trusted nested constructor must + // not turn into a spurious per-row validation. + sb.append(formatJsonText(i, v)) + } else { + sb.append(renderElement(i, v)) + } + } + i += 1 + } + sb.append("]") + val jsonStr = UTF8String.fromString(sb.toString) + if (returning == StringType) { + jsonStr + } else { + castInput.update(0, jsonStr) + returningCast.eval(castInput) + } + } + + override def prettyName: String = "json_array" + + override def sql: String = { + val valuesSQL = values.zip(formatJson).map { case (v, isJson) => + // Emit SQL that reparses to the same splice/quote decision as the frozen `formatJson` flag. + // The parser splices a value iff it is an explicit `FORMAT JSON` or a lexically-nested JSON + // constructor (see `AstBuilder.visitJsonArray`), so the rendering depends on the flag and the + // (possibly analyzer/optimizer-rewritten) child: + // - spliced + child is not a *bare* JSON constructor: render an explicit `FORMAT JSON` so + // reparse splices it. This covers a plain FORMAT JSON string, an inlined `Cast`, and a + // `Collate`-wrapped nested constructor alike -- crucially without depending on reparse + // re-deriving implicit JSON through the wrapper's rendering (e.g. `Collate.sql` renders + // function-style `collate(child, c)`, which reparse would not recognize as implicit). + // - quoted + child would reparse as implicit JSON (a bare or `Collate`-wrapped constructor + // an optimizer inlined into a quoted position): render through a neutral + // `CAST(... AS STRING)` so reparse keeps it quoted (splicing would flip ["[1]"] to [[1]]). + // - otherwise the child's shape already reproduces the flag, so render it as-is. + // Note: the splice test uses the *direct* child class (only a bare constructor round-trips + // implicitly); the quote test sees through a value-preserving `Collate` (but not a `Cast`, + // since the neutralization above relies on a cast reparsing as not-implicit). + val directlyImplicit = v.isInstanceOf[ImplicitlyFormattedAsJson] + val transitivelyImplicit = JsonArray.isImplicitlyJson(v) + if (isJson && !directlyImplicit) { + s"${v.sql} FORMAT JSON" + } else if (!isJson && transitivelyImplicit) { + s"CAST(${v.sql} AS STRING)" + } else { + v.sql + } + }.mkString(", ") + // Use reference identity, not value equality: an explicit `RETURNING STRING COLLATE ...` + // produces a distinct StringType instance that `==` the default companion `StringType`, so `==` + // would drop it. Only the omitted default (the companion, by reference) should render nothing. + val returningSQL = if (returning.eq(StringType)) "" else s" RETURNING ${returning.sql}" + val nullSQL = nullBehavior match { + case JsonConstructorNullBehavior.Null => " NULL ON NULL" + case JsonConstructorNullBehavior.Absent => "" + } + s"JSON_ARRAY($valuesSQL$nullSQL$returningSQL)" + } + + override protected def withNewChildrenInternal( + newChildren: IndexedSeq[Expression]): JsonArray = + copy(values = newChildren) +} + +object JsonArray { + /** + * Whether `e` is a lexically-nested JSON constructor -- i.e. carries an implicit SQL/JSON + * `FORMAT JSON` and should be spliced raw rather than quoted. Sees through a value-preserving + * `Collate` (a postfix `... COLLATE c` only annotates the collation of the JSON text, e.g. + * `JSON_ARRAY(JSON_ARRAY(1) COLLATE UTF8_LCASE)`), but deliberately NOT through a `Cast`: + * `AstBuilder`/`JsonArray.sql` use a `CAST(... AS STRING)` wrapper specifically to detach a value + * from implicit `FORMAT JSON`, so unwrapping it would defeat that neutralization. + */ + @scala.annotation.tailrec + def isImplicitlyJson(e: Expression): Boolean = e match { + case c: Collate => isImplicitlyJson(c.child) + case _: ImplicitlyFormattedAsJson => true + case _ => false + } + + /** + * `JSON_ARRAY` returns JSON text, so RETURNING is restricted to a plain STRING (VARIANT is + * deferred). `CharType` / `VarcharType` extend `StringType` but carry a length that `JSON_ARRAY` + * does not enforce -- it serializes the fragment itself without a length-checking cast -- so they + * are rejected: the parser normalizes a SQL `CHAR`/`VARCHAR` RETURNING to STRING before + * construction, and this guards a raw `CharType`/`VarcharType` supplied by direct Catalyst + * construction. + */ + def isValidReturningType(dt: DataType): Boolean = dt match { + case _: CharType | _: VarcharType => false + case _: StringType => true + case _ => false + } + + /** + * Whether `dt` contains a spatial atomic (`GEOMETRY` / `GEOGRAPHY`) in a position that + * `JacksonGenerator` would have to serialize. These are `AtomicType`s that + * `JacksonUtils.verifyType` accepts but the JSON writer has no serializer for, so they must be + * rejected at analysis. The traversal mirrors `verifyType`: it descends into struct fields, array + * elements, and map *values* (map keys are written via `toString`, so a spatial key is fine) and + * unwraps UDTs. + */ + def containsUnsupportedJsonType(dt: DataType): Boolean = dt match { + case _: GeometryType | _: GeographyType => true + case st: StructType => st.exists(f => containsUnsupportedJsonType(f.dataType)) + case at: ArrayType => containsUnsupportedJsonType(at.elementType) + case mt: MapType => containsUnsupportedJsonType(mt.valueType) + case udt: UserDefinedType[_] => containsUnsupportedJsonType(udt.sqlType) + case _ => false + } +} + +/** + * Converts a JSON input string to a [[StructType]], [[ArrayType]] or [[MapType]] * with the specified schema. */ // scalastyle:off line.size.limit diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 74c691699609..a97e6e101d3c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -4200,6 +4200,60 @@ class AstBuilder extends DataTypeAstBuilder JsonValue(jsonExpr, path, returning, onEmpty, onError, emptyDefault, errorDefault) } + /** + * Resolve a `jsonConstructorNullBehavior` clause (`NULL` / `ABSENT`) into a + * [[JsonConstructorNullBehavior]]. + */ + private def buildJsonConstructorNullBehavior( + ctx: JsonConstructorNullBehaviorContext): JsonConstructorNullBehavior = + ctx match { + case _: JsonConstructorNullBehaviorNullContext => + JsonConstructorNullBehavior.Null + case _: JsonConstructorNullBehaviorAbsentContext => + JsonConstructorNullBehavior.Absent + } + + /** + * Create a [[JsonArray]] expression for the SQL:2016 `JSON_ARRAY` constructor function. + * The `ON NULL` clause defaults to `ABSENT ON NULL` (drops NULL elements), and RETURNING + * defaults to STRING. + */ + override def visitJsonArray(ctx: JsonArrayContext): Expression = withOrigin(ctx) { + val arrayValues = ctx.values.asScala.map(v => expression(v.value)).toSeq + // Freeze the FORMAT JSON decisions here, from the lexical argument, so a later + // analyzer/optimizer rewrite that wraps or swaps the child cannot change them (see + // [[ImplicitlyFormattedAsJson]]). For each element: + // - `formatJson`: whether it is already-JSON text spliced raw. True when it carries an + // explicit `FORMAT JSON` clause, or is a (lexically) nested JSON constructor -- seen through + // a value-preserving `COLLATE` via `JsonArray.isImplicitlyJson`. + // - `needsValidation`: whether its raw text is arbitrary user input to JSON-validate at eval. + // True only for an explicit `FORMAT JSON` on something that is NOT a JSON constructor; a + // nested constructor emits well-formed JSON by construction and is trusted. + val formatArgs = ctx.values.asScala.zip(arrayValues).map { case (v, expr) => + val explicit = v.FORMAT() != null + val implicitlyJson = JsonArray.isImplicitlyJson(expr) + (explicit || implicitlyJson, explicit && !implicitlyJson) + }.toSeq + val formatJson = formatArgs.map(_._1) + val needsValidation = formatArgs.map(_._2) + // Default RETURNING type is STRING; the result is JSON text. A CHAR/VARCHAR RETURNING is + // normalized to STRING unconditionally: JSON_ARRAY serializes the fragment itself and never + // advertises a CHAR/VARCHAR length it does not enforce. The CharVarcharUtils helpers cannot be + // used here -- they honor spark.sql.preserveCharVarcharTypeInfo and would leave a VARCHAR(n) + // length in the output type when that flag is set. A non-string RETURNING is left intact for + // checkInputDataTypes to fail. + val returning = Option(ctx.returning).map(typedVisit[DataType]).map { + case c: CharType => c.toStringType + case v: VarcharType => v.toStringType + case other => other + }.getOrElse(StringType) + // Default NULL ON NULL behavior is ABSENT (drop NULL elements). + val nullBehavior = Option(ctx.nullBehavior) + .map(buildJsonConstructorNullBehavior) + .getOrElse(JsonConstructorNullBehavior.Absent) + JsonArray(arrayValues, formatJson, needsValidation, nullBehavior, returning) + } + /** * Create a (windowed) Function expression. */ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala index 174855cd69d0..3ae7f09e1e83 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala @@ -124,7 +124,7 @@ object ParserUtils extends SparkParserUtils { case term: TerminalNodeImpl => val termText = term.getText val tt = term.getSymbol.getType - val current = if ((SqlBaseParser.ADD <= tt && tt <= SqlBaseParser.ZONE) || + val current = if ((SqlBaseParser.ABSENT <= tt && tt <= SqlBaseParser.ZONE) || (SqlBaseParser.BIGINT_LITERAL <= tt && tt <= SqlBaseParser.BIGDECIMAL_LITERAL)) { termText.toUpperCase(Locale.ROOT) } else { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 646d296c11c6..9653d0995896 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -1387,6 +1387,25 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE messageParameters = Map.empty) } + def invalidJsonFormatJsonValueError( + functionName: String, position: Int, value: String): SparkRuntimeException = { + // Bound the echoed value: a FORMAT JSON argument can be a large payload column, and inlining it + // whole would bloat task-failure messages, logs, and UI/event metadata. Show a capped preview + // plus the full length instead. + val maxPreviewLen = 100 + val preview = if (value.length > maxPreviewLen) { + s"${value.take(maxPreviewLen)}... (${value.length} characters)" + } else { + value + } + new SparkRuntimeException( + errorClass = "INVALID_JSON_FORMAT_JSON_VALUE", + messageParameters = Map( + "functionName" -> toSQLId(functionName), + "position" -> position.toString, + "value" -> toSQLValue(preview, StringType))) + } + def paramExceedOneCharError(paramName: String, actualValue: String): SparkRuntimeException = { new SparkRuntimeException( errorClass = "OPTION_VALUE_EXCEEDS_ONE_CHARACTER", diff --git a/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectDatabaseMetaDataSuite.scala b/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectDatabaseMetaDataSuite.scala index b07f9da119a4..ba027621d2f4 100644 --- a/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectDatabaseMetaDataSuite.scala +++ b/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectDatabaseMetaDataSuite.scala @@ -210,7 +210,7 @@ class SparkConnectDatabaseMetaDataSuite extends ConnectFunSuite with RemoteSpark val metadata = conn.getMetaData // scalastyle:off line.size.limit // CURRENT_PATH and SYSTEM are excluded: getSQLKeywords drops SQL:2003 reserved words (see companion). - assert(metadata.getSQLKeywords === "ADD,AFTER,AGGREGATE,ALIGN,ALWAYS,ANALYZE,ANTI,ANY_VALUE,APPLY,APPROX,ARCHIVE,ASC,ASOF,AUTO,BERNOULLI,BIN,BINDING,BIN_DISTRIBUTE_RATIO,BIN_END,BIN_START,BUCKET,BUCKETS,BYTE,CACHE,CASCADE,CATALOG,CATALOGS,CDC,CHANGE,CHANGES,CLEAR,CLUSTER,CLUSTERED,CODEGEN,COLLATION,COLLATIONS,COLLECTION,COLUMNS,COMMENT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONTAINS,CONTINUE,COST,CURRENT_DATABASE,CURRENT_SCHEMA,DATA,DATABASE,DATABASES,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAYOFYEAR,DAYS,DBPROPERTIES,DEFAULT_PATH,DEFINED,DEFINER,DELAY,DELIMITED,DESC,DFS,DIRECTORIES,DIRECTORY,DISTANCE,DISTRIBUTE,DIV,DO,ELSEIF,EMPTY,ENFORCED,ERROR,ESCAPED,EVOLUTION,EXACT,EXCHANGE,EXCLUDE,EXCLUSIVE,EXIT,EXPLAIN,EXPORT,EXTEND,EXTENDED,FIELDS,FILEFORMAT,FIRST,FLOW,FOLLOWING,FORMAT,FORMATTED,FOUND,FUNCTIONS,GENERATED,GEOGRAPHY,GEOMETRY,HANDLER,HISTORY,HOURS,IDENTIFIED,IDENTIFIER,IF,IGNORE,ILIKE,IMMEDIATE,INCLUDE,INCLUSIVE,INCREMENT,INDEX,INDEXES,INPATH,INPUT,INPUTFORMAT,INVOKER,ITEMS,ITERATE,JSON,JSON_TABLE,JSON_VALUE,KEY,KEYS,LAST,LAZY,LEAVE,LEVEL,LIMIT,LINES,LIST,LOAD,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MATCH_CONDITION,MATERIALIZED,MEASURE,METRICS,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTES,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NEAREST,NORELY,NULLS,OFFSET,OPTION,OPTIONS,ORDINALITY,OUTPUTFORMAT,OVERWRITE,PARTITIONED,PARTITIONS,PATH,PERCENT,PIVOT,PLACING,PRECEDING,PRINCIPALS,PROCEDURES,PROPERTIES,PURGE,QUALIFY,QUARTER,QUERY,RECORDREADER,RECORDWRITER,RECOVER,RECURSION,REDUCE,REFRESH,RELY,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURNING,ROLE,ROLES,SCD,SCHEMA,SCHEMAS,SECONDS,SECURITY,SEMI,SEPARATED,SEQUENCE,SERDE,SERDEPROPERTIES,SETS,SHORT,SHOW,SIMILARITY,SINGLE,SKEWED,SORT,SORTED,SOURCE,STATISTICS,STORED,STRATIFY,STREAM,STREAMING,STRING,STRUCT,SUBSTR,SYNC,SYSTEM_PATH,SYSTEM_TIME,SYSTEM_VERSION,TABLES,TARGET,TBLPROPERTIES,TERMINATED,TIMEDIFF,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TOUCH,TRACK,TRANSACTION,TRANSACTIONS,TRANSFORM,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNIFORM,UNLOCK,UNPIVOT,UNSET,UNTIL,USE,VAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WATERMARK,WEEK,WEEKS,WHILE,WIDTH,X,YEARS,ZONE") + assert(metadata.getSQLKeywords === "ABSENT,ADD,AFTER,AGGREGATE,ALIGN,ALWAYS,ANALYZE,ANTI,ANY_VALUE,APPLY,APPROX,ARCHIVE,ASC,ASOF,AUTO,BERNOULLI,BIN,BINDING,BIN_DISTRIBUTE_RATIO,BIN_END,BIN_START,BUCKET,BUCKETS,BYTE,CACHE,CASCADE,CATALOG,CATALOGS,CDC,CHANGE,CHANGES,CLEAR,CLUSTER,CLUSTERED,CODEGEN,COLLATION,COLLATIONS,COLLECTION,COLUMNS,COMMENT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONTAINS,CONTINUE,COST,CURRENT_DATABASE,CURRENT_SCHEMA,DATA,DATABASE,DATABASES,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAYOFYEAR,DAYS,DBPROPERTIES,DEFAULT_PATH,DEFINED,DEFINER,DELAY,DELIMITED,DESC,DFS,DIRECTORIES,DIRECTORY,DISTANCE,DISTRIBUTE,DIV,DO,ELSEIF,EMPTY,ENFORCED,ERROR,ESCAPED,EVOLUTION,EXACT,EXCHANGE,EXCLUDE,EXCLUSIVE,EXIT,EXPLAIN,EXPORT,EXTEND,EXTENDED,FIELDS,FILEFORMAT,FIRST,FLOW,FOLLOWING,FORMAT,FORMATTED,FOUND,FUNCTIONS,GENERATED,GEOGRAPHY,GEOMETRY,HANDLER,HISTORY,HOURS,IDENTIFIED,IDENTIFIER,IF,IGNORE,ILIKE,IMMEDIATE,INCLUDE,INCLUSIVE,INCREMENT,INDEX,INDEXES,INPATH,INPUT,INPUTFORMAT,INVOKER,ITEMS,ITERATE,JSON,JSON_ARRAY,JSON_TABLE,JSON_VALUE,KEY,KEYS,LAST,LAZY,LEAVE,LEVEL,LIMIT,LINES,LIST,LOAD,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MATCH_CONDITION,MATERIALIZED,MEASURE,METRICS,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTES,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NEAREST,NORELY,NULLS,OFFSET,OPTION,OPTIONS,ORDINALITY,OUTPUTFORMAT,OVERWRITE,PARTITIONED,PARTITIONS,PATH,PERCENT,PIVOT,PLACING,PRECEDING,PRINCIPALS,PROCEDURES,PROPERTIES,PURGE,QUALIFY,QUARTER,QUERY,RECORDREADER,RECORDWRITER,RECOVER,RECURSION,REDUCE,REFRESH,RELY,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURNING,ROLE,ROLES,SCD,SCHEMA,SCHEMAS,SECONDS,SECURITY,SEMI,SEPARATED,SEQUENCE,SERDE,SERDEPROPERTIES,SETS,SHORT,SHOW,SIMILARITY,SINGLE,SKEWED,SORT,SORTED,SOURCE,STATISTICS,STORED,STRATIFY,STREAM,STREAMING,STRING,STRUCT,SUBSTR,SYNC,SYSTEM_PATH,SYSTEM_TIME,SYSTEM_VERSION,TABLES,TARGET,TBLPROPERTIES,TERMINATED,TIMEDIFF,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TOUCH,TRACK,TRANSACTION,TRANSACTIONS,TRANSFORM,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNIFORM,UNLOCK,UNPIVOT,UNSET,UNTIL,USE,VAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WATERMARK,WEEK,WEEKS,WHILE,WIDTH,X,YEARS,ZONE") // scalastyle:on line.size.limit } } diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/json-functions.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/json-functions.sql.out index 9b33afd10db8..62407ef8b9ce 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/json-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/json-functions.sql.out @@ -1358,3 +1358,116 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException "fragment" : "json_value('{}', '$.x' RETURNING INT DEFAULT array(1) ON EMPTY)" } ] } + + +-- !query +select json_array(1, 'x', true) +-- !query analysis +Project [json_array(1, x, true, false, false, false, false, false, false, Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY(1, x, true)#x] ++- OneRowRelation + + +-- !query +select json_array(1, NULL, 3) +-- !query analysis +Project [json_array(1, null, 3, false, false, false, false, false, false, Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY(1, NULL, 3)#x] ++- OneRowRelation + + +-- !query +select json_array(1, NULL, 3 NULL ON NULL) +-- !query analysis +Project [json_array(1, null, 3, false, false, false, false, false, false, Null, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY(1, NULL, 3 NULL ON NULL)#x] ++- OneRowRelation + + +-- !query +select json_array() +-- !query analysis +Project [json_array(Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY()#x] ++- OneRowRelation + + +-- !query +select json_array(CAST(1.50 AS DECIMAL(5,2)), DATE'2020-01-02') +-- !query analysis +[Analyzer test output redacted due to nondeterminism] + + +-- !query +select json_array(json_array(1, 2), 3) +-- !query analysis +Project [json_array(json_array(1, 2, false, false, false, false, Absent, StringType, Some(America/Los_Angeles)), 3, true, false, false, false, Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY(JSON_ARRAY(1, 2), 3)#x] ++- OneRowRelation + + +-- !query +select json_array('[1,2]') +-- !query analysis +Project [json_array([1,2], false, false, Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY([1,2])#x] ++- OneRowRelation + + +-- !query +select json_array('[1,2]' FORMAT JSON) +-- !query analysis +Project [json_array([1,2], true, true, Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY([1,2] FORMAT JSON)#x] ++- OneRowRelation + + +-- !query +select json_array('{"a":1}' FORMAT JSON, 'x') +-- !query analysis +Project [json_array({"a":1}, x, true, false, true, false, Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY({"a":1} FORMAT JSON, x)#x] ++- OneRowRelation + + +-- !query +select json_array(123 FORMAT JSON) +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.INVALID_JSON_FORMAT_JSON_INPUT", + "sqlState" : "42K09", + "messageParameters" : { + "functionName" : "`json_array`", + "inputType" : "\"INT\"", + "position" : "1", + "sqlExpr" : "\"JSON_ARRAY(123 FORMAT JSON)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 34, + "fragment" : "json_array(123 FORMAT JSON)" + } ] +} + + +-- !query +select json_array('1,2' FORMAT JSON) +-- !query analysis +Project [json_array(1,2, true, true, Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY(1,2 FORMAT JSON)#x] ++- OneRowRelation + + +-- !query +select json_array(NULL FORMAT JSON) +-- !query analysis +Project [json_array(null, true, true, Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY(NULL FORMAT JSON)#x] ++- OneRowRelation + + +-- !query +select json_array(NULL FORMAT JSON NULL ON NULL) +-- !query analysis +Project [json_array(null, true, true, Null, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY(NULL FORMAT JSON NULL ON NULL)#x] ++- OneRowRelation + + +-- !query +select json_array(1 IS NULL, 2 > 1) +-- !query analysis +Project [json_array(isnull(1), (2 > 1), false, false, false, false, Absent, StringType, Some(America/Los_Angeles)) AS JSON_ARRAY((1 IS NULL), (2 > 1))#x] ++- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql index fa1cb6628ec9..3d4678cb8f8d 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql @@ -211,3 +211,29 @@ select json_value('{"a":[1,2]}', '$.a[*]'); select json_value('{"a":1}', '$.a' RETURNING STRUCT); -- invalid: a DEFAULT that cannot cast to the RETURNING type select json_value('{}', '$.x' RETURNING INT DEFAULT array(1) ON EMPTY); + +-- JSON_ARRAY: construct a JSON array from an argument list +select json_array(1, 'x', true); +-- default ABSENT ON NULL drops NULL elements +select json_array(1, NULL, 3); +-- NULL ON NULL keeps NULL elements +select json_array(1, NULL, 3 NULL ON NULL); +-- empty array +select json_array(); +-- decimals and dates render via the JSON generator +select json_array(CAST(1.50 AS DECIMAL(5,2)), DATE'2020-01-02'); +-- nested JSON_ARRAY is spliced raw, not re-quoted (implicit FORMAT JSON) +select json_array(json_array(1, 2), 3); +-- a plain string is quoted; FORMAT JSON splices already-JSON text verbatim +select json_array('[1,2]'); +select json_array('[1,2]' FORMAT JSON); +select json_array('{"a":1}' FORMAT JSON, 'x'); +-- FORMAT JSON requires a string argument +select json_array(123 FORMAT JSON); +-- FORMAT JSON requires the string to be exactly one well-formed JSON value +select json_array('1,2' FORMAT JSON); +-- an untyped NULL under FORMAT JSON is allowed and follows ON NULL handling +select json_array(NULL FORMAT JSON); +select json_array(NULL FORMAT JSON NULL ON NULL); +-- values accept unparenthesized predicate expressions +select json_array(1 IS NULL, 2 > 1); diff --git a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out index 6656a7723cd1..8284160ffb40 100644 --- a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out @@ -1546,3 +1546,141 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException "fragment" : "json_value('{}', '$.x' RETURNING INT DEFAULT array(1) ON EMPTY)" } ] } + + +-- !query +select json_array(1, 'x', true) +-- !query schema +struct +-- !query output +[1,"x",true] + + +-- !query +select json_array(1, NULL, 3) +-- !query schema +struct +-- !query output +[1,3] + + +-- !query +select json_array(1, NULL, 3 NULL ON NULL) +-- !query schema +struct +-- !query output +[1,null,3] + + +-- !query +select json_array() +-- !query schema +struct +-- !query output +[] + + +-- !query +select json_array(CAST(1.50 AS DECIMAL(5,2)), DATE'2020-01-02') +-- !query schema +struct +-- !query output +[1.50,"2020-01-02"] + + +-- !query +select json_array(json_array(1, 2), 3) +-- !query schema +struct +-- !query output +[[1,2],3] + + +-- !query +select json_array('[1,2]') +-- !query schema +struct +-- !query output +["[1,2]"] + + +-- !query +select json_array('[1,2]' FORMAT JSON) +-- !query schema +struct +-- !query output +[[1,2]] + + +-- !query +select json_array('{"a":1}' FORMAT JSON, 'x') +-- !query schema +struct +-- !query output +[{"a":1},"x"] + + +-- !query +select json_array(123 FORMAT JSON) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.INVALID_JSON_FORMAT_JSON_INPUT", + "sqlState" : "42K09", + "messageParameters" : { + "functionName" : "`json_array`", + "inputType" : "\"INT\"", + "position" : "1", + "sqlExpr" : "\"JSON_ARRAY(123 FORMAT JSON)\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 34, + "fragment" : "json_array(123 FORMAT JSON)" + } ] +} + + +-- !query +select json_array('1,2' FORMAT JSON) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkRuntimeException +{ + "errorClass" : "INVALID_JSON_FORMAT_JSON_VALUE", + "sqlState" : "22032", + "messageParameters" : { + "functionName" : "`json_array`", + "position" : "1", + "value" : "'1,2'" + } +} + + +-- !query +select json_array(NULL FORMAT JSON) +-- !query schema +struct +-- !query output +[] + + +-- !query +select json_array(NULL FORMAT JSON NULL ON NULL) +-- !query schema +struct +-- !query output +[null] + + +-- !query +select json_array(1 IS NULL, 2 > 1) +-- !query schema +struct 1)):string> +-- !query output +[false,true] diff --git a/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out b/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out index 37f174acfe53..8ab0873e5b71 100644 --- a/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out @@ -4,6 +4,7 @@ SELECT * from SQL_KEYWORDS() -- !query schema struct -- !query output +ABSENT false ADD false AFTER false AGGREGATE false @@ -212,6 +213,7 @@ ITEMS false ITERATE false JOIN true JSON false +JSON_ARRAY false JSON_TABLE false JSON_VALUE false KEY false diff --git a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out index f6b910a54c3a..9fa0ad919c13 100644 --- a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out @@ -4,6 +4,7 @@ SELECT * from SQL_KEYWORDS() -- !query schema struct -- !query output +ABSENT false ADD false AFTER false AGGREGATE false @@ -212,6 +213,7 @@ ITEMS false ITERATE false JOIN false JSON false +JSON_ARRAY false JSON_TABLE false JSON_VALUE false KEY false diff --git a/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out index f6b910a54c3a..9fa0ad919c13 100644 --- a/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out @@ -4,6 +4,7 @@ SELECT * from SQL_KEYWORDS() -- !query schema struct -- !query output +ABSENT false ADD false AFTER false AGGREGATE false @@ -212,6 +213,7 @@ ITEMS false ITERATE false JOIN false JSON false +JSON_ARRAY false JSON_TABLE false JSON_VALUE false KEY false diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonArraySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonArraySuite.scala new file mode 100644 index 000000000000..e55be15b65dc --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonArraySuite.scala @@ -0,0 +1,579 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkRuntimeException +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch +import org.apache.spark.sql.catalyst.expressions.{Cast, Collate, JsonArray, JsonConstructorNullBehavior, Literal, ResolvedCollation} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.{CharType, GeographyType, GeometryType, IntegerType, MapType, StringType, VarcharType} + +/** + * Test suite for the `JSON_ARRAY` ANSI SQL:2016 constructor function. + */ +class JsonArraySuite extends QueryTest with SharedSparkSession { + + import testImplicits._ + + test("JSON_ARRAY with simple scalar values") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, 'x', true)"), + Row("""[1,"x",true]""")) + } + + test("JSON_ARRAY with NULL elements - ABSENT ON NULL (default)") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3)"), + Row("[1,3]")) + } + + test("JSON_ARRAY with NULL elements - NULL ON NULL") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3 NULL ON NULL)"), + Row("[1,null,3]")) + } + + test("JSON_ARRAY with NULL elements - explicit ABSENT ON NULL") { + // Exercise the explicit `ABSENT ON NULL` grammar branch (the default is implicit absent, so + // this spelling is otherwise untested); it drops NULL elements just like the default. + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3 ABSENT ON NULL)"), + Row("[1,3]")) + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3 ABSENT ON NULL RETURNING STRING)"), + Row("[1,3]")) + } + + test("JSON_ARRAY with empty list") { + checkAnswer( + sql("SELECT JSON_ARRAY()"), + Row("[]")) + } + + test("JSON_ARRAY with floating point numbers") { + checkAnswer( + sql("SELECT JSON_ARRAY(1.5, 2.7)"), + Row("[1.5,2.7]")) + } + + test("JSON_ARRAY with mixed types") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, 'text', 3.14, true, false)"), + Row("""[1,"text",3.14,true,false]""")) + } + + test("JSON_ARRAY with all NULLs and ABSENT ON NULL") { + checkAnswer( + sql("SELECT JSON_ARRAY(NULL, NULL)"), + Row("[]")) + } + + test("JSON_ARRAY with RETURNING STRING (explicit)") { + checkAnswer( + sql("SELECT JSON_ARRAY(1, 2, 3 RETURNING STRING)"), + Row("[1,2,3]")) + } + + test("JSON_ARRAY with both NULL ON NULL and RETURNING clauses") { + // The grammar allows `... ON NULL` and `RETURNING` together, in that order; exercise both. + checkAnswer( + sql("SELECT JSON_ARRAY(1, NULL, 3 NULL ON NULL RETURNING STRING)"), + Row("[1,null,3]")) + } + + test("JSON_ARRAY over non-foldable columns exercises row-wise eval") { + val df = Seq((1, "a", true), (2, "b", false)).toDF("i", "s", "b") + checkAnswer( + df.selectExpr("JSON_ARRAY(i, s, b)"), + Seq(Row("""[1,"a",true]"""), Row("""[2,"b",false]"""))) + } + + test("JSON_ARRAY renders decimals and dates via Jackson, not toString") { + checkAnswer( + sql("SELECT JSON_ARRAY(CAST(1.50 AS DECIMAL(5,2)), DATE'2020-01-02')"), + Row("""[1.50,"2020-01-02"]""")) + } + + test("JSON_ARRAY renders a TIMESTAMP via to_json's writer in the session time zone") { + // The constructor is TimeZoneAware and shares to_json's writer, so a TIMESTAMP element must + // render identically to to_json of the singleton array, formatted in the session time zone. + // Assert agreement with that writer (rather than pinning a fragile format string), and that the + // rendering tracks the session time zone by differing between two zones. + def render(tz: String): String = withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz) { + val out = + sql("SELECT JSON_ARRAY(TIMESTAMP'2020-01-02 03:04:05')").collect().head.getString(0) + val expected = + sql("SELECT to_json(array(TIMESTAMP'2020-01-02 03:04:05'))").collect().head.getString(0) + assert(out == expected, s"for tz=$tz") + out + } + assert(render("UTC") != render("America/Los_Angeles")) + } + + test("JSON_ARRAY renders array and map elements as JSON structures, like to_json") { + // The docs state array/map/struct arguments render via the same writer as to_json (as nested + // JSON structures, not quoted strings). Cover arrays and maps explicitly (structs are covered + // by the ignoreNullFields test); a nested array element serializes to [1,2], a map to {"k":1}. + checkAnswer( + sql("SELECT JSON_ARRAY(array(1, 2), map('k', 1))"), + Row("""[[1,2],{"k":1}]""")) + checkAnswer( + sql("SELECT JSON_ARRAY(array(array(1), array(2, 3)))"), + Row("[[[1],[2,3]]]")) + } + + test("JSON_ARRAY strings are escaped") { + checkAnswer( + sql("""SELECT JSON_ARRAY('a"b', 'c\td')"""), + Row("""["a\"b","c\td"]""")) + } + + test("nested JSON_ARRAY is spliced raw, not re-quoted (implicit FORMAT JSON)") { + checkAnswer( + sql("SELECT JSON_ARRAY(JSON_ARRAY(1, 2), 3)"), + Row("[[1,2],3]")) + checkAnswer( + sql("SELECT JSON_ARRAY(JSON_ARRAY(1))"), + Row("[[1]]")) + } + + test("explicit FORMAT JSON splices a string verbatim; a plain string is quoted") { + // A plain string element is quoted and escaped like any other string value... + checkAnswer(sql("""SELECT JSON_ARRAY('[1,2]')"""), Row("""["[1,2]"]""")) + // ...while FORMAT JSON marks it as already-JSON text, spliced in verbatim. + checkAnswer(sql("""SELECT JSON_ARRAY('[1,2]' FORMAT JSON)"""), Row("[[1,2]]")) + checkAnswer( + sql("""SELECT JSON_ARRAY('{"a":1}' FORMAT JSON, 'x')"""), + Row("""[{"a":1},"x"]""")) + } + + test("splicing is decided from the source, not the optimized plan shape") { + // A JSON_ARRAY result surfaced as a column is a plain STRING and must be quoted -- even though + // CollapseProject may inline the inner JSON_ARRAY into the outer argument position. The FORMAT + // JSON decision is frozen from the lexical argument at parse time, so it does not depend on + // whether that inlining happens: the result is ["[1]"], never [[1]]. + val inlined = sql("SELECT JSON_ARRAY(a) AS r FROM (SELECT JSON_ARRAY(1) AS a) t") + checkAnswer(inlined, Row("""["[1]"]""")) + // Referencing the alias twice blocks CollapseProject from inlining it; the result is identical, + // confirming independence from plan shape. + val notInlined = + sql("SELECT JSON_ARRAY(a) AS r, a FROM (SELECT JSON_ARRAY(1) AS a) t") + checkAnswer(notInlined, Row("""["[1]"]""", "[1]")) + } + + test("JSON_ARRAY column with NULL under both ON NULL modes") { + val df = Seq(Some(1), None).toDF("i") + checkAnswer( + df.selectExpr("JSON_ARRAY(i)"), + Seq(Row("[1]"), Row("[]"))) + checkAnswer( + df.selectExpr("JSON_ARRAY(i NULL ON NULL)"), + Seq(Row("[1]"), Row("[null]"))) + } + + test("nested JSON_ARRAY with a collated STRING RETURNING is still spliced raw") { + // The inner array carries implicit FORMAT JSON regardless of its (collated) result collation, + // so it is spliced raw as [[1],2], not re-quoted as ["[1]",2]. + checkAnswer( + sql("SELECT JSON_ARRAY(JSON_ARRAY(1 RETURNING STRING COLLATE UTF8_LCASE), 2)"), + Row("[[1],2]")) + } + + test("a nested constructor wrapped in a postfix COLLATE is still spliced raw") { + // `... COLLATE c` wraps the nested constructor in a value-preserving Collate. The implicit + // FORMAT JSON must be seen through that wrapper, so the inner array is spliced ([[1]]), not + // treated as a plain string and quoted (["[1]"]). + checkAnswer( + sql("SELECT JSON_ARRAY(JSON_ARRAY(1) COLLATE UTF8_LCASE)"), + Row("[[1]]")) + checkAnswer( + sql("SELECT JSON_ARRAY(JSON_ARRAY(1, 2) COLLATE UTF8_LCASE, 3)"), + Row("[[1,2],3]")) + } + + test("FORMAT JSON on a non-string argument is rejected at analysis") { + val e = intercept[AnalysisException] { + sql("SELECT JSON_ARRAY(123 FORMAT JSON)").collect() + } + assert(e.getCondition == "DATATYPE_MISMATCH.INVALID_JSON_FORMAT_JSON_INPUT") + } + + test("explicit FORMAT JSON with valid but whitespaced JSON is spliced verbatim") { + // Validation only checks well-formedness; the original text (including insignificant + // whitespace) is spliced as-is, not re-serialized. + checkAnswer(sql("""SELECT JSON_ARRAY('[1, 2]' FORMAT JSON)"""), Row("[[1, 2]]")) + checkAnswer(sql("""SELECT JSON_ARRAY(' true ' FORMAT JSON)"""), Row("[ true ]")) + } + + test("explicit FORMAT JSON with a malformed value is rejected at runtime") { + // A single string-typed argument passes analysis, but a value that is not exactly one + // well-formed JSON value would corrupt the surrounding array, so it fails at eval. + Seq( + "'1,2'", // two values, not one -- would splice as [1,2] + "'{\"a\":1'", // truncated object + "'[1,'", // truncated array + "'not json'", // bare word + "''").foreach { arg => // empty string carries no JSON value + val e = intercept[SparkRuntimeException] { + sql(s"SELECT JSON_ARRAY($arg FORMAT JSON)").collect() + } + assert(e.getCondition == "INVALID_JSON_FORMAT_JSON_VALUE", s"for argument $arg") + } + } + + test("malformed FORMAT JSON error truncates a long value to a bounded preview") { + // A large malformed payload must not be inlined whole into the error message. The preview is + // capped (100 chars) and the full length is reported instead. + val long = "z" * 500 // not valid JSON (bare word) and longer than the preview cap + val e = intercept[SparkRuntimeException] { + sql(s"SELECT JSON_ARRAY('$long' FORMAT JSON)").collect() + } + assert(e.getCondition == "INVALID_JSON_FORMAT_JSON_VALUE") + val msg = e.getMessage + assert(msg.contains("(500 characters)"), msg) + assert(!msg.contains("z" * 101), "the full value must not be inlined; preview is capped") + } + + test("explicit FORMAT JSON validates per-row over non-foldable columns") { + val df = Seq("[1,2]", "1,2").toDF("s") + val e = intercept[SparkRuntimeException] { + df.selectExpr("JSON_ARRAY(s FORMAT JSON)").collect() + } + assert(e.getCondition == "INVALID_JSON_FORMAT_JSON_VALUE") + } + + test("explicit FORMAT JSON over a nullable column follows ON NULL, validating only non-nulls") { + // A nullable string column: NULL rows must be handled by ON NULL (dropped / kept as JSON null) + // before any validation, and only the non-null rows are validated as JSON text. + val df = Seq(Some("[1,2]"), None, Some("{\"a\":1}")).toDF("s") + checkAnswer( + df.selectExpr("JSON_ARRAY(s FORMAT JSON)"), + Seq(Row("[[1,2]]"), Row("[]"), Row("""[{"a":1}]"""))) + checkAnswer( + df.selectExpr("JSON_ARRAY(s FORMAT JSON NULL ON NULL)"), + Seq(Row("[[1,2]]"), Row("[null]"), Row("""[{"a":1}]"""))) + // A non-null but malformed row still fails; the NULL row does not shield it. + val bad = Seq(None, Some("1,2")).toDF("s") + val e = intercept[SparkRuntimeException] { + bad.selectExpr("JSON_ARRAY(s FORMAT JSON NULL ON NULL)").collect() + } + assert(e.getCondition == "INVALID_JSON_FORMAT_JSON_VALUE") + } + + test("SQL round-trips FORMAT JSON and neutralizes an inlined implicit-JSON child") { + val inner = JsonArray( + Seq(Literal(1)), Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + // A nested constructor left in an implicit (formatJson = true, trusted) position round-trips + // as-is: reparse re-derives implicit FORMAT JSON. + val spliced = JsonArray( + Seq(inner), Seq(true), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + assert(spliced.sql == "JSON_ARRAY(JSON_ARRAY(1))") + // But a constructor inlined into a quoted (formatJson = false) position must be wrapped so + // reparse keeps it quoted -- otherwise ["[1]"] would round-trip to [[1]]. + val quoted = JsonArray( + Seq(inner), Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + assert(quoted.sql == "JSON_ARRAY(CAST(JSON_ARRAY(1) AS STRING))") + } + + test("emitted SQL reparses and evaluates with raw-vs-quoted semantics preserved") { + // The .sql renderings above are round-trip contracts: reparsing and evaluating them must + // reproduce the original splicing. A bare nested constructor stays spliced; a cast-neutralized + // one stays quoted. + checkAnswer(sql("SELECT JSON_ARRAY(JSON_ARRAY(1))"), Row("[[1]]")) + checkAnswer(sql("SELECT JSON_ARRAY(CAST(JSON_ARRAY(1) AS STRING))"), Row("""["[1]"]""")) + // An explicit FORMAT JSON string literal round-trips through the emitted SQL too. + val spliced = JsonArray( + Seq(Literal("[1,2]")), Seq(true), Seq(true), JsonConstructorNullBehavior.Absent, StringType) + assert(spliced.sql == "JSON_ARRAY('[1,2]' FORMAT JSON)") + checkAnswer(sql(s"SELECT ${spliced.sql}"), Row("[[1,2]]")) + } + + test("SQL forces FORMAT JSON for a spliced value whose child is not a bare constructor") { + // A spliced element whose direct child is a wrapper (e.g. a Collate around a nested + // constructor) must render an explicit `FORMAT JSON`, not rely on reparse re-deriving implicit + // JSON through the wrapper's rendering: `Collate.sql` renders function-style + // (collate(child, c)), which reparse would not recognize as an implicit nested constructor. + val inner = JsonArray( + Seq(Literal(1)), Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + val collated = JsonArray( + Seq(Collate(inner, ResolvedCollation("UTF8_LCASE"))), + Seq(true), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + assert(collated.sql.contains("FORMAT JSON"), + s"expected FORMAT JSON to force the splice, got: ${collated.sql}") + } + + test("SQL renders an explicit collated RETURNING and omits only the default") { + val collated = JsonArray( + Seq(Literal(1)), Seq(false), Seq(false), + JsonConstructorNullBehavior.Absent, StringType("UTF8_LCASE")) + assert(collated.sql.contains("RETURNING STRING COLLATE UTF8_LCASE")) + // The omitted default is the companion StringType (by reference) and renders no RETURNING. + val default = JsonArray( + Seq(Literal(1)), Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + assert(default.sql == "JSON_ARRAY(1)") + } + + test("a constant JSON_ARRAY is foldable unless it has an explicit FORMAT JSON") { + assert(JsonArray( + Seq(Literal(1), Literal("x")), Seq(false, false), Seq(false, false), + JsonConstructorNullBehavior.Absent, StringType).foldable) + // An explicit FORMAT JSON value is validated at eval and can throw, so it must not be folded + // (which would move the error to optimization time, even for rows a filter would drop). + assert(!JsonArray( + Seq(Literal("[1]")), Seq(true), Seq(true), + JsonConstructorNullBehavior.Absent, StringType).foldable) + } + + test("an explicit FORMAT JSON is not evaluated for rows a filter drops") { + // Because such a JSON_ARRAY is not foldable, its validation stays at runtime: a row the WHERE + // removes never triggers the malformed-JSON error (constant folding would have thrown eagerly). + checkAnswer( + sql("SELECT JSON_ARRAY('1,2' FORMAT JSON) AS x FROM VALUES (1) t(a) WHERE a > 100"), + Seq.empty) + // A surviving row still errors. + val e = intercept[SparkRuntimeException] { + sql("SELECT JSON_ARRAY('1,2' FORMAT JSON) AS x FROM VALUES (1) t(a)").collect() + } + assert(e.getCondition == "INVALID_JSON_FORMAT_JSON_VALUE") + } + + test("IS NULL checks over malformed FORMAT JSON still evaluate the constructor") { + // JsonArray is conservatively nullable when it can throw, so NullPropagation must not fold + // these predicates to literals before the FORMAT JSON validation runs. + Seq("IS NULL", "IS NOT NULL").foreach { predicate => + val e = intercept[SparkRuntimeException] { + sql(s"SELECT JSON_ARRAY('1,2' FORMAT JSON) $predicate").collect() + } + assert(e.getCondition == "INVALID_JSON_FORMAT_JSON_VALUE", s"for predicate $predicate") + } + } + + test("CHAR/VARCHAR RETURNING is normalized to STRING regardless of preserveCharVarcharTypeInfo") { + Seq("CHAR(2)", "VARCHAR(2)").foreach { returning => + Seq("true", "false").foreach { preserve => + withSQLConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO.key -> preserve) { + assert( + sql(s"SELECT JSON_ARRAY(1 RETURNING $returning)").schema.head.dataType === StringType, + s"for RETURNING $returning, preserveCharVarcharTypeInfo=$preserve") + } + } + } + } + + test("object default collation applies only when RETURNING is not explicitly collated") { + withSQLConf(SQLConf.OBJECT_LEVEL_COLLATIONS_ENABLED.key -> "true") { + withTable("t") { + sql( + """CREATE TABLE t DEFAULT COLLATION UTF8_LCASE AS + |SELECT json_array(1) AS a, + | json_array(1 RETURNING STRING COLLATE UTF8_BINARY) AS b""".stripMargin) + val schema = spark.table("t").schema + // Omitted RETURNING (default STRING) follows the table's default collation. + assert(schema("a").dataType === StringType("UTF8_LCASE")) + // Explicit RETURNING ... COLLATE is the user's choice and must not be overwritten. + assert(schema("b").dataType === StringType("UTF8_BINARY")) + } + } + } + + test("default collation recurses into a nested JSON_ARRAY value") { + // The rule casts each DefaultStringProducingExpression, recursing through a nested constructor + // (the flat cases above only cover a top-level constructor). This CTAS runs the default + // analyzer (single-pass included). Confirm the schema collation and that raw splicing still + // produces well-formed nested JSON at runtime. + withSQLConf(SQLConf.OBJECT_LEVEL_COLLATIONS_ENABLED.key -> "true") { + withTable("t") { + sql( + """CREATE TABLE t DEFAULT COLLATION UTF8_LCASE AS + |SELECT json_array(json_array(1)) AS a""".stripMargin) + assert(spark.table("t").schema("a").dataType === StringType("UTF8_LCASE")) + checkAnswer(spark.table("t"), Row("[[1]]")) + } + } + } + + test("view default collation preserves an explicit collated RETURNING") { + // Exercises the CREATE VIEW resolution path (in addition to the CTAS path above): the explicit + // RETURNING collation must survive the view's default collation. Pin the fixed-point analyzer: + // the single-pass resolver does not yet resolve a TimeZoneAware JSON constructor's timezone + // when re-resolving a view (a pre-existing gap independent of collation); the CTAS test above + // already exercises the single-pass path via the dual-run analyzer. + withSQLConf( + SQLConf.ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER.key -> "false", + SQLConf.OBJECT_LEVEL_COLLATIONS_ENABLED.key -> "true") { + withView("v") { + sql( + """CREATE VIEW v DEFAULT COLLATION UTF8_LCASE AS + |SELECT json_array(1) AS a, + | json_array(1 RETURNING STRING COLLATE UTF8_BINARY) AS b""".stripMargin) + val schema = spark.table("v").schema + assert(schema("a").dataType === StringType("UTF8_LCASE")) + assert(schema("b").dataType === StringType("UTF8_BINARY")) + } + } + } + + test("throwable is set only when the constructor can actually throw at eval") { + // An explicit FORMAT JSON value is validated at eval and can throw on malformed text, so the + // constructor must be throwable even when its children are not. + assert(JsonArray( + Seq(Literal("[1]")), Seq(true), Seq(true), + JsonConstructorNullBehavior.Absent, StringType).throwable) + // A plain JSON_ARRAY with no explicit FORMAT JSON cannot throw (RETURNING is STRING -> STRING), + // so it stays non-throwable and remains eligible for predicate pushdown. + assert(!JsonArray( + Seq(Literal(1)), Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + .throwable) + // A nested (implicit FORMAT JSON) constructor emits well-formed JSON by construction and is not + // validated (needsValidation = false), so it alone does not make the outer throwable -- even + // though it sits in a spliced (formatJson = true) position. + val nested = JsonArray( + Seq(Literal(1)), Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + assert(!JsonArray( + Seq(nested), Seq(true), Seq(false), JsonConstructorNullBehavior.Absent, StringType).throwable) + } + + test("frozen needsValidation survives an analyzer cast around a trusted nested constructor") { + // ApplyDefaultCollation / DefaultCollationTypeCoercion may wrap a trusted nested constructor in + // a Cast under a non-default object/view collation. The parse-time needsValidation = false must + // survive that rewrite (rather than being re-derived from the now-Cast child), so the outer + // stays foldable and non-throwable and does not spuriously validate the (trusted) nested output + // per row. + val nested = JsonArray( + Seq(Literal(1)), Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + val castWrapped = Cast(nested, StringType("UTF8_LCASE")) + val outer = JsonArray( + Seq(castWrapped), Seq(true), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + assert(outer.foldable, "trusted nested value must stay foldable even when Cast-wrapped") + assert(!outer.throwable, "trusted nested value must not become throwable when Cast-wrapped") + } + + test("throwable keeps a FORMAT JSON predicate above a filtering join") { + // The optimizer must not push a throwable predicate below the join (PushPredicateThroughJoin + // only pushes non-throwable conditions), so a malformed-JSON row the join eliminates is never + // evaluated and does not throw. Were the constructor not throwable, the predicate would push to + // the probe side and throw on the eliminated row. Use an equality predicate rather than + // `IS NOT NULL`: the constructor is non-nullable, so `NullPropagation` would rewrite + // `JSON_ARRAY(...) IS NOT NULL` to `true` and elide it before the join even matters -- an + // equality forces per-row evaluation and is not null-propagated. + withTempView("t", "u") { + Seq((1, "[1]"), (2, "1,2")).toDF("id", "s").createOrReplaceTempView("t") + Seq(1).toDF("id").createOrReplaceTempView("u") + // id=2 carries malformed FORMAT JSON text but does not join u, so it is dropped first and its + // predicate is never evaluated (it was not pushed below the join). + checkAnswer( + sql("""SELECT t.id FROM t JOIN u ON t.id = u.id + |WHERE JSON_ARRAY(t.s FORMAT JSON) = '[[1]]'""".stripMargin), + Row(1)) + // With the malformed row surviving the join, evaluation still throws. + Seq(2).toDF("id").createOrReplaceTempView("u") + val e = intercept[SparkRuntimeException] { + sql("""SELECT t.id FROM t JOIN u ON t.id = u.id + |WHERE JSON_ARRAY(t.s FORMAT JSON) = '[[1]]'""".stripMargin).collect() + } + assert(e.getCondition == "INVALID_JSON_FORMAT_JSON_VALUE") + } + } + + test("a spatial-typed element is rejected at analysis") { + // GEOMETRY / GEOGRAPHY are AtomicTypes that JacksonUtils.verifyType accepts but + // JacksonGenerator cannot serialize, so JSON_ARRAY rejects them up front, not at runtime. + Seq(GeometryType(4326), GeographyType(4326)).foreach { dt => + val expr = JsonArray( + Seq(Literal.create(null, dt)), Seq(false), Seq(false), + JsonConstructorNullBehavior.Absent, StringType) + expr.checkInputDataTypes() match { + case DataTypeMismatch(errorSubClass, _) => + assert(errorSubClass == "CANNOT_CONVERT_TO_JSON", s"for $dt") + case other => fail(s"expected DataTypeMismatch for $dt, got $other") + } + } + } + + test("a spatial type is accepted when it appears only as a map key") { + // JacksonGenerator writes map keys via toString, so a spatial *key* is serializable; only map + // values (and struct fields / array elements / top-level) go through a typed writer. The guard + // must therefore mirror verifyType and not over-reject a spatial map key. + val ok = JsonArray( + Seq(Literal.create(null, MapType(GeometryType(4326), IntegerType))), + Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + assert(ok.checkInputDataTypes().isSuccess) + // But a spatial map *value* is still rejected. + val bad = JsonArray( + Seq(Literal.create(null, MapType(StringType, GeometryType(4326)))), + Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, StringType) + assert(bad.checkInputDataTypes().isFailure) + } + + test("NULL FORMAT JSON is accepted and follows ON NULL handling") { + // An untyped NULL under FORMAT JSON must not be rejected at analysis: eval handles nulls before + // it would ever splice, so it behaves like any other NULL element. + checkAnswer(sql("SELECT JSON_ARRAY(NULL FORMAT JSON)"), Row("[]")) + checkAnswer(sql("SELECT JSON_ARRAY(NULL FORMAT JSON NULL ON NULL)"), Row("[null]")) + checkAnswer(sql("SELECT JSON_ARRAY(1, NULL FORMAT JSON, 3 NULL ON NULL)"), Row("[1,null,3]")) + } + + test("value accepts an unparenthesized predicate expression") { + // jsonArrayValue is parsed as a full `expression`, so predicates work without parentheses. + checkAnswer(sql("SELECT JSON_ARRAY(1 IS NULL, 2 > 1)"), Row("[false,true]")) + } + + test("widening the value to expression does not change documented forms") { + // Design-doc examples where a value abuts ON NULL / FORMAT JSON must parse and evaluate + // identically after widening valueExpression -> expression. + checkAnswer(sql("SELECT JSON_ARRAY(1, NULL, 3 NULL ON NULL)"), Row("[1,null,3]")) + checkAnswer(sql("SELECT JSON_ARRAY('[1,2]' FORMAT JSON)"), Row("[[1,2]]")) + checkAnswer(sql("SELECT JSON_ARRAY(1, 'x', true)"), Row("""[1,"x",true]""")) + } + + test("a non-string RETURNING type is rejected at analysis") { + val e = intercept[AnalysisException] { + sql("SELECT JSON_ARRAY(1 RETURNING INT)").collect() + } + assert(e.getCondition == "DATATYPE_MISMATCH.INVALID_JSON_RETURNING_TYPE") + } + + test("a directly-constructed JsonArray with a CHAR/VARCHAR RETURNING is rejected") { + // The parser normalizes CHAR/VARCHAR RETURNING to STRING, but a raw CharType/VarcharType from + // direct Catalyst construction would advertise a length JSON_ARRAY does not enforce. + Seq(VarcharType(2), CharType(2)).foreach { returning => + val expr = JsonArray( + Seq(Literal(1)), Seq(false), Seq(false), JsonConstructorNullBehavior.Absent, returning) + expr.checkInputDataTypes() match { + case DataTypeMismatch(errorSubClass, _) => + assert(errorSubClass == "INVALID_JSON_RETURNING_TYPE", s"for $returning") + case other => fail(s"expected DataTypeMismatch for $returning, got $other") + } + } + } + + test("JSON_ARRAY of a struct follows to_json null-field handling (ignoreNullFields)") { + // Nested struct field nulls are governed by spark.sql.jsonGenerator.ignoreNullFields, exactly + // as to_json -- JSON_ARRAY intentionally reuses that JSON writer. The (NULL | ABSENT) ON NULL + // clause controls only top-level array elements, not fields inside a struct element. + val q = "SELECT JSON_ARRAY(named_struct('a', 1, 'b', CAST(NULL AS INT)))" + withSQLConf(SQLConf.JSON_GENERATOR_IGNORE_NULL_FIELDS.key -> "true") { + checkAnswer(sql(q), Row("""[{"a":1}]""")) + } + withSQLConf(SQLConf.JSON_GENERATOR_IGNORE_NULL_FIELDS.key -> "false") { + checkAnswer(sql(q), Row("""[{"a":1,"b":null}]""")) + } + } + +} diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala index e6aa18256c90..fd83757e8573 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala @@ -214,7 +214,7 @@ trait ThriftServerWithSparkContextSuite extends SharedThriftServer { val sessionHandle = client.openSession(user, "") val infoValue = client.getInfo(sessionHandle, GetInfoType.CLI_ODBC_KEYWORDS) // scalastyle:off line.size.limit - assert(infoValue.getStringValue == "ADD,AFTER,AGGREGATE,ALIGN,ALL,ALTER,ALWAYS,ANALYZE,AND,ANTI,ANY,ANY_VALUE,APPLY,APPROX,ARCHIVE,ARRAY,AS,ASC,ASENSITIVE,ASOF,AT,ATOMIC,AUTHORIZATION,AUTO,BEGIN,BERNOULLI,BETWEEN,BIGINT,BIN,BINARY,BINDING,BIN_DISTRIBUTE_RATIO,BIN_END,BIN_START,BOOLEAN,BOTH,BUCKET,BUCKETS,BY,BYTE,CACHE,CALL,CALLED,CASCADE,CASE,CAST,CATALOG,CATALOGS,CDC,CHANGE,CHANGES,CHAR,CHARACTER,CHECK,CLEAR,CLOSE,CLUSTER,CLUSTERED,CODEGEN,COLLATE,COLLATION,COLLATIONS,COLLECTION,COLUMN,COLUMNS,COMMENT,COMMIT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONDITION,CONSTRAINT,CONTAINS,CONTINUE,COST,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATABASE,CURRENT_DATE,CURRENT_PATH,CURRENT_SCHEMA,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,DATA,DATABASE,DATABASES,DATE,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAY,DAYOFYEAR,DAYS,DBPROPERTIES,DEC,DECIMAL,DECLARE,DEFAULT,DEFAULT_PATH,DEFINED,DEFINER,DELAY,DELETE,DELIMITED,DESC,DESCRIBE,DETERMINISTIC,DFS,DIRECTORIES,DIRECTORY,DISTANCE,DISTINCT,DISTRIBUTE,DIV,DO,DOUBLE,DROP,ELSE,ELSEIF,EMPTY,END,ENFORCED,ERROR,ESCAPE,ESCAPED,EVOLUTION,EXACT,EXCEPT,EXCHANGE,EXCLUDE,EXCLUSIVE,EXECUTE,EXISTS,EXIT,EXPLAIN,EXPORT,EXTEND,EXTENDED,EXTERNAL,EXTRACT,FALSE,FETCH,FIELDS,FILEFORMAT,FILTER,FIRST,FLOAT,FLOW,FOLLOWING,FOR,FOREIGN,FORMAT,FORMATTED,FOUND,FROM,FULL,FUNCTION,FUNCTIONS,GENERATED,GEOGRAPHY,GEOMETRY,GLOBAL,GRANT,GROUP,GROUPING,HANDLER,HAVING,HISTORY,HOUR,HOURS,IDENTIFIED,IDENTIFIER,IDENTITY,IF,IGNORE,ILIKE,IMMEDIATE,IMPORT,IN,INCLUDE,INCLUSIVE,INCREMENT,INDEX,INDEXES,INNER,INPATH,INPUT,INPUTFORMAT,INSENSITIVE,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,INVOKER,IS,ITEMS,ITERATE,JOIN,JSON,JSON_TABLE,JSON_VALUE,KEY,KEYS,LANGUAGE,LAST,LATERAL,LAZY,LEADING,LEAVE,LEFT,LEVEL,LIKE,LIMIT,LINES,LIST,LOAD,LOCAL,LOCALTIME,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MATCH_CONDITION,MATERIALIZED,MAX,MEASURE,MERGE,METRICS,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTE,MINUTES,MODIFIES,MONTH,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NATURAL,NEAREST,NEXT,NO,NONE,NORELY,NOT,NULL,NULLS,NUMERIC,OF,OFFSET,ON,ONLY,OPEN,OPTION,OPTIONS,OR,ORDER,ORDINALITY,OUT,OUTER,OUTPUTFORMAT,OVER,OVERLAPS,OVERLAY,OVERWRITE,PARTITION,PARTITIONED,PARTITIONS,PATH,PERCENT,PIVOT,PLACING,POSITION,PRECEDING,PRIMARY,PRINCIPALS,PROCEDURE,PROCEDURES,PROPERTIES,PURGE,QUALIFY,QUARTER,QUERY,RANGE,READ,READS,REAL,RECORDREADER,RECORDWRITER,RECOVER,RECURSION,RECURSIVE,REDUCE,REFERENCES,REFRESH,RELY,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURN,RETURNING,RETURNS,REVOKE,RIGHT,ROLE,ROLES,ROLLBACK,ROLLUP,ROW,ROWS,SCD,SCHEMA,SCHEMAS,SECOND,SECONDS,SECURITY,SELECT,SEMI,SEPARATED,SEQUENCE,SERDE,SERDEPROPERTIES,SESSION_USER,SET,SETS,SHORT,SHOW,SIMILARITY,SINGLE,SKEWED,SMALLINT,SOME,SORT,SORTED,SOURCE,SPECIFIC,SQL,SQLEXCEPTION,SQLSTATE,START,STATISTICS,STORED,STRATIFY,STREAM,STREAMING,STRING,STRUCT,SUBSTR,SUBSTRING,SYNC,SYSTEM,SYSTEM_PATH,SYSTEM_TIME,SYSTEM_VERSION,TABLE,TABLES,TABLESAMPLE,TARGET,TBLPROPERTIES,TERMINATED,THEN,TIME,TIMEDIFF,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TO,TOUCH,TRACK,TRAILING,TRANSACTION,TRANSACTIONS,TRANSFORM,TRIM,TRUE,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNIFORM,UNION,UNIQUE,UNKNOWN,UNLOCK,UNNEST,UNPIVOT,UNSET,UNTIL,UPDATE,USE,USER,USING,VALUE,VALUES,VAR,VARCHAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WATERMARK,WEEK,WEEKS,WHEN,WHERE,WHILE,WIDTH,WINDOW,WITH,WITHIN,WITHOUT,X,YEAR,YEARS,ZONE") + assert(infoValue.getStringValue == "ABSENT,ADD,AFTER,AGGREGATE,ALIGN,ALL,ALTER,ALWAYS,ANALYZE,AND,ANTI,ANY,ANY_VALUE,APPLY,APPROX,ARCHIVE,ARRAY,AS,ASC,ASENSITIVE,ASOF,AT,ATOMIC,AUTHORIZATION,AUTO,BEGIN,BERNOULLI,BETWEEN,BIGINT,BIN,BINARY,BINDING,BIN_DISTRIBUTE_RATIO,BIN_END,BIN_START,BOOLEAN,BOTH,BUCKET,BUCKETS,BY,BYTE,CACHE,CALL,CALLED,CASCADE,CASE,CAST,CATALOG,CATALOGS,CDC,CHANGE,CHANGES,CHAR,CHARACTER,CHECK,CLEAR,CLOSE,CLUSTER,CLUSTERED,CODEGEN,COLLATE,COLLATION,COLLATIONS,COLLECTION,COLUMN,COLUMNS,COMMENT,COMMIT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONDITION,CONSTRAINT,CONTAINS,CONTINUE,COST,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATABASE,CURRENT_DATE,CURRENT_PATH,CURRENT_SCHEMA,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,DATA,DATABASE,DATABASES,DATE,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAY,DAYOFYEAR,DAYS,DBPROPERTIES,DEC,DECIMAL,DECLARE,DEFAULT,DEFAULT_PATH,DEFINED,DEFINER,DELAY,DELETE,DELIMITED,DESC,DESCRIBE,DETERMINISTIC,DFS,DIRECTORIES,DIRECTORY,DISTANCE,DISTINCT,DISTRIBUTE,DIV,DO,DOUBLE,DROP,ELSE,ELSEIF,EMPTY,END,ENFORCED,ERROR,ESCAPE,ESCAPED,EVOLUTION,EXACT,EXCEPT,EXCHANGE,EXCLUDE,EXCLUSIVE,EXECUTE,EXISTS,EXIT,EXPLAIN,EXPORT,EXTEND,EXTENDED,EXTERNAL,EXTRACT,FALSE,FETCH,FIELDS,FILEFORMAT,FILTER,FIRST,FLOAT,FLOW,FOLLOWING,FOR,FOREIGN,FORMAT,FORMATTED,FOUND,FROM,FULL,FUNCTION,FUNCTIONS,GENERATED,GEOGRAPHY,GEOMETRY,GLOBAL,GRANT,GROUP,GROUPING,HANDLER,HAVING,HISTORY,HOUR,HOURS,IDENTIFIED,IDENTIFIER,IDENTITY,IF,IGNORE,ILIKE,IMMEDIATE,IMPORT,IN,INCLUDE,INCLUSIVE,INCREMENT,INDEX,INDEXES,INNER,INPATH,INPUT,INPUTFORMAT,INSENSITIVE,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,INVOKER,IS,ITEMS,ITERATE,JOIN,JSON,JSON_ARRAY,JSON_TABLE,JSON_VALUE,KEY,KEYS,LANGUAGE,LAST,LATERAL,LAZY,LEADING,LEAVE,LEFT,LEVEL,LIKE,LIMIT,LINES,LIST,LOAD,LOCAL,LOCALTIME,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MATCH_CONDITION,MATERIALIZED,MAX,MEASURE,MERGE,METRICS,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTE,MINUTES,MODIFIES,MONTH,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NATURAL,NEAREST,NEXT,NO,NONE,NORELY,NOT,NULL,NULLS,NUMERIC,OF,OFFSET,ON,ONLY,OPEN,OPTION,OPTIONS,OR,ORDER,ORDINALITY,OUT,OUTER,OUTPUTFORMAT,OVER,OVERLAPS,OVERLAY,OVERWRITE,PARTITION,PARTITIONED,PARTITIONS,PATH,PERCENT,PIVOT,PLACING,POSITION,PRECEDING,PRIMARY,PRINCIPALS,PROCEDURE,PROCEDURES,PROPERTIES,PURGE,QUALIFY,QUARTER,QUERY,RANGE,READ,READS,REAL,RECORDREADER,RECORDWRITER,RECOVER,RECURSION,RECURSIVE,REDUCE,REFERENCES,REFRESH,RELY,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURN,RETURNING,RETURNS,REVOKE,RIGHT,ROLE,ROLES,ROLLBACK,ROLLUP,ROW,ROWS,SCD,SCHEMA,SCHEMAS,SECOND,SECONDS,SECURITY,SELECT,SEMI,SEPARATED,SEQUENCE,SERDE,SERDEPROPERTIES,SESSION_USER,SET,SETS,SHORT,SHOW,SIMILARITY,SINGLE,SKEWED,SMALLINT,SOME,SORT,SORTED,SOURCE,SPECIFIC,SQL,SQLEXCEPTION,SQLSTATE,START,STATISTICS,STORED,STRATIFY,STREAM,STREAMING,STRING,STRUCT,SUBSTR,SUBSTRING,SYNC,SYSTEM,SYSTEM_PATH,SYSTEM_TIME,SYSTEM_VERSION,TABLE,TABLES,TABLESAMPLE,TARGET,TBLPROPERTIES,TERMINATED,THEN,TIME,TIMEDIFF,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TO,TOUCH,TRACK,TRAILING,TRANSACTION,TRANSACTIONS,TRANSFORM,TRIM,TRUE,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNIFORM,UNION,UNIQUE,UNKNOWN,UNLOCK,UNNEST,UNPIVOT,UNSET,UNTIL,UPDATE,USE,USER,USING,VALUE,VALUES,VAR,VARCHAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WATERMARK,WEEK,WEEKS,WHEN,WHERE,WHILE,WIDTH,WINDOW,WITH,WITHIN,WITHOUT,X,YEAR,YEARS,ZONE") // scalastyle:on line.size.limit } }