'" -> "end of input")
/** Get the user-facing display of the error token. */
override def getTokenErrorDisplay(t: Token): String = {
@@ -76,9 +76,7 @@ class SparkParserErrorStrategy() extends DefaultErrorStrategy {
val exceptionWithErrorClass = new SparkRecognitionException(
e,
"PARSE_SYNTAX_ERROR",
- messageParameters = Map(
- "error" -> getTokenErrorDisplay(e.getOffendingToken),
- "hint" -> ""))
+ messageParameters = Map("error" -> getTokenErrorDisplay(e.getOffendingToken), "hint" -> ""))
recognizer.notifyErrorListeners(e.getOffendingToken, "", exceptionWithErrorClass)
}
@@ -116,18 +114,17 @@ class SparkParserErrorStrategy() extends DefaultErrorStrategy {
/**
* Inspired by [[org.antlr.v4.runtime.BailErrorStrategy]], which is used in two-stage parsing:
- * This error strategy allows the first stage of two-stage parsing to immediately terminate
- * if an error is encountered, and immediately fall back to the second stage. In addition to
- * avoiding wasted work by attempting to recover from errors here, the empty implementation
- * of sync improves the performance of the first stage.
+ * This error strategy allows the first stage of two-stage parsing to immediately terminate if an
+ * error is encountered, and immediately fall back to the second stage. In addition to avoiding
+ * wasted work by attempting to recover from errors here, the empty implementation of sync
+ * improves the performance of the first stage.
*/
class SparkParserBailErrorStrategy() extends SparkParserErrorStrategy {
/**
- * Instead of recovering from exception e, re-throw it wrapped
- * in a [[ParseCancellationException]] so it is not caught by the
- * rule function catches. Use [[Exception#getCause]] to get the
- * original [[RecognitionException]].
+ * Instead of recovering from exception e, re-throw it wrapped in a
+ * [[ParseCancellationException]] so it is not caught by the rule function catches. Use
+ * [[Exception#getCause]] to get the original [[RecognitionException]].
*/
override def recover(recognizer: Parser, e: RecognitionException): Unit = {
var context = recognizer.getContext
@@ -139,8 +136,8 @@ class SparkParserBailErrorStrategy() extends SparkParserErrorStrategy {
}
/**
- * Make sure we don't attempt to recover inline; if the parser
- * successfully recovers, it won't throw an exception.
+ * Make sure we don't attempt to recover inline; if the parser successfully recovers, it won't
+ * throw an exception.
*/
@throws[RecognitionException]
override def recoverInline(recognizer: Parser): Token = {
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala
index 0b9e6ea021be1..10da24567545b 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.types.{DataType, StructType}
* Base SQL parsing infrastructure.
*/
abstract class AbstractParser extends DataTypeParserInterface with Logging {
+
/** Creates/Resolves DataType for a given SQL string. */
override def parseDataType(sqlText: String): DataType = parse(sqlText) { parser =>
astBuilder.visitSingleDataType(parser.singleDataType())
@@ -78,8 +79,7 @@ abstract class AbstractParser extends DataTypeParserInterface with Logging {
parser.setErrorHandler(new SparkParserBailErrorStrategy())
parser.getInterpreter.setPredictionMode(PredictionMode.SLL)
toResult(parser)
- }
- catch {
+ } catch {
case e: ParseCancellationException =>
// if we fail, parse with LL mode w/ SparkParserErrorStrategy
tokenStream.seek(0) // rewind input stream
@@ -90,8 +90,7 @@ abstract class AbstractParser extends DataTypeParserInterface with Logging {
parser.getInterpreter.setPredictionMode(PredictionMode.LL)
toResult(parser)
}
- }
- catch {
+ } catch {
case e: ParseException if e.command.isDefined =>
throw e
case e: ParseException =>
@@ -187,7 +186,7 @@ case object ParseErrorListener extends BaseErrorListener {
* A [[ParseException]] is an [[SparkException]] that is thrown during the parse process. It
* contains fields and an extended error message that make reporting and diagnosing errors easier.
*/
-class ParseException private(
+class ParseException private (
val command: Option[String],
message: String,
val start: Origin,
@@ -195,17 +194,18 @@ class ParseException private(
errorClass: Option[String] = None,
messageParameters: Map[String, String] = Map.empty,
queryContext: Array[QueryContext] = ParseException.getQueryContext())
- extends AnalysisException(
- message,
- start.line,
- start.startPosition,
- None,
- errorClass,
- messageParameters,
- queryContext) {
+ extends AnalysisException(
+ message,
+ start.line,
+ start.startPosition,
+ None,
+ errorClass,
+ messageParameters,
+ queryContext) {
def this(errorClass: String, messageParameters: Map[String, String], ctx: ParserRuleContext) =
- this(Option(SparkParserUtils.command(ctx)),
+ this(
+ Option(SparkParserUtils.command(ctx)),
SparkThrowableHelper.getMessage(errorClass, messageParameters),
SparkParserUtils.position(ctx.getStart),
SparkParserUtils.position(ctx.getStop),
@@ -310,14 +310,16 @@ case object PostProcessor extends SqlBaseParserBaseListener {
throw QueryParsingErrors.invalidIdentifierError(ident, ctx)
}
- /** Throws error message when unquoted identifier contains characters outside a-z, A-Z, 0-9, _ */
+ /**
+ * Throws error message when unquoted identifier contains characters outside a-z, A-Z, 0-9, _
+ */
override def exitUnquotedIdentifier(ctx: SqlBaseParser.UnquotedIdentifierContext): Unit = {
val ident = ctx.getText
if (ident.exists(c =>
- !(c >= 'a' && c <= 'z') &&
- !(c >= 'A' && c <= 'Z') &&
- !(c >= '0' && c <= '9') &&
- c != '_')) {
+ !(c >= 'a' && c <= 'z') &&
+ !(c >= 'A' && c <= 'Z') &&
+ !(c >= '0' && c <= '9') &&
+ c != '_')) {
throw QueryParsingErrors.invalidIdentifierError(ident, ctx)
}
}
@@ -353,9 +355,7 @@ case object PostProcessor extends SqlBaseParserBaseListener {
replaceTokenByIdentifier(ctx, 0)(identity)
}
- private def replaceTokenByIdentifier(
- ctx: ParserRuleContext,
- stripMargins: Int)(
+ private def replaceTokenByIdentifier(ctx: ParserRuleContext, stripMargins: Int)(
f: CommonToken => CommonToken = identity): Unit = {
val parent = ctx.getParent
parent.removeLastChild()
@@ -373,8 +373,8 @@ case object PostProcessor extends SqlBaseParserBaseListener {
/**
* The post-processor checks the unclosed bracketed comment.
*/
-case class UnclosedCommentProcessor(
- command: String, tokenStream: CommonTokenStream) extends SqlBaseParserBaseListener {
+case class UnclosedCommentProcessor(command: String, tokenStream: CommonTokenStream)
+ extends SqlBaseParserBaseListener {
override def exitSingleDataType(ctx: SqlBaseParser.SingleDataTypeContext): Unit = {
checkUnclosedComment(tokenStream, command)
@@ -384,7 +384,8 @@ case class UnclosedCommentProcessor(
checkUnclosedComment(tokenStream, command)
}
- override def exitSingleTableIdentifier(ctx: SqlBaseParser.SingleTableIdentifierContext): Unit = {
+ override def exitSingleTableIdentifier(
+ ctx: SqlBaseParser.SingleTableIdentifierContext): Unit = {
checkUnclosedComment(tokenStream, command)
}
@@ -422,7 +423,8 @@ case class UnclosedCommentProcessor(
// The last token is 'EOF' and the penultimate is unclosed bracketed comment
val failedToken = tokenStream.get(tokenStream.size() - 2)
assert(failedToken.getType() == SqlBaseParser.BRACKETED_COMMENT)
- val position = Origin(Option(failedToken.getLine), Option(failedToken.getCharPositionInLine))
+ val position =
+ Origin(Option(failedToken.getLine), Option(failedToken.getCharPositionInLine))
throw QueryParsingErrors.unclosedBracketedCommentError(
command = command,
start = Origin(Option(failedToken.getStartIndex)),
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeMode.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeMode.scala
index 4c7c87504ffc4..e870a83ec4ae6 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeMode.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/TimeMode.scala
@@ -37,9 +37,10 @@ object TimeModes {
ProcessingTime
case "eventtime" =>
EventTime
- case _ => throw new SparkIllegalArgumentException(
- errorClass = "STATEFUL_PROCESSOR_UNKNOWN_TIME_MODE",
- messageParameters = Map("timeMode" -> timeMode))
+ case _ =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "STATEFUL_PROCESSOR_UNKNOWN_TIME_MODE",
+ messageParameters = Map("timeMode" -> timeMode))
}
}
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/streaming/InternalOutputModes.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/streaming/InternalOutputModes.scala
index 0ccfc3cbc7bf2..8c54758ebc0bc 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/streaming/InternalOutputModes.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/streaming/InternalOutputModes.scala
@@ -28,27 +28,26 @@ import org.apache.spark.sql.streaming.OutputMode
private[sql] object InternalOutputModes {
/**
- * OutputMode in which only the new rows in the streaming DataFrame/Dataset will be
- * written to the sink. This output mode can be only be used in queries that do not
- * contain any aggregation.
+ * OutputMode in which only the new rows in the streaming DataFrame/Dataset will be written to
+ * the sink. This output mode can be only be used in queries that do not contain any
+ * aggregation.
*/
case object Append extends OutputMode
/**
- * OutputMode in which all the rows in the streaming DataFrame/Dataset will be written
- * to the sink every time these is some updates. This output mode can only be used in queries
- * that contain aggregations.
+ * OutputMode in which all the rows in the streaming DataFrame/Dataset will be written to the
+ * sink every time these is some updates. This output mode can only be used in queries that
+ * contain aggregations.
*/
case object Complete extends OutputMode
/**
- * OutputMode in which only the rows in the streaming DataFrame/Dataset that were updated will be
- * written to the sink every time these is some updates. If the query doesn't contain
+ * OutputMode in which only the rows in the streaming DataFrame/Dataset that were updated will
+ * be written to the sink every time these is some updates. If the query doesn't contain
* aggregations, it will be equivalent to `Append` mode.
*/
case object Update extends OutputMode
-
def apply(outputMode: String): OutputMode = {
outputMode.toLowerCase(Locale.ROOT) match {
case "append" =>
@@ -57,7 +56,8 @@ private[sql] object InternalOutputModes {
OutputMode.Complete
case "update" =>
OutputMode.Update
- case _ => throw new SparkIllegalArgumentException(
+ case _ =>
+ throw new SparkIllegalArgumentException(
errorClass = "STREAMING_OUTPUT_MODE.INVALID",
messageParameters = Map("outputMode" -> outputMode))
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/QueryContexts.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/QueryContexts.scala
index 2b3f4674539e3..d5be65a2f36cf 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/QueryContexts.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/QueryContexts.scala
@@ -27,7 +27,8 @@ case class SQLQueryContext(
originStopIndex: Option[Int],
sqlText: Option[String],
originObjectType: Option[String],
- originObjectName: Option[String]) extends QueryContext {
+ originObjectName: Option[String])
+ extends QueryContext {
override val contextType = QueryContextType.SQL
override val objectType = originObjectType.getOrElse("")
@@ -37,9 +38,8 @@ case class SQLQueryContext(
/**
* The SQL query context of current node. For example:
- * == SQL of VIEW v1 (line 1, position 25) ==
- * SELECT '' AS five, i.f1, i.f1 - int('2') AS x FROM INT4_TBL i
- * ^^^^^^^^^^^^^^^
+ * ==SQL of VIEW v1 (line 1, position 25)==
+ * SELECT '' AS five, i.f1, i.f1 - int('2') AS x FROM INT4_TBL i ^^^^^^^^^^^^^^^
*/
override lazy val summary: String = {
// If the query context is missing or incorrect, simply return an empty string.
@@ -127,8 +127,8 @@ case class SQLQueryContext(
def isValid: Boolean = {
sqlText.isDefined && originStartIndex.isDefined && originStopIndex.isDefined &&
- originStartIndex.get >= 0 && originStopIndex.get < sqlText.get.length &&
- originStartIndex.get <= originStopIndex.get
+ originStartIndex.get >= 0 && originStopIndex.get < sqlText.get.length &&
+ originStartIndex.get <= originStopIndex.get
}
override def callSite: String = throw SparkUnsupportedOperationException()
@@ -136,7 +136,8 @@ case class SQLQueryContext(
case class DataFrameQueryContext(
stackTrace: Seq[StackTraceElement],
- pysparkErrorContext: Option[(String, String)]) extends QueryContext {
+ pysparkErrorContext: Option[(String, String)])
+ extends QueryContext {
override val contextType = QueryContextType.DataFrame
override def objectType: String = throw SparkUnsupportedOperationException()
@@ -146,19 +147,21 @@ case class DataFrameQueryContext(
override val fragment: String = {
pysparkErrorContext.map(_._1).getOrElse {
- stackTrace.headOption.map { firstElem =>
- val methodName = firstElem.getMethodName
- if (methodName.length > 1 && methodName(0) == '$') {
- methodName.substring(1)
- } else {
- methodName
+ stackTrace.headOption
+ .map { firstElem =>
+ val methodName = firstElem.getMethodName
+ if (methodName.length > 1 && methodName(0) == '$') {
+ methodName.substring(1)
+ } else {
+ methodName
+ }
}
- }.getOrElse("")
+ .getOrElse("")
}
}
- override val callSite: String = pysparkErrorContext.map(
- _._2).getOrElse(stackTrace.tail.mkString("\n"))
+ override val callSite: String =
+ pysparkErrorContext.map(_._2).getOrElse(stackTrace.tail.mkString("\n"))
override lazy val summary: String = {
val builder = new StringBuilder
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/origin.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/origin.scala
index 3e0ebdd627b63..33fa17433abbd 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/origin.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/trees/origin.scala
@@ -23,9 +23,9 @@ import org.apache.spark.sql.internal.SqlApiConf
import org.apache.spark.util.ArrayImplicits._
/**
- * Contexts of TreeNodes, including location, SQL text, object type and object name.
- * The only supported object type is "VIEW" now. In the future, we may support SQL UDF or other
- * objects which contain SQL text.
+ * Contexts of TreeNodes, including location, SQL text, object type and object name. The only
+ * supported object type is "VIEW" now. In the future, we may support SQL UDF or other objects
+ * which contain SQL text.
*/
case class Origin(
line: Option[Int] = None,
@@ -41,8 +41,7 @@ case class Origin(
lazy val context: QueryContext = if (stackTrace.isDefined) {
DataFrameQueryContext(stackTrace.get.toImmutableArraySeq, pysparkErrorContext)
} else {
- SQLQueryContext(
- line, startPosition, startIndex, stopIndex, sqlText, objectType, objectName)
+ SQLQueryContext(line, startPosition, startIndex, stopIndex, sqlText, objectType, objectName)
}
def getQueryContext: Array[QueryContext] = {
@@ -61,7 +60,7 @@ trait WithOrigin {
}
/**
- * Provides a location for TreeNodes to ask about the context of their origin. For example, which
+ * Provides a location for TreeNodes to ask about the context of their origin. For example, which
* line of code is currently being parsed.
*/
object CurrentOrigin {
@@ -75,8 +74,7 @@ object CurrentOrigin {
def reset(): Unit = value.set(Origin())
def setPosition(line: Int, start: Int): Unit = {
- value.set(
- value.get.copy(line = Some(line), startPosition = Some(start)))
+ value.set(value.get.copy(line = Some(line), startPosition = Some(start)))
}
def withOrigin[A](o: Origin)(f: => A): A = {
@@ -84,25 +82,29 @@ object CurrentOrigin {
// this way withOrigin can be recursive
val previous = get
set(o)
- val ret = try f finally { set(previous) }
+ val ret =
+ try f
+ finally { set(previous) }
ret
}
/**
- * This helper function captures the Spark API and its call site in the user code from the current
- * stacktrace.
+ * This helper function captures the Spark API and its call site in the user code from the
+ * current stacktrace.
*
* As adding `withOrigin` explicitly to all Spark API definition would be a huge change,
* `withOrigin` is used only at certain places where all API implementation surely pass through
- * and the current stacktrace is filtered to the point where first Spark API code is invoked from
- * the user code.
+ * and the current stacktrace is filtered to the point where first Spark API code is invoked
+ * from the user code.
*
* As there might be multiple nested `withOrigin` calls (e.g. any Spark API implementations can
* invoke other APIs) only the first `withOrigin` is captured because that is closer to the user
* code.
*
- * @param f The function that can use the origin.
- * @return The result of `f`.
+ * @param f
+ * The function that can use the origin.
+ * @return
+ * The result of `f`.
*/
private[sql] def withOrigin[T](f: => T): T = {
if (CurrentOrigin.get.stackTrace.isDefined) {
@@ -114,9 +116,9 @@ object CurrentOrigin {
while (i < st.length && !sparkCode(st(i))) i += 1
// Stop at the end of the first Spark code traces
while (i < st.length && sparkCode(st(i))) i += 1
- val origin = Origin(stackTrace = Some(st.slice(
- from = i - 1,
- until = i + SqlApiConf.get.stackTracesInDataFrameContext)),
+ val origin = Origin(
+ stackTrace =
+ Some(st.slice(from = i - 1, until = i + SqlApiConf.get.stackTracesInDataFrameContext)),
pysparkErrorContext = PySparkCurrentOrigin.get())
CurrentOrigin.withOrigin(origin)(f)
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/AttributeNameParser.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/AttributeNameParser.scala
index e47ab1978d0ed..533b09e82df13 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/AttributeNameParser.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/AttributeNameParser.scala
@@ -19,11 +19,11 @@ package org.apache.spark.sql.catalyst.util
import org.apache.spark.sql.errors.DataTypeErrors
trait AttributeNameParser {
+
/**
- * Used to split attribute name by dot with backticks rule.
- * Backticks must appear in pairs, and the quoted string must be a complete name part,
- * which means `ab..c`e.f is not allowed.
- * We can use backtick only inside quoted name parts.
+ * Used to split attribute name by dot with backticks rule. Backticks must appear in pairs, and
+ * the quoted string must be a complete name part, which means `ab..c`e.f is not allowed. We can
+ * use backtick only inside quoted name parts.
*/
def parseAttributeName(name: String): Seq[String] = {
def e = DataTypeErrors.attributeNameSyntaxError(name)
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index 640304efce4b4..9e90feeb782d6 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -23,12 +23,12 @@ import java.util.Locale
* Builds a map in which keys are case insensitive. Input map can be accessed for cases where
* case-sensitive information is required. The primary constructor is marked private to avoid
* nested case-insensitive map creation, otherwise the keys in the original map will become
- * case-insensitive in this scenario.
- * Note: CaseInsensitiveMap is serializable. However, after transformation, e.g. `filterKeys()`,
- * it may become not serializable.
+ * case-insensitive in this scenario. Note: CaseInsensitiveMap is serializable. However, after
+ * transformation, e.g. `filterKeys()`, it may become not serializable.
*/
-class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Map[String, T]
- with Serializable {
+class CaseInsensitiveMap[T] private (val originalMap: Map[String, T])
+ extends Map[String, T]
+ with Serializable {
val keyLowerCasedMap = originalMap.map(kv => kv.copy(_1 = kv._1.toLowerCase(Locale.ROOT)))
@@ -62,4 +62,3 @@ object CaseInsensitiveMap {
case _ => new CaseInsensitiveMap(params)
}
}
-
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala
index e75429c58cc7b..b8ab633b2047f 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala
@@ -25,15 +25,16 @@ import org.json4s.jackson.{JValueDeserializer, JValueSerializer}
import org.apache.spark.sql.types.DataType
object DataTypeJsonUtils {
+
/**
* Jackson serializer for [[DataType]]. Internally this delegates to json4s based serialization.
*/
class DataTypeJsonSerializer extends JsonSerializer[DataType] {
private val delegate = new JValueSerializer
override def serialize(
- value: DataType,
- gen: JsonGenerator,
- provider: SerializerProvider): Unit = {
+ value: DataType,
+ gen: JsonGenerator,
+ provider: SerializerProvider): Unit = {
delegate.serialize(value.jsonValue, gen, provider)
}
}
@@ -46,8 +47,8 @@ object DataTypeJsonUtils {
private val delegate = new JValueDeserializer(classOf[Any])
override def deserialize(
- jsonParser: JsonParser,
- deserializationContext: DeserializationContext): DataType = {
+ jsonParser: JsonParser,
+ deserializationContext: DeserializationContext): DataType = {
val json = delegate.deserialize(jsonParser, deserializationContext)
DataType.parseDataType(json.asInstanceOf[JValue])
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 34d19bb67b71a..5eada9a7be670 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -43,7 +43,8 @@ class Iso8601DateFormatter(
locale: Locale,
legacyFormat: LegacyDateFormats.LegacyDateFormat,
isParsing: Boolean)
- extends DateFormatter with DateTimeFormatterHelper {
+ extends DateFormatter
+ with DateTimeFormatterHelper {
@transient
private lazy val formatter = getOrCreateFormatter(pattern, locale, isParsing)
@@ -62,8 +63,7 @@ class Iso8601DateFormatter(
override def format(localDate: LocalDate): String = {
try {
localDate.format(formatter)
- } catch checkFormattedDiff(toJavaDate(localDateToDays(localDate)),
- (d: Date) => format(d))
+ } catch checkFormattedDiff(toJavaDate(localDateToDays(localDate)), (d: Date) => format(d))
}
override def format(days: Int): String = {
@@ -83,19 +83,22 @@ class Iso8601DateFormatter(
}
/**
- * The formatter for dates which doesn't require users to specify a pattern. While formatting,
- * it uses the default pattern [[DateFormatter.defaultPattern]]. In parsing, it follows the CAST
+ * The formatter for dates which doesn't require users to specify a pattern. While formatting, it
+ * uses the default pattern [[DateFormatter.defaultPattern]]. In parsing, it follows the CAST
* logic in conversion of strings to Catalyst's DateType.
*
- * @param locale The locale overrides the system locale and is used in formatting.
- * @param legacyFormat Defines the formatter used for legacy dates.
- * @param isParsing Whether the formatter is used for parsing (`true`) or for formatting (`false`).
+ * @param locale
+ * The locale overrides the system locale and is used in formatting.
+ * @param legacyFormat
+ * Defines the formatter used for legacy dates.
+ * @param isParsing
+ * Whether the formatter is used for parsing (`true`) or for formatting (`false`).
*/
class DefaultDateFormatter(
locale: Locale,
legacyFormat: LegacyDateFormats.LegacyDateFormat,
isParsing: Boolean)
- extends Iso8601DateFormatter(DateFormatter.defaultPattern, locale, legacyFormat, isParsing) {
+ extends Iso8601DateFormatter(DateFormatter.defaultPattern, locale, legacyFormat, isParsing) {
override def parse(s: String): Int = {
try {
@@ -125,11 +128,13 @@ trait LegacyDateFormatter extends DateFormatter {
* JVM time zone intentionally for compatibility with Spark 2.4 and earlier versions.
*
* Note: Using of the default JVM time zone makes the formatter compatible with the legacy
- * `SparkDateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default
- * JVM time zone too.
+ * `SparkDateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default JVM
+ * time zone too.
*
- * @param pattern `java.text.SimpleDateFormat` compatible pattern.
- * @param locale The locale overrides the system locale and is used in parsing/formatting.
+ * @param pattern
+ * `java.text.SimpleDateFormat` compatible pattern.
+ * @param locale
+ * The locale overrides the system locale and is used in parsing/formatting.
*/
class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
@transient
@@ -145,14 +150,16 @@ class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDat
* JVM time zone intentionally for compatibility with Spark 2.4 and earlier versions.
*
* Note: Using of the default JVM time zone makes the formatter compatible with the legacy
- * `SparkDateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default
- * JVM time zone too.
+ * `SparkDateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default JVM
+ * time zone too.
*
- * @param pattern The pattern describing the date and time format.
- * See
- * Date and Time Patterns
- * @param locale The locale whose date format symbols should be used. It overrides the system
- * locale in parsing/formatting.
+ * @param pattern
+ * The pattern describing the date and time format. See Date and
+ * Time Patterns
+ * @param locale
+ * The locale whose date format symbols should be used. It overrides the system locale in
+ * parsing/formatting.
*/
// scalastyle:on line.size.limit
class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 067e58893126c..71777906f868e 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -40,13 +40,18 @@ trait DateTimeFormatterHelper {
}
private def verifyLocalDate(
- accessor: TemporalAccessor, field: ChronoField, candidate: LocalDate): Unit = {
+ accessor: TemporalAccessor,
+ field: ChronoField,
+ candidate: LocalDate): Unit = {
if (accessor.isSupported(field)) {
val actual = accessor.get(field)
val expected = candidate.get(field)
if (actual != expected) {
throw ExecutionErrors.fieldDiffersFromDerivedLocalDateError(
- field, actual, expected, candidate)
+ field,
+ actual,
+ expected,
+ candidate)
}
}
}
@@ -133,7 +138,8 @@ trait DateTimeFormatterHelper {
// SparkUpgradeException. On the contrary, if the legacy policy set to CORRECTED,
// DateTimeParseException will address by the caller side.
protected def checkParsedDiff[T](
- s: String, legacyParseFunc: String => T): PartialFunction[Throwable, T] = {
+ s: String,
+ legacyParseFunc: String => T): PartialFunction[Throwable, T] = {
case e if needConvertToSparkUpgradeException(e) =>
try {
legacyParseFunc(s)
@@ -151,11 +157,12 @@ trait DateTimeFormatterHelper {
d: T,
legacyFormatFunc: T => String): PartialFunction[Throwable, String] = {
case e if needConvertToSparkUpgradeException(e) =>
- val resultCandidate = try {
- legacyFormatFunc(d)
- } catch {
- case _: Throwable => throw e
- }
+ val resultCandidate =
+ try {
+ legacyFormatFunc(d)
+ } catch {
+ case _: Throwable => throw e
+ }
throw ExecutionErrors.failToParseDateTimeInNewParserError(resultCandidate, e)
}
@@ -166,9 +173,11 @@ trait DateTimeFormatterHelper {
* policy or follow our guide to correct their pattern. Otherwise, the original
* IllegalArgumentException will be thrown.
*
- * @param pattern the date time pattern
- * @param tryLegacyFormatter a func to capture exception, identically which forces a legacy
- * datetime formatter to be initialized
+ * @param pattern
+ * the date time pattern
+ * @param tryLegacyFormatter
+ * a func to capture exception, identically which forces a legacy datetime formatter to be
+ * initialized
*/
protected def checkLegacyFormatter(
pattern: String,
@@ -214,8 +223,7 @@ private object DateTimeFormatterHelper {
/**
* Building a formatter for parsing seconds fraction with variable length
*/
- def createBuilderWithVarLengthSecondFraction(
- pattern: String): DateTimeFormatterBuilder = {
+ def createBuilderWithVarLengthSecondFraction(pattern: String): DateTimeFormatterBuilder = {
val builder = createBuilder()
pattern.split("'").zipWithIndex.foreach {
// Split string starting with the regex itself which is `'` here will produce an extra empty
@@ -229,12 +237,14 @@ private object DateTimeFormatterHelper {
case extractor(prefix, secondFraction, suffix) =>
builder.appendPattern(prefix)
if (secondFraction.nonEmpty) {
- builder.appendFraction(ChronoField.NANO_OF_SECOND, 1, secondFraction.length, false)
+ builder
+ .appendFraction(ChronoField.NANO_OF_SECOND, 1, secondFraction.length, false)
}
rest = suffix
- case _ => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_DATETIME_PATTERN",
- messageParameters = Map("pattern" -> pattern))
+ case _ =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_DATETIME_PATTERN.SECONDS_FRACTION",
+ messageParameters = Map("pattern" -> pattern))
}
}
case (patternPart, _) => builder.appendLiteral(patternPart)
@@ -258,8 +268,10 @@ private object DateTimeFormatterHelper {
val builder = createBuilder()
.append(DateTimeFormatter.ISO_LOCAL_DATE)
.appendLiteral(' ')
- .appendValue(ChronoField.HOUR_OF_DAY, 2).appendLiteral(':')
- .appendValue(ChronoField.MINUTE_OF_HOUR, 2).appendLiteral(':')
+ .appendValue(ChronoField.HOUR_OF_DAY, 2)
+ .appendLiteral(':')
+ .appendValue(ChronoField.MINUTE_OF_HOUR, 2)
+ .appendLiteral(':')
.appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
toFormatter(builder, TimestampFormatter.defaultLocale)
@@ -299,17 +311,21 @@ private object DateTimeFormatterHelper {
* parsing/formatting datetime values. The pattern string is incompatible with the one defined
* by SimpleDateFormat in Spark 2.4 and earlier. This function converts all incompatible pattern
* for the new parser in Spark 3.0. See more details in SPARK-31030.
- * @param pattern The input pattern.
- * @return The pattern for new parser
+ * @param pattern
+ * The input pattern.
+ * @return
+ * The pattern for new parser
*/
def convertIncompatiblePattern(pattern: String, isParsing: Boolean): String = {
- val eraDesignatorContained = pattern.split("'").zipWithIndex.exists {
- case (patternPart, index) =>
+ val eraDesignatorContained =
+ pattern.split("'").zipWithIndex.exists { case (patternPart, index) =>
// Text can be quoted using single quotes, we only check the non-quote parts.
index % 2 == 0 && patternPart.contains("G")
- }
- (pattern + " ").split("'").zipWithIndex.map {
- case (patternPart, index) =>
+ }
+ (pattern + " ")
+ .split("'")
+ .zipWithIndex
+ .map { case (patternPart, index) =>
if (index % 2 == 0) {
for (c <- patternPart if weekBasedLetters.contains(c)) {
throw new SparkIllegalArgumentException(
@@ -317,12 +333,10 @@ private object DateTimeFormatterHelper {
messageParameters = Map("c" -> c.toString))
}
for (c <- patternPart if unsupportedLetters.contains(c) ||
- (isParsing && unsupportedLettersForParsing.contains(c))) {
+ (isParsing && unsupportedLettersForParsing.contains(c))) {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_DATETIME_PATTERN.ILLEGAL_CHARACTER",
- messageParameters = Map(
- "c" -> c.toString,
- "pattern" -> pattern))
+ messageParameters = Map("c" -> c.toString, "pattern" -> pattern))
}
for (style <- unsupportedPatternLengths if patternPart.contains(style)) {
throw new SparkIllegalArgumentException(
@@ -340,6 +354,8 @@ private object DateTimeFormatterHelper {
} else {
patternPart
}
- }.mkString("'").stripSuffix(" ")
+ }
+ .mkString("'")
+ .stripSuffix(" ")
}
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/MathUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/MathUtils.scala
index 96c3fb81aa66f..b113bccc74dfb 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/MathUtils.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/MathUtils.scala
@@ -89,10 +89,7 @@ object MathUtils {
def floorMod(a: Long, b: Long): Long = withOverflow(Math.floorMod(a, b))
- def withOverflow[A](
- f: => A,
- hint: String = "",
- context: QueryContext = null): A = {
+ def withOverflow[A](f: => A, hint: String = "", context: QueryContext = null): A = {
try {
f
} catch {
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index f9566b0e1fb13..9c043320dc812 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -34,8 +34,8 @@ import org.apache.spark.util.SparkClassUtils
/**
* The collection of functions for rebasing days and microseconds from/to the hybrid calendar
- * (Julian + Gregorian since 1582-10-15) which is used by Spark 2.4 and earlier versions
- * to/from Proleptic Gregorian calendar which is used by Spark since version 3.0. See SPARK-26651.
+ * (Julian + Gregorian since 1582-10-15) which is used by Spark 2.4 and earlier versions to/from
+ * Proleptic Gregorian calendar which is used by Spark since version 3.0. See SPARK-26651.
*/
object RebaseDateTime {
@@ -46,20 +46,22 @@ object RebaseDateTime {
}
/**
- * Rebases days since the epoch from an original to an target calendar, for instance,
- * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
+ * Rebases days since the epoch from an original to an target calendar, for instance, from a
+ * hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
*
* It finds the latest switch day which is less than the given `days`, and adds the difference
- * in days associated with the switch days to the given `days`.
- * The function is based on linear search which starts from the most recent switch days.
- * This allows to perform less comparisons for modern dates.
+ * in days associated with the switch days to the given `days`. The function is based on linear
+ * search which starts from the most recent switch days. This allows to perform less comparisons
+ * for modern dates.
*
- * @param switches The days when difference in days between original and target calendar
- * was changed.
- * @param diffs The differences in days between calendars.
- * @param days The number of days since the epoch 1970-01-01 to be rebased
- * to the target calendar.
- * @return The rebased days.
+ * @param switches
+ * The days when difference in days between original and target calendar was changed.
+ * @param diffs
+ * The differences in days between calendars.
+ * @param days
+ * The number of days since the epoch 1970-01-01 to be rebased to the target calendar.
+ * @return
+ * The rebased days.
*/
private def rebaseDays(switches: Array[Int], diffs: Array[Int], days: Int): Int = {
var i = switches.length
@@ -77,9 +79,8 @@ object RebaseDateTime {
// Julian calendar). This array is not applicable for dates before the staring point.
// Rebasing switch days and diffs `julianGregDiffSwitchDay` and `julianGregDiffs`
// was generated by the `localRebaseJulianToGregorianDays` function.
- private val julianGregDiffSwitchDay = Array(
- -719164, -682945, -646420, -609895, -536845, -500320, -463795,
- -390745, -354220, -317695, -244645, -208120, -171595, -141427)
+ private val julianGregDiffSwitchDay = Array(-719164, -682945, -646420, -609895, -536845,
+ -500320, -463795, -390745, -354220, -317695, -244645, -208120, -171595, -141427)
final val lastSwitchJulianDay: Int = julianGregDiffSwitchDay.last
@@ -88,20 +89,20 @@ object RebaseDateTime {
/**
* Converts the given number of days since the epoch day 1970-01-01 to a local date in Julian
- * calendar, interprets the result as a local date in Proleptic Gregorian calendar, and takes the
- * number of days since the epoch from the Gregorian local date.
+ * calendar, interprets the result as a local date in Proleptic Gregorian calendar, and takes
+ * the number of days since the epoch from the Gregorian local date.
*
* This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
- * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
- * calendar. See SPARK-26651.
+ * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic
+ * Gregorian calendar. See SPARK-26651.
*
- * For example:
- * Julian calendar: 1582-01-01 -> -141704
- * Proleptic Gregorian calendar: 1582-01-01 -> -141714
- * The code below converts -141704 to -141714.
+ * For example: Julian calendar: 1582-01-01 -> -141704 Proleptic Gregorian calendar: 1582-01-01
+ * -> -141714 The code below converts -141704 to -141714.
*
- * @param days The number of days since the epoch in Julian calendar. It can be negative.
- * @return The rebased number of days in Gregorian calendar.
+ * @param days
+ * The number of days since the epoch in Julian calendar. It can be negative.
+ * @return
+ * The rebased number of days in Gregorian calendar.
*/
private[sql] def localRebaseJulianToGregorianDays(days: Int): Int = {
val utcCal = new Calendar.Builder()
@@ -111,14 +112,15 @@ object RebaseDateTime {
.setTimeZone(TimeZoneUTC)
.setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
.build()
- val localDate = LocalDate.of(
- utcCal.get(YEAR),
- utcCal.get(MONTH) + 1,
- // The number of days will be added later to handle non-existing
- // Julian dates in Proleptic Gregorian calendar.
- // For example, 1000-02-29 exists in Julian calendar because 1000
- // is a leap year but it is not a leap year in Gregorian calendar.
- 1)
+ val localDate = LocalDate
+ .of(
+ utcCal.get(YEAR),
+ utcCal.get(MONTH) + 1,
+ // The number of days will be added later to handle non-existing
+ // Julian dates in Proleptic Gregorian calendar.
+ // For example, 1000-02-29 exists in Julian calendar because 1000
+ // is a leap year but it is not a leap year in Gregorian calendar.
+ 1)
.`with`(ChronoField.ERA, utcCal.get(ERA))
.plusDays(utcCal.get(DAY_OF_MONTH) - 1)
Math.toIntExact(localDate.toEpochDay)
@@ -129,8 +131,10 @@ object RebaseDateTime {
* pre-calculated rebasing array to save calculation. For dates of Before Common Era, the
* function falls back to the regular unoptimized version.
*
- * @param days The number of days since the epoch in Julian calendar. It can be negative.
- * @return The rebased number of days in Gregorian calendar.
+ * @param days
+ * The number of days since the epoch in Julian calendar. It can be negative.
+ * @return
+ * The rebased number of days in Gregorian calendar.
*/
def rebaseJulianToGregorianDays(days: Int): Int = {
if (days < julianCommonEraStartDay) {
@@ -143,18 +147,17 @@ object RebaseDateTime {
// The differences in days between Proleptic Gregorian and Julian dates.
// The diff at the index `i` is applicable for all days in the date interval:
// [gregJulianDiffSwitchDay(i), gregJulianDiffSwitchDay(i+1))
- private val gregJulianDiffs = Array(
- -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+ private val gregJulianDiffs =
+ Array(-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
// The sorted days in Proleptic Gregorian calendar when difference in days between
// Proleptic Gregorian and Julian was changed.
// The starting point is the `0001-01-01` (-719162 days since the epoch in
// Proleptic Gregorian calendar). This array is not applicable for dates before the staring point.
// Rebasing switch days and diffs `gregJulianDiffSwitchDay` and `gregJulianDiffs`
// was generated by the `localRebaseGregorianToJulianDays` function.
- private val gregJulianDiffSwitchDay = Array(
- -719162, -682944, -646420, -609896, -536847, -500323, -463799, -390750,
- -354226, -317702, -244653, -208129, -171605, -141436, -141435, -141434,
- -141433, -141432, -141431, -141430, -141429, -141428, -141427)
+ private val gregJulianDiffSwitchDay = Array(-719162, -682944, -646420, -609896, -536847,
+ -500323, -463799, -390750, -354226, -317702, -244653, -208129, -171605, -141436, -141435,
+ -141434, -141433, -141432, -141431, -141430, -141429, -141428, -141427)
final val lastSwitchGregorianDay: Int = gregJulianDiffSwitchDay.last
@@ -171,17 +174,16 @@ object RebaseDateTime {
* number of days since the epoch from the Julian local date.
*
* This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
- * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
- * calendar. See SPARK-26651.
+ * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic
+ * Gregorian calendar. See SPARK-26651.
*
- * For example:
- * Proleptic Gregorian calendar: 1582-01-01 -> -141714
- * Julian calendar: 1582-01-01 -> -141704
- * The code below converts -141714 to -141704.
+ * For example: Proleptic Gregorian calendar: 1582-01-01 -> -141714 Julian calendar: 1582-01-01
+ * -> -141704 The code below converts -141714 to -141704.
*
- * @param days The number of days since the epoch in Proleptic Gregorian calendar.
- * It can be negative.
- * @return The rebased number of days in Julian calendar.
+ * @param days
+ * The number of days since the epoch in Proleptic Gregorian calendar. It can be negative.
+ * @return
+ * The rebased number of days in Julian calendar.
*/
private[sql] def localRebaseGregorianToJulianDays(days: Int): Int = {
var localDate = LocalDate.ofEpochDay(days)
@@ -204,8 +206,10 @@ object RebaseDateTime {
* pre-calculated rebasing array to save calculation. For dates of Before Common Era, the
* function falls back to the regular unoptimized version.
*
- * @param days The number of days since the epoch in Gregorian calendar. It can be negative.
- * @return The rebased number of days since the epoch in Julian calendar.
+ * @param days
+ * The number of days since the epoch in Gregorian calendar. It can be negative.
+ * @return
+ * The rebased number of days since the epoch in Julian calendar.
*/
def rebaseGregorianToJulianDays(days: Int): Int = {
if (days < gregorianCommonEraStartDay) {
@@ -215,10 +219,9 @@ object RebaseDateTime {
}
}
-
/**
- * The class describes JSON records with microseconds rebasing info.
- * Here is an example of JSON file:
+ * The class describes JSON records with microseconds rebasing info. Here is an example of JSON
+ * file:
* {{{
* [
* {
@@ -229,37 +232,44 @@ object RebaseDateTime {
* ]
* }}}
*
- * @param tz One of time zone ID which is expected to be acceptable by `ZoneId.of`.
- * @param switches An ordered array of seconds since the epoch when the diff between
- * two calendars are changed.
- * @param diffs Differences in seconds associated with elements of `switches`.
+ * @param tz
+ * One of time zone ID which is expected to be acceptable by `ZoneId.of`.
+ * @param switches
+ * An ordered array of seconds since the epoch when the diff between two calendars are
+ * changed.
+ * @param diffs
+ * Differences in seconds associated with elements of `switches`.
*/
private case class JsonRebaseRecord(tz: String, switches: Array[Long], diffs: Array[Long])
/**
* Rebasing info used to convert microseconds from an original to a target calendar.
*
- * @param switches An ordered array of microseconds since the epoch when the diff between
- * two calendars are changed.
- * @param diffs Differences in microseconds associated with elements of `switches`.
+ * @param switches
+ * An ordered array of microseconds since the epoch when the diff between two calendars are
+ * changed.
+ * @param diffs
+ * Differences in microseconds associated with elements of `switches`.
*/
private[sql] case class RebaseInfo(switches: Array[Long], diffs: Array[Long])
/**
- * Rebases micros since the epoch from an original to an target calendar, for instance,
- * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
+ * Rebases micros since the epoch from an original to an target calendar, for instance, from a
+ * hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
*
* It finds the latest switch micros which is less than the given `micros`, and adds the
- * difference in micros associated with the switch micros to the given `micros`.
- * The function is based on linear search which starts from the most recent switch micros.
- * This allows to perform less comparisons for modern timestamps.
+ * difference in micros associated with the switch micros to the given `micros`. The function is
+ * based on linear search which starts from the most recent switch micros. This allows to
+ * perform less comparisons for modern timestamps.
*
- * @param rebaseInfo The rebasing info contains an ordered micros when difference in micros
- * between original and target calendar was changed,
- * and differences in micros between calendars
- * @param micros The number of micros since the epoch 1970-01-01T00:00:00Z to be rebased
- * to the target calendar. It can be negative.
- * @return The rebased micros.
+ * @param rebaseInfo
+ * The rebasing info contains an ordered micros when difference in micros between original and
+ * target calendar was changed, and differences in micros between calendars
+ * @param micros
+ * The number of micros since the epoch 1970-01-01T00:00:00Z to be rebased to the target
+ * calendar. It can be negative.
+ * @return
+ * The rebased micros.
*/
private def rebaseMicros(rebaseInfo: RebaseInfo, micros: Long): Long = {
val switches = rebaseInfo.switches
@@ -296,18 +306,19 @@ object RebaseDateTime {
/**
* A map of time zone IDs to its ordered time points (instants in microseconds since the epoch)
- * when the difference between 2 instances associated with the same local timestamp in
- * Proleptic Gregorian and the hybrid calendar was changed, and to the diff at the index `i` is
- * applicable for all microseconds in the time interval:
- * [gregJulianDiffSwitchMicros(i), gregJulianDiffSwitchMicros(i+1))
+ * when the difference between 2 instances associated with the same local timestamp in Proleptic
+ * Gregorian and the hybrid calendar was changed, and to the diff at the index `i` is applicable
+ * for all microseconds in the time interval: [gregJulianDiffSwitchMicros(i),
+ * gregJulianDiffSwitchMicros(i+1))
*/
private val gregJulianRebaseMap = loadRebaseRecords("gregorian-julian-rebase-micros.json")
private def getLastSwitchTs(rebaseMap: AnyRefMap[String, RebaseInfo]): Long = {
val latestTs = rebaseMap.values.map(_.switches.last).max
- require(rebaseMap.values.forall(_.diffs.last == 0),
+ require(
+ rebaseMap.values.forall(_.diffs.last == 0),
s"Differences between Julian and Gregorian calendar after ${microsToInstant(latestTs)} " +
- "are expected to be zero for all available time zones.")
+ "are expected to be zero for all available time zones.")
latestTs
}
// The switch time point after which all diffs between Gregorian and Julian calendars
@@ -315,29 +326,30 @@ object RebaseDateTime {
final val lastSwitchGregorianTs: Long = getLastSwitchTs(gregJulianRebaseMap)
private final val gregorianStartTs = LocalDateTime.of(gregorianStartDate, LocalTime.MIDNIGHT)
- private final val julianEndTs = LocalDateTime.of(
- julianEndDate,
- LocalTime.of(23, 59, 59, 999999999))
+ private final val julianEndTs =
+ LocalDateTime.of(julianEndDate, LocalTime.of(23, 59, 59, 999999999))
/**
* Converts the given number of microseconds since the epoch '1970-01-01T00:00:00Z', to a local
- * date-time in Proleptic Gregorian calendar with timezone identified by `zoneId`, interprets the
- * result as a local date-time in Julian calendar with the same timezone, and takes microseconds
- * since the epoch from the Julian local date-time.
+ * date-time in Proleptic Gregorian calendar with timezone identified by `zoneId`, interprets
+ * the result as a local date-time in Julian calendar with the same timezone, and takes
+ * microseconds since the epoch from the Julian local date-time.
*
* This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
- * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
- * calendar. See SPARK-26651.
+ * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic
+ * Gregorian calendar. See SPARK-26651.
*
- * For example:
- * Proleptic Gregorian calendar: 1582-01-01 00:00:00.123456 -> -12244061221876544
- * Julian calendar: 1582-01-01 00:00:00.123456 -> -12243196799876544
- * The code below converts -12244061221876544 to -12243196799876544.
+ * For example: Proleptic Gregorian calendar: 1582-01-01 00:00:00.123456 -> -12244061221876544
+ * Julian calendar: 1582-01-01 00:00:00.123456 -> -12243196799876544 The code below converts
+ * -12244061221876544 to -12243196799876544.
*
- * @param tz The time zone at which the rebasing should be performed.
- * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
- * in Proleptic Gregorian calendar. It can be negative.
- * @return The rebased microseconds since the epoch in Julian calendar.
+ * @param tz
+ * The time zone at which the rebasing should be performed.
+ * @param micros
+ * The number of microseconds since the epoch '1970-01-01T00:00:00Z' in Proleptic Gregorian
+ * calendar. It can be negative.
+ * @return
+ * The rebased microseconds since the epoch in Julian calendar.
*/
private[sql] def rebaseGregorianToJulianMicros(tz: TimeZone, micros: Long): Long = {
val instant = microsToInstant(micros)
@@ -380,10 +392,13 @@ object RebaseDateTime {
* contain information about the given time zone `timeZoneId` or `micros` is related to Before
* Common Era, the function falls back to the regular unoptimized version.
*
- * @param timeZoneId A string identifier of a time zone.
- * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
- * in Proleptic Gregorian calendar. It can be negative.
- * @return The rebased microseconds since the epoch in Julian calendar.
+ * @param timeZoneId
+ * A string identifier of a time zone.
+ * @param micros
+ * The number of microseconds since the epoch '1970-01-01T00:00:00Z' in Proleptic Gregorian
+ * calendar. It can be negative.
+ * @return
+ * The rebased microseconds since the epoch in Julian calendar.
*/
def rebaseGregorianToJulianMicros(timeZoneId: String, micros: Long): Long = {
if (micros >= lastSwitchGregorianTs) {
@@ -404,12 +419,14 @@ object RebaseDateTime {
* contain information about the current JVM system time zone or `micros` is related to Before
* Common Era, the function falls back to the regular unoptimized version.
*
- * Note: The function assumes that the input micros was derived from a local timestamp
- * at the default system JVM time zone in Proleptic Gregorian calendar.
+ * Note: The function assumes that the input micros was derived from a local timestamp at the
+ * default system JVM time zone in Proleptic Gregorian calendar.
*
- * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
- * in Proleptic Gregorian calendar. It can be negative.
- * @return The rebased microseconds since the epoch in Julian calendar.
+ * @param micros
+ * The number of microseconds since the epoch '1970-01-01T00:00:00Z' in Proleptic Gregorian
+ * calendar. It can be negative.
+ * @return
+ * The rebased microseconds since the epoch in Julian calendar.
*/
def rebaseGregorianToJulianMicros(micros: Long): Long = {
rebaseGregorianToJulianMicros(TimeZone.getDefault.getID, micros)
@@ -418,22 +435,24 @@ object RebaseDateTime {
/**
* Converts the given number of microseconds since the epoch '1970-01-01T00:00:00Z', to a local
* date-time in Julian calendar with timezone identified by `zoneId`, interprets the result as a
- * local date-time in Proleptic Gregorian calendar with the same timezone, and takes microseconds
- * since the epoch from the Gregorian local date-time.
+ * local date-time in Proleptic Gregorian calendar with the same timezone, and takes
+ * microseconds since the epoch from the Gregorian local date-time.
*
* This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
- * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
- * calendar. See SPARK-26651.
+ * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic
+ * Gregorian calendar. See SPARK-26651.
*
- * For example:
- * Julian calendar: 1582-01-01 00:00:00.123456 -> -12243196799876544
- * Proleptic Gregorian calendar: 1582-01-01 00:00:00.123456 -> -12244061221876544
- * The code below converts -12243196799876544 to -12244061221876544.
+ * For example: Julian calendar: 1582-01-01 00:00:00.123456 -> -12243196799876544 Proleptic
+ * Gregorian calendar: 1582-01-01 00:00:00.123456 -> -12244061221876544 The code below converts
+ * -12243196799876544 to -12244061221876544.
*
- * @param tz The time zone at which the rebasing should be performed.
- * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
- * in the Julian calendar. It can be negative.
- * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
+ * @param tz
+ * The time zone at which the rebasing should be performed.
+ * @param micros
+ * The number of microseconds since the epoch '1970-01-01T00:00:00Z' in the Julian calendar.
+ * It can be negative.
+ * @return
+ * The rebased microseconds since the epoch in Proleptic Gregorian calendar.
*/
private[sql] def rebaseJulianToGregorianMicros(tz: TimeZone, micros: Long): Long = {
val cal = new Calendar.Builder()
@@ -442,18 +461,19 @@ object RebaseDateTime {
.setInstant(microsToMillis(micros))
.setTimeZone(tz)
.build()
- val localDateTime = LocalDateTime.of(
- cal.get(YEAR),
- cal.get(MONTH) + 1,
- // The number of days will be added later to handle non-existing
- // Julian dates in Proleptic Gregorian calendar.
- // For example, 1000-02-29 exists in Julian calendar because 1000
- // is a leap year but it is not a leap year in Gregorian calendar.
- 1,
- cal.get(HOUR_OF_DAY),
- cal.get(MINUTE),
- cal.get(SECOND),
- (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
+ val localDateTime = LocalDateTime
+ .of(
+ cal.get(YEAR),
+ cal.get(MONTH) + 1,
+ // The number of days will be added later to handle non-existing
+ // Julian dates in Proleptic Gregorian calendar.
+ // For example, 1000-02-29 exists in Julian calendar because 1000
+ // is a leap year but it is not a leap year in Gregorian calendar.
+ 1,
+ cal.get(HOUR_OF_DAY),
+ cal.get(MINUTE),
+ cal.get(SECOND),
+ (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
.`with`(ChronoField.ERA, cal.get(ERA))
.plusDays(cal.get(DAY_OF_MONTH) - 1)
val zoneId = tz.toZoneId
@@ -494,10 +514,13 @@ object RebaseDateTime {
* contain information about the given time zone `timeZoneId` or `micros` is related to Before
* Common Era, the function falls back to the regular unoptimized version.
*
- * @param timeZoneId A string identifier of a time zone.
- * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
- * in the Julian calendar. It can be negative.
- * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
+ * @param timeZoneId
+ * A string identifier of a time zone.
+ * @param micros
+ * The number of microseconds since the epoch '1970-01-01T00:00:00Z' in the Julian calendar.
+ * It can be negative.
+ * @return
+ * The rebased microseconds since the epoch in Proleptic Gregorian calendar.
*/
def rebaseJulianToGregorianMicros(timeZoneId: String, micros: Long): Long = {
if (micros >= lastSwitchJulianTs) {
@@ -518,12 +541,14 @@ object RebaseDateTime {
* contain information about the current JVM system time zone or `micros` is related to Before
* Common Era, the function falls back to the regular unoptimized version.
*
- * Note: The function assumes that the input micros was derived from a local timestamp
- * at the default system JVM time zone in the Julian calendar.
+ * Note: The function assumes that the input micros was derived from a local timestamp at the
+ * default system JVM time zone in the Julian calendar.
*
- * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
- * in the Julian calendar. It can be negative.
- * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
+ * @param micros
+ * The number of microseconds since the epoch '1970-01-01T00:00:00Z' in the Julian calendar.
+ * It can be negative.
+ * @return
+ * The rebased microseconds since the epoch in Proleptic Gregorian calendar.
*/
def rebaseJulianToGregorianMicros(micros: Long): Long = {
rebaseJulianToGregorianMicros(TimeZone.getDefault.getID, micros)
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkCharVarcharUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkCharVarcharUtils.scala
index 498eb83566eb3..2a26c079e8d4d 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkCharVarcharUtils.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkCharVarcharUtils.scala
@@ -21,15 +21,18 @@ import org.apache.spark.sql.internal.SqlApiConf
import org.apache.spark.sql.types.{ArrayType, CharType, DataType, MapType, StringType, StructType, VarcharType}
trait SparkCharVarcharUtils {
+
/**
- * Returns true if the given data type is CharType/VarcharType or has nested CharType/VarcharType.
+ * Returns true if the given data type is CharType/VarcharType or has nested
+ * CharType/VarcharType.
*/
def hasCharVarchar(dt: DataType): Boolean = {
dt.existsRecursively(f => f.isInstanceOf[CharType] || f.isInstanceOf[VarcharType])
}
/**
- * Validate the given [[DataType]] to fail if it is char or varchar types or contains nested ones
+ * Validate the given [[DataType]] to fail if it is char or varchar types or contains nested
+ * ones
*/
def failIfHasCharVarchar(dt: DataType): DataType = {
if (!SqlApiConf.get.charVarcharAsString && hasCharVarchar(dt)) {
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkDateTimeUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkDateTimeUtils.scala
index a6592ad51c65c..4e94bc6617357 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkDateTimeUtils.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkDateTimeUtils.scala
@@ -54,8 +54,10 @@ trait SparkDateTimeUtils {
/**
* Converts an Java object to days.
*
- * @param obj Either an object of `java.sql.Date` or `java.time.LocalDate`.
- * @return The number of days since 1970-01-01.
+ * @param obj
+ * Either an object of `java.sql.Date` or `java.time.LocalDate`.
+ * @return
+ * The number of days since 1970-01-01.
*/
def anyToDays(obj: Any): Int = obj match {
case d: Date => fromJavaDate(d)
@@ -65,8 +67,10 @@ trait SparkDateTimeUtils {
/**
* Converts an Java object to microseconds.
*
- * @param obj Either an object of `java.sql.Timestamp` or `java.time.{Instant,LocalDateTime}`.
- * @return The number of micros since the epoch.
+ * @param obj
+ * Either an object of `java.sql.Timestamp` or `java.time.{Instant,LocalDateTime}`.
+ * @return
+ * The number of micros since the epoch.
*/
def anyToMicros(obj: Any): Long = obj match {
case t: Timestamp => fromJavaTimestamp(t)
@@ -75,8 +79,8 @@ trait SparkDateTimeUtils {
}
/**
- * Converts the timestamp to milliseconds since epoch. In Spark timestamp values have microseconds
- * precision, so this conversion is lossy.
+ * Converts the timestamp to milliseconds since epoch. In Spark timestamp values have
+ * microseconds precision, so this conversion is lossy.
*/
def microsToMillis(micros: Long): Long = {
// When the timestamp is negative i.e before 1970, we need to adjust the milliseconds portion.
@@ -97,8 +101,8 @@ trait SparkDateTimeUtils {
private val MIN_SECONDS = Math.floorDiv(Long.MinValue, MICROS_PER_SECOND)
/**
- * Obtains an instance of `java.time.Instant` using microseconds from
- * the epoch of 1970-01-01 00:00:00Z.
+ * Obtains an instance of `java.time.Instant` using microseconds from the epoch of 1970-01-01
+ * 00:00:00Z.
*/
def microsToInstant(micros: Long): Instant = {
val secs = Math.floorDiv(micros, MICROS_PER_SECOND)
@@ -110,8 +114,8 @@ trait SparkDateTimeUtils {
/**
* Gets the number of microseconds since the epoch of 1970-01-01 00:00:00Z from the given
- * instance of `java.time.Instant`. The epoch microsecond count is a simple incrementing count of
- * microseconds where microsecond 0 is 1970-01-01 00:00:00Z.
+ * instance of `java.time.Instant`. The epoch microsecond count is a simple incrementing count
+ * of microseconds where microsecond 0 is 1970-01-01 00:00:00Z.
*/
def instantToMicros(instant: Instant): Long = {
val secs = instant.getEpochSecond
@@ -127,8 +131,8 @@ trait SparkDateTimeUtils {
/**
* Converts the timestamp `micros` from one timezone to another.
*
- * Time-zone rules, such as daylight savings, mean that not every local date-time
- * is valid for the `toZone` time zone, thus the local date-time may be adjusted.
+ * Time-zone rules, such as daylight savings, mean that not every local date-time is valid for
+ * the `toZone` time zone, thus the local date-time may be adjusted.
*/
def convertTz(micros: Long, fromZone: ZoneId, toZone: ZoneId): Long = {
val rebasedDateTime = getLocalDateTime(micros, toZone).atZone(fromZone)
@@ -160,14 +164,16 @@ trait SparkDateTimeUtils {
def daysToLocalDate(days: Int): LocalDate = LocalDate.ofEpochDay(days)
/**
- * Converts microseconds since 1970-01-01 00:00:00Z to days since 1970-01-01 at the given zone ID.
+ * Converts microseconds since 1970-01-01 00:00:00Z to days since 1970-01-01 at the given zone
+ * ID.
*/
def microsToDays(micros: Long, zoneId: ZoneId): Int = {
localDateToDays(getLocalDateTime(micros, zoneId).toLocalDate)
}
/**
- * Converts days since 1970-01-01 at the given zone ID to microseconds since 1970-01-01 00:00:00Z.
+ * Converts days since 1970-01-01 at the given zone ID to microseconds since 1970-01-01
+ * 00:00:00Z.
*/
def daysToMicros(days: Int, zoneId: ZoneId): Long = {
val instant = daysToLocalDate(days).atStartOfDay(zoneId).toInstant
@@ -175,20 +181,22 @@ trait SparkDateTimeUtils {
}
/**
- * Converts a local date at the default JVM time zone to the number of days since 1970-01-01
- * in the hybrid calendar (Julian + Gregorian) by discarding the time part. The resulted days are
+ * Converts a local date at the default JVM time zone to the number of days since 1970-01-01 in
+ * the hybrid calendar (Julian + Gregorian) by discarding the time part. The resulted days are
* rebased from the hybrid to Proleptic Gregorian calendar. The days rebasing is performed via
- * UTC time zone for simplicity because the difference between two calendars is the same in
- * any given time zone and UTC time zone.
+ * UTC time zone for simplicity because the difference between two calendars is the same in any
+ * given time zone and UTC time zone.
*
- * Note: The date is shifted by the offset of the default JVM time zone for backward compatibility
- * with Spark 2.4 and earlier versions. The goal of the shift is to get a local date derived
- * from the number of days that has the same date fields (year, month, day) as the original
- * `date` at the default JVM time zone.
+ * Note: The date is shifted by the offset of the default JVM time zone for backward
+ * compatibility with Spark 2.4 and earlier versions. The goal of the shift is to get a local
+ * date derived from the number of days that has the same date fields (year, month, day) as the
+ * original `date` at the default JVM time zone.
*
- * @param date It represents a specific instant in time based on the hybrid calendar which
- * combines Julian and Gregorian calendars.
- * @return The number of days since the epoch in Proleptic Gregorian calendar.
+ * @param date
+ * It represents a specific instant in time based on the hybrid calendar which combines Julian
+ * and Gregorian calendars.
+ * @return
+ * The number of days since the epoch in Proleptic Gregorian calendar.
*/
def fromJavaDate(date: Date): Int = {
val millisUtc = date.getTime
@@ -207,18 +215,20 @@ trait SparkDateTimeUtils {
}
/**
- * Converts days since the epoch 1970-01-01 in Proleptic Gregorian calendar to a local date
- * at the default JVM time zone in the hybrid calendar (Julian + Gregorian). It rebases the given
+ * Converts days since the epoch 1970-01-01 in Proleptic Gregorian calendar to a local date at
+ * the default JVM time zone in the hybrid calendar (Julian + Gregorian). It rebases the given
* days from Proleptic Gregorian to the hybrid calendar at UTC time zone for simplicity because
* the difference between two calendars doesn't depend on any time zone. The result is shifted
- * by the time zone offset in wall clock to have the same date fields (year, month, day)
- * at the default JVM time zone as the input `daysSinceEpoch` in Proleptic Gregorian calendar.
+ * by the time zone offset in wall clock to have the same date fields (year, month, day) at the
+ * default JVM time zone as the input `daysSinceEpoch` in Proleptic Gregorian calendar.
*
- * Note: The date is shifted by the offset of the default JVM time zone for backward compatibility
- * with Spark 2.4 and earlier versions.
+ * Note: The date is shifted by the offset of the default JVM time zone for backward
+ * compatibility with Spark 2.4 and earlier versions.
*
- * @param days The number of days since 1970-01-01 in Proleptic Gregorian calendar.
- * @return A local date in the hybrid calendar as `java.sql.Date` from number of days since epoch.
+ * @param days
+ * The number of days since 1970-01-01 in Proleptic Gregorian calendar.
+ * @return
+ * A local date in the hybrid calendar as `java.sql.Date` from number of days since epoch.
*/
def toJavaDate(days: Int): Date = {
val rebasedDays = rebaseGregorianToJulianDays(days)
@@ -233,20 +243,22 @@ trait SparkDateTimeUtils {
}
/**
- * Converts microseconds since the epoch to an instance of `java.sql.Timestamp`
- * via creating a local timestamp at the system time zone in Proleptic Gregorian
- * calendar, extracting date and time fields like `year` and `hours`, and forming
- * new timestamp in the hybrid calendar from the extracted fields.
+ * Converts microseconds since the epoch to an instance of `java.sql.Timestamp` via creating a
+ * local timestamp at the system time zone in Proleptic Gregorian calendar, extracting date and
+ * time fields like `year` and `hours`, and forming new timestamp in the hybrid calendar from
+ * the extracted fields.
*
- * The conversion is based on the JVM system time zone because the `java.sql.Timestamp`
- * uses the time zone internally.
+ * The conversion is based on the JVM system time zone because the `java.sql.Timestamp` uses the
+ * time zone internally.
*
* The method performs the conversion via local timestamp fields to have the same date-time
- * representation as `year`, `month`, `day`, ..., `seconds` in the original calendar
- * and in the target calendar.
+ * representation as `year`, `month`, `day`, ..., `seconds` in the original calendar and in the
+ * target calendar.
*
- * @param micros The number of microseconds since 1970-01-01T00:00:00.000000Z.
- * @return A `java.sql.Timestamp` from number of micros since epoch.
+ * @param micros
+ * The number of microseconds since 1970-01-01T00:00:00.000000Z.
+ * @return
+ * A `java.sql.Timestamp` from number of micros since epoch.
*/
def toJavaTimestamp(micros: Long): Timestamp =
toJavaTimestampNoRebase(rebaseGregorianToJulianMicros(micros))
@@ -257,8 +269,10 @@ trait SparkDateTimeUtils {
/**
* Converts microseconds since the epoch to an instance of `java.sql.Timestamp`.
*
- * @param micros The number of microseconds since 1970-01-01T00:00:00.000000Z.
- * @return A `java.sql.Timestamp` from number of micros since epoch.
+ * @param micros
+ * The number of microseconds since 1970-01-01T00:00:00.000000Z.
+ * @return
+ * A `java.sql.Timestamp` from number of micros since epoch.
*/
def toJavaTimestampNoRebase(micros: Long): Timestamp = {
val seconds = Math.floorDiv(micros, MICROS_PER_SECOND)
@@ -270,22 +284,22 @@ trait SparkDateTimeUtils {
/**
* Converts an instance of `java.sql.Timestamp` to the number of microseconds since
- * 1970-01-01T00:00:00.000000Z. It extracts date-time fields from the input, builds
- * a local timestamp in Proleptic Gregorian calendar from the fields, and binds
- * the timestamp to the system time zone. The resulted instant is converted to
- * microseconds since the epoch.
+ * 1970-01-01T00:00:00.000000Z. It extracts date-time fields from the input, builds a local
+ * timestamp in Proleptic Gregorian calendar from the fields, and binds the timestamp to the
+ * system time zone. The resulted instant is converted to microseconds since the epoch.
*
- * The conversion is performed via the system time zone because it is used internally
- * in `java.sql.Timestamp` while extracting date-time fields.
+ * The conversion is performed via the system time zone because it is used internally in
+ * `java.sql.Timestamp` while extracting date-time fields.
*
* The goal of the function is to have the same local date-time in the original calendar
- * - the hybrid calendar (Julian + Gregorian) and in the target calendar which is
- * Proleptic Gregorian calendar, see SPARK-26651.
+ * - the hybrid calendar (Julian + Gregorian) and in the target calendar which is Proleptic
+ * Gregorian calendar, see SPARK-26651.
*
- * @param t It represents a specific instant in time based on
- * the hybrid calendar which combines Julian and
- * Gregorian calendars.
- * @return The number of micros since epoch from `java.sql.Timestamp`.
+ * @param t
+ * It represents a specific instant in time based on the hybrid calendar which combines Julian
+ * and Gregorian calendars.
+ * @return
+ * The number of micros since epoch from `java.sql.Timestamp`.
*/
def fromJavaTimestamp(t: Timestamp): Long =
rebaseJulianToGregorianMicros(fromJavaTimestampNoRebase(t))
@@ -297,30 +311,27 @@ trait SparkDateTimeUtils {
* Converts an instance of `java.sql.Timestamp` to the number of microseconds since
* 1970-01-01T00:00:00.000000Z.
*
- * @param t an instance of `java.sql.Timestamp`.
- * @return The number of micros since epoch from `java.sql.Timestamp`.
+ * @param t
+ * an instance of `java.sql.Timestamp`.
+ * @return
+ * The number of micros since epoch from `java.sql.Timestamp`.
*/
def fromJavaTimestampNoRebase(t: Timestamp): Long =
millisToMicros(t.getTime) + (t.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS
/**
- * Trims and parses a given UTF8 date string to a corresponding [[Int]] value.
- * The return type is [[Option]] in order to distinguish between 0 and null. The following
- * formats are allowed:
+ * Trims and parses a given UTF8 date string to a corresponding [[Int]] value. The return type
+ * is [[Option]] in order to distinguish between 0 and null. The following formats are allowed:
*
- * `[+-]yyyy*`
- * `[+-]yyyy*-[m]m`
- * `[+-]yyyy*-[m]m-[d]d`
- * `[+-]yyyy*-[m]m-[d]d `
- * `[+-]yyyy*-[m]m-[d]d *`
- * `[+-]yyyy*-[m]m-[d]dT*`
+ * `[+-]yyyy*` `[+-]yyyy*-[m]m` `[+-]yyyy*-[m]m-[d]d` `[+-]yyyy*-[m]m-[d]d `
+ * `[+-]yyyy*-[m]m-[d]d *` `[+-]yyyy*-[m]m-[d]dT*`
*/
def stringToDate(s: UTF8String): Option[Int] = {
def isValidDigits(segment: Int, digits: Int): Boolean = {
// An integer is able to represent a date within [+-]5 million years.
val maxDigitsYear = 7
(segment == 0 && digits >= 4 && digits <= maxDigitsYear) ||
- (segment != 0 && digits > 0 && digits <= 2)
+ (segment != 0 && digits > 0 && digits <= 2)
}
if (s == null) {
return None
@@ -380,24 +391,18 @@ trait SparkDateTimeUtils {
}
}
- def stringToDateAnsi(
- s: UTF8String,
- context: QueryContext = null): Int = {
+ def stringToDateAnsi(s: UTF8String, context: QueryContext = null): Int = {
stringToDate(s).getOrElse {
throw ExecutionErrors.invalidInputInCastToDatetimeError(s, DateType, context)
}
}
/**
- * Trims and parses a given UTF8 timestamp string to the corresponding timestamp segments,
- * time zone id and whether it is just time without a date.
- * value. The return type is [[Option]] in order to distinguish between 0L and null. The following
- * formats are allowed:
+ * Trims and parses a given UTF8 timestamp string to the corresponding timestamp segments, time
+ * zone id and whether it is just time without a date. value. The return type is [[Option]] in
+ * order to distinguish between 0L and null. The following formats are allowed:
*
- * `[+-]yyyy*`
- * `[+-]yyyy*-[m]m`
- * `[+-]yyyy*-[m]m-[d]d`
- * `[+-]yyyy*-[m]m-[d]d `
+ * `[+-]yyyy*` `[+-]yyyy*-[m]m` `[+-]yyyy*-[m]m-[d]d` `[+-]yyyy*-[m]m-[d]d `
* `[+-]yyyy*-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
* `[+-]yyyy*-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
* `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
@@ -407,16 +412,17 @@ trait SparkDateTimeUtils {
* - Z - Zulu time zone UTC+0
* - +|-[h]h:[m]m
* - A short id, see https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html#SHORT_IDS
- * - An id with one of the prefixes UTC+, UTC-, GMT+, GMT-, UT+ or UT-,
- * and a suffix in the formats:
+ * - An id with one of the prefixes UTC+, UTC-, GMT+, GMT-, UT+ or UT-, and a suffix in the
+ * formats:
* - +|-h[h]
* - +|-hh[:]mm
* - +|-hh:mm:ss
* - +|-hhmmss
- * - Region-based zone IDs in the form `area/city`, such as `Europe/Paris`
+ * - Region-based zone IDs in the form `area/city`, such as `Europe/Paris`
*
- * @return timestamp segments, time zone id and whether the input is just time without a date. If
- * the input string can't be parsed as timestamp, the result timestamp segments are empty.
+ * @return
+ * timestamp segments, time zone id and whether the input is just time without a date. If the
+ * input string can't be parsed as timestamp, the result timestamp segments are empty.
*/
def parseTimestampString(s: UTF8String): (Array[Int], Option[ZoneId], Boolean) = {
def isValidDigits(segment: Int, digits: Int): Boolean = {
@@ -424,9 +430,9 @@ trait SparkDateTimeUtils {
val maxDigitsYear = 6
// For the nanosecond part, more than 6 digits is allowed, but will be truncated.
segment == 6 || (segment == 0 && digits >= 4 && digits <= maxDigitsYear) ||
- // For the zoneId segment(7), it's could be zero digits when it's a region-based zone ID
- (segment == 7 && digits <= 2) ||
- (segment != 0 && segment != 6 && segment != 7 && digits > 0 && digits <= 2)
+ // For the zoneId segment(7), it's could be zero digits when it's a region-based zone ID
+ (segment == 7 && digits <= 2) ||
+ (segment != 0 && segment != 6 && segment != 7 && digits > 0 && digits <= 2)
}
if (s == null) {
return (Array.empty, None, false)
@@ -523,7 +529,7 @@ trait SparkDateTimeUtils {
tz = Some(new String(bytes, j, strEndTrimmed - j))
j = strEndTrimmed - 1
}
- if (i == 6 && b != '.') {
+ if (i == 6 && b != '.') {
i += 1
}
} else {
@@ -612,11 +618,11 @@ trait SparkDateTimeUtils {
*
* If the input string contains a component associated with time zone, the method will return
* `None` if `allowTimeZone` is set to `false`. If `allowTimeZone` is set to `true`, the method
- * will simply discard the time zone component. Enable the check to detect situations like parsing
- * a timestamp with time zone as TimestampNTZType.
+ * will simply discard the time zone component. Enable the check to detect situations like
+ * parsing a timestamp with time zone as TimestampNTZType.
*
- * The return type is [[Option]] in order to distinguish between 0L and null. Please
- * refer to `parseTimestampString` for the allowed formats.
+ * The return type is [[Option]] in order to distinguish between 0L and null. Please refer to
+ * `parseTimestampString` for the allowed formats.
*/
def stringToTimestampWithoutTimeZone(s: UTF8String, allowTimeZone: Boolean): Option[Long] = {
try {
@@ -637,10 +643,13 @@ trait SparkDateTimeUtils {
}
/**
- * Returns the index of the first non-whitespace and non-ISO control character in the byte array.
+ * Returns the index of the first non-whitespace and non-ISO control character in the byte
+ * array.
*
- * @param bytes The byte array to be processed.
- * @return The start index after trimming.
+ * @param bytes
+ * The byte array to be processed.
+ * @return
+ * The start index after trimming.
*/
@inline private def getTrimmedStart(bytes: Array[Byte]) = {
var start = 0
@@ -655,9 +664,12 @@ trait SparkDateTimeUtils {
/**
* Returns the index of the last non-whitespace and non-ISO control character in the byte array.
*
- * @param start The starting index for the search.
- * @param bytes The byte array to be processed.
- * @return The end index after trimming.
+ * @param start
+ * The starting index for the search.
+ * @param bytes
+ * The byte array to be processed.
+ * @return
+ * The end index after trimming.
*/
@inline private def getTrimmedEnd(start: Int, bytes: Array[Byte]) = {
var end = bytes.length - 1
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkIntervalUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkIntervalUtils.scala
index 5e236187a4a0f..b8387b78ae3e2 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkIntervalUtils.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkIntervalUtils.scala
@@ -38,17 +38,16 @@ trait SparkIntervalUtils {
private final val minDurationSeconds = Math.floorDiv(Long.MinValue, MICROS_PER_SECOND)
/**
- * Converts this duration to the total length in microseconds.
- *
- * If this duration is too large to fit in a [[Long]] microseconds, then an
- * exception is thrown.
- *
- * If this duration has greater than microsecond precision, then the conversion
- * will drop any excess precision information as though the amount in nanoseconds
- * was subject to integer division by one thousand.
+ * Converts this duration to the total length in microseconds.
If this duration is too large
+ * to fit in a [[Long]] microseconds, then an exception is thrown.
If this duration has
+ * greater than microsecond precision, then the conversion will drop any excess precision
+ * information as though the amount in nanoseconds was subject to integer division by one
+ * thousand.
*
- * @return The total length of the duration in microseconds
- * @throws ArithmeticException If numeric overflow occurs
+ * @return
+ * The total length of the duration in microseconds
+ * @throws ArithmeticException
+ * If numeric overflow occurs
*/
def durationToMicros(duration: Duration): Long = {
durationToMicros(duration, DT.SECOND)
@@ -59,7 +58,8 @@ trait SparkIntervalUtils {
val micros = if (seconds == minDurationSeconds) {
val microsInSeconds = (minDurationSeconds + 1) * MICROS_PER_SECOND
val nanoAdjustment = duration.getNano
- assert(0 <= nanoAdjustment && nanoAdjustment < NANOS_PER_SECOND,
+ assert(
+ 0 <= nanoAdjustment && nanoAdjustment < NANOS_PER_SECOND,
"Duration.getNano() must return the adjustment to the seconds field " +
"in the range from 0 to 999999999 nanoseconds, inclusive.")
Math.addExact(microsInSeconds, (nanoAdjustment - NANOS_PER_SECOND) / NANOS_PER_MICROS)
@@ -77,14 +77,13 @@ trait SparkIntervalUtils {
}
/**
- * Gets the total number of months in this period.
- *
- * This returns the total number of months in the period by multiplying the
- * number of years by 12 and adding the number of months.
- *
+ * Gets the total number of months in this period.
This returns the total number of months
+ * in the period by multiplying the number of years by 12 and adding the number of months.
*
- * @return The total number of months in the period, may be negative
- * @throws ArithmeticException If numeric overflow occurs
+ * @return
+ * The total number of months in the period, may be negative
+ * @throws ArithmeticException
+ * If numeric overflow occurs
*/
def periodToMonths(period: Period): Int = {
periodToMonths(period, YM.MONTH)
@@ -103,39 +102,41 @@ trait SparkIntervalUtils {
/**
* Obtains a [[Duration]] representing a number of microseconds.
*
- * @param micros The number of microseconds, positive or negative
- * @return A [[Duration]], not null
+ * @param micros
+ * The number of microseconds, positive or negative
+ * @return
+ * A [[Duration]], not null
*/
def microsToDuration(micros: Long): Duration = Duration.of(micros, ChronoUnit.MICROS)
/**
- * Obtains a [[Period]] representing a number of months. The days unit will be zero, and the years
- * and months units will be normalized.
+ * Obtains a [[Period]] representing a number of months. The days unit will be zero, and the
+ * years and months units will be normalized.
*
- *
- * The months unit is adjusted to have an absolute value < 12, with the years unit being adjusted
- * to compensate. For example, the method returns "2 years and 3 months" for the 27 input months.
- *
- * The sign of the years and months units will be the same after normalization.
- * For example, -13 months will be converted to "-1 year and -1 month".
+ *
The months unit is adjusted to have an absolute value < 12, with the years unit being
+ * adjusted to compensate. For example, the method returns "2 years and 3 months" for the 27
+ * input months.
The sign of the years and months units will be the same after
+ * normalization. For example, -13 months will be converted to "-1 year and -1 month".
*
- * @param months The number of months, positive or negative
- * @return The period of months, not null
+ * @param months
+ * The number of months, positive or negative
+ * @return
+ * The period of months, not null
*/
def monthsToPeriod(months: Int): Period = Period.ofMonths(months).normalized()
/**
* Converts a string to [[CalendarInterval]] case-insensitively.
*
- * @throws IllegalArgumentException if the input string is not in valid interval format.
+ * @throws IllegalArgumentException
+ * if the input string is not in valid interval format.
*/
def stringToInterval(input: UTF8String): CalendarInterval = {
import ParseState._
if (input == null) {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.INPUT_IS_NULL",
- messageParameters = Map(
- "input" -> "null"))
+ messageParameters = Map("input" -> "null"))
}
// scalastyle:off caselocale .toLowerCase
val s = input.trimAll().toLowerCase
@@ -144,8 +145,7 @@ trait SparkIntervalUtils {
if (bytes.isEmpty) {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.INPUT_IS_EMPTY",
- messageParameters = Map(
- "input" -> input.toString))
+ messageParameters = Map("input" -> input.toString))
}
var state = PREFIX
var i = 0
@@ -182,14 +182,11 @@ trait SparkIntervalUtils {
if (s.numBytes() == intervalStr.numBytes()) {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.INPUT_IS_EMPTY",
- messageParameters = Map(
- "input" -> input.toString))
+ messageParameters = Map("input" -> input.toString))
} else if (!Character.isWhitespace(bytes(i + intervalStr.numBytes()))) {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.INVALID_PREFIX",
- messageParameters = Map(
- "input" -> input.toString,
- "prefix" -> currentWord))
+ messageParameters = Map("input" -> input.toString, "prefix" -> currentWord))
} else {
i += intervalStr.numBytes() + 1
}
@@ -224,11 +221,10 @@ trait SparkIntervalUtils {
pointPrefixed = true
i += 1
state = VALUE_FRACTIONAL_PART
- case _ => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_INTERVAL_FORMAT.UNRECOGNIZED_NUMBER",
- messageParameters = Map(
- "input" -> input.toString,
- "number" -> currentWord))
+ case _ =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_INTERVAL_FORMAT.UNRECOGNIZED_NUMBER",
+ messageParameters = Map("input" -> input.toString, "number" -> currentWord))
}
case TRIM_BEFORE_VALUE => trimToNextState(b, VALUE)
case VALUE =>
@@ -237,20 +233,19 @@ trait SparkIntervalUtils {
try {
currentValue = Math.addExact(Math.multiplyExact(10, currentValue), (b - '0'))
} catch {
- case e: ArithmeticException => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_INTERVAL_FORMAT.ARITHMETIC_EXCEPTION",
- messageParameters = Map(
- "input" -> input.toString))
+ case e: ArithmeticException =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_INTERVAL_FORMAT.ARITHMETIC_EXCEPTION",
+ messageParameters = Map("input" -> input.toString))
}
case _ if Character.isWhitespace(b) => state = TRIM_BEFORE_UNIT
case '.' =>
fractionScale = initialFractionScale
state = VALUE_FRACTIONAL_PART
- case _ => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_INTERVAL_FORMAT.INVALID_VALUE",
- messageParameters = Map(
- "input" -> input.toString,
- "value" -> currentWord))
+ case _ =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_INTERVAL_FORMAT.INVALID_VALUE",
+ messageParameters = Map("input" -> input.toString, "value" -> currentWord))
}
i += 1
case VALUE_FRACTIONAL_PART =>
@@ -264,15 +259,11 @@ trait SparkIntervalUtils {
} else if ('0' <= b && b <= '9') {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.INVALID_PRECISION",
- messageParameters = Map(
- "input" -> input.toString,
- "value" -> currentWord))
+ messageParameters = Map("input" -> input.toString, "value" -> currentWord))
} else {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.INVALID_VALUE",
- messageParameters = Map(
- "input" -> input.toString,
- "value" -> currentWord))
+ messageParameters = Map("input" -> input.toString, "value" -> currentWord))
}
i += 1
case TRIM_BEFORE_UNIT => trimToNextState(b, UNIT_BEGIN)
@@ -281,9 +272,7 @@ trait SparkIntervalUtils {
if (b != 's' && fractionScale >= 0) {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.INVALID_FRACTION",
- messageParameters = Map(
- "input" -> input.toString,
- "unit" -> currentWord))
+ messageParameters = Map("input" -> input.toString, "unit" -> currentWord))
}
if (isNegative) {
currentValue = -currentValue
@@ -328,44 +317,38 @@ trait SparkIntervalUtils {
} else {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.INVALID_UNIT",
- messageParameters = Map(
- "input" -> input.toString,
- "unit" -> currentWord))
+ messageParameters = Map("input" -> input.toString, "unit" -> currentWord))
}
- case _ => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_INTERVAL_FORMAT.INVALID_UNIT",
- messageParameters = Map(
- "input" -> input.toString,
- "unit" -> currentWord))
+ case _ =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_INTERVAL_FORMAT.INVALID_UNIT",
+ messageParameters = Map("input" -> input.toString, "unit" -> currentWord))
}
} catch {
- case e: ArithmeticException => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_INTERVAL_FORMAT.ARITHMETIC_EXCEPTION",
- messageParameters = Map(
- "input" -> input.toString))
+ case e: ArithmeticException =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_INTERVAL_FORMAT.ARITHMETIC_EXCEPTION",
+ messageParameters = Map("input" -> input.toString))
}
state = UNIT_SUFFIX
case UNIT_SUFFIX =>
b match {
case 's' => state = UNIT_END
case _ if Character.isWhitespace(b) => state = TRIM_BEFORE_SIGN
- case _ => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_INTERVAL_FORMAT.INVALID_UNIT",
- messageParameters = Map(
- "input" -> input.toString,
- "unit" -> currentWord))
+ case _ =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_INTERVAL_FORMAT.INVALID_UNIT",
+ messageParameters = Map("input" -> input.toString, "unit" -> currentWord))
}
i += 1
case UNIT_END =>
- if (Character.isWhitespace(b) ) {
+ if (Character.isWhitespace(b)) {
i += 1
state = TRIM_BEFORE_SIGN
} else {
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.INVALID_UNIT",
- messageParameters = Map(
- "input" -> input.toString,
- "unit" -> currentWord))
+ messageParameters = Map("input" -> input.toString, "unit" -> currentWord))
}
}
}
@@ -373,36 +356,37 @@ trait SparkIntervalUtils {
val result = state match {
case UNIT_SUFFIX | UNIT_END | TRIM_BEFORE_SIGN =>
new CalendarInterval(months, days, microseconds)
- case TRIM_BEFORE_VALUE => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_INTERVAL_FORMAT.MISSING_NUMBER",
- messageParameters = Map(
- "input" -> input.toString,
- "word" -> currentWord))
+ case TRIM_BEFORE_VALUE =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_INTERVAL_FORMAT.MISSING_NUMBER",
+ messageParameters = Map("input" -> input.toString, "word" -> currentWord))
case VALUE | VALUE_FRACTIONAL_PART =>
throw new SparkIllegalArgumentException(
errorClass = "INVALID_INTERVAL_FORMAT.MISSING_UNIT",
- messageParameters = Map(
- "input" -> input.toString,
- "word" -> currentWord))
- case _ => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_INTERVAL_FORMAT.UNKNOWN_PARSING_ERROR",
- messageParameters = Map(
- "input" -> input.toString,
- "word" -> currentWord))
+ messageParameters = Map("input" -> input.toString, "word" -> currentWord))
+ case _ =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_INTERVAL_FORMAT.UNKNOWN_PARSING_ERROR",
+ messageParameters = Map("input" -> input.toString, "word" -> currentWord))
}
result
}
/**
- * Converts an year-month interval as a number of months to its textual representation
- * which conforms to the ANSI SQL standard.
+ * Converts an year-month interval as a number of months to its textual representation which
+ * conforms to the ANSI SQL standard.
*
- * @param months The number of months, positive or negative
- * @param style The style of textual representation of the interval
- * @param startField The start field (YEAR or MONTH) which the interval comprises of.
- * @param endField The end field (YEAR or MONTH) which the interval comprises of.
- * @return Year-month interval string
+ * @param months
+ * The number of months, positive or negative
+ * @param style
+ * The style of textual representation of the interval
+ * @param startField
+ * The start field (YEAR or MONTH) which the interval comprises of.
+ * @param endField
+ * The end field (YEAR or MONTH) which the interval comprises of.
+ * @return
+ * Year-month interval string
*/
def toYearMonthIntervalString(
months: Int,
@@ -434,14 +418,19 @@ trait SparkIntervalUtils {
}
/**
- * Converts a day-time interval as a number of microseconds to its textual representation
- * which conforms to the ANSI SQL standard.
+ * Converts a day-time interval as a number of microseconds to its textual representation which
+ * conforms to the ANSI SQL standard.
*
- * @param micros The number of microseconds, positive or negative
- * @param style The style of textual representation of the interval
- * @param startField The start field (DAY, HOUR, MINUTE, SECOND) which the interval comprises of.
- * @param endField The end field (DAY, HOUR, MINUTE, SECOND) which the interval comprises of.
- * @return Day-time interval string
+ * @param micros
+ * The number of microseconds, positive or negative
+ * @param style
+ * The style of textual representation of the interval
+ * @param startField
+ * The start field (DAY, HOUR, MINUTE, SECOND) which the interval comprises of.
+ * @param endField
+ * The end field (DAY, HOUR, MINUTE, SECOND) which the interval comprises of.
+ * @return
+ * Day-time interval string
*/
def toDayTimeIntervalString(
micros: Long,
@@ -514,8 +503,9 @@ trait SparkIntervalUtils {
rest %= MICROS_PER_MINUTE
case DT.SECOND =>
val leadZero = if (rest < 10 * MICROS_PER_SECOND) "0" else ""
- formatBuilder.append(s"$leadZero" +
- s"${java.math.BigDecimal.valueOf(rest, 6).stripTrailingZeros.toPlainString}")
+ formatBuilder.append(
+ s"$leadZero" +
+ s"${java.math.BigDecimal.valueOf(rest, 6).stripTrailingZeros.toPlainString}")
}
if (startField < DT.HOUR && DT.HOUR <= endField) {
@@ -565,20 +555,11 @@ trait SparkIntervalUtils {
protected val microsStr: UTF8String = unitToUtf8("microsecond")
protected val nanosStr: UTF8String = unitToUtf8("nanosecond")
-
private object ParseState extends Enumeration {
type ParseState = Value
- val PREFIX,
- TRIM_BEFORE_SIGN,
- SIGN,
- TRIM_BEFORE_VALUE,
- VALUE,
- VALUE_FRACTIONAL_PART,
- TRIM_BEFORE_UNIT,
- UNIT_BEGIN,
- UNIT_SUFFIX,
- UNIT_END = Value
+ val PREFIX, TRIM_BEFORE_SIGN, SIGN, TRIM_BEFORE_VALUE, VALUE, VALUE_FRACTIONAL_PART,
+ TRIM_BEFORE_UNIT, UNIT_BEGIN, UNIT_SUFFIX, UNIT_END = Value
}
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkParserUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkParserUtils.scala
index 7597cb1d9087d..01ee899085701 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkParserUtils.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkParserUtils.scala
@@ -62,8 +62,8 @@ trait SparkParserUtils {
val secondChar = s.charAt(start + 1)
val thirdChar = s.charAt(start + 2)
(firstChar == '0' || firstChar == '1') &&
- (secondChar >= '0' && secondChar <= '7') &&
- (thirdChar >= '0' && thirdChar <= '7')
+ (secondChar >= '0' && secondChar <= '7') &&
+ (thirdChar >= '0' && thirdChar <= '7')
}
val isRawString = {
@@ -97,15 +97,18 @@ trait SparkParserUtils {
// \u0000 style 16-bit unicode character literals.
sb.append(Integer.parseInt(b, i + 1, i + 1 + 4, 16).toChar)
i += 1 + 4
- } else if (cAfterBackslash == 'U' && i + 1 + 8 <= length && allCharsAreHex(b, i + 1, 8)) {
+ } else if (cAfterBackslash == 'U' && i + 1 + 8 <= length && allCharsAreHex(
+ b,
+ i + 1,
+ 8)) {
// \U00000000 style 32-bit unicode character literals.
// Use Long to treat codePoint as unsigned in the range of 32-bit.
val codePoint = JLong.parseLong(b, i + 1, i + 1 + 8, 16)
if (codePoint < 0x10000) {
- sb.append((codePoint & 0xFFFF).toChar)
+ sb.append((codePoint & 0xffff).toChar)
} else {
- val highSurrogate = (codePoint - 0x10000) / 0x400 + 0xD800
- val lowSurrogate = (codePoint - 0x10000) % 0x400 + 0xDC00
+ val highSurrogate = (codePoint - 0x10000) / 0x400 + 0xd800
+ val lowSurrogate = (codePoint - 0x10000) % 0x400 + 0xdc00
sb.append(highSurrogate.toChar)
sb.append(lowSurrogate.toChar)
}
@@ -147,8 +150,13 @@ trait SparkParserUtils {
if (text.isEmpty) {
CurrentOrigin.set(position(ctx.getStart))
} else {
- CurrentOrigin.set(positionAndText(ctx.getStart, ctx.getStop, text.get,
- current.objectType, current.objectName))
+ CurrentOrigin.set(
+ positionAndText(
+ ctx.getStart,
+ ctx.getStop,
+ text.get,
+ current.objectType,
+ current.objectName))
}
try {
f
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
index edb1ee371b156..0608322be13b3 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
@@ -24,9 +24,8 @@ import org.apache.spark.unsafe.array.ByteArrayUtils
import org.apache.spark.util.ArrayImplicits._
/**
- * Concatenation of sequence of strings to final string with cheap append method
- * and one memory allocation for the final string. Can also bound the final size of
- * the string.
+ * Concatenation of sequence of strings to final string with cheap append method and one memory
+ * allocation for the final string. Can also bound the final size of the string.
*/
class StringConcat(val maxLength: Int = ByteArrayUtils.MAX_ROUNDED_ARRAY_LENGTH) {
protected val strings = new java.util.ArrayList[String]
@@ -35,9 +34,9 @@ class StringConcat(val maxLength: Int = ByteArrayUtils.MAX_ROUNDED_ARRAY_LENGTH)
def atLimit: Boolean = length >= maxLength
/**
- * Appends a string and accumulates its length to allocate a string buffer for all
- * appended strings once in the toString method. Returns true if the string still
- * has room for further appends before it hits its max limit.
+ * Appends a string and accumulates its length to allocate a string buffer for all appended
+ * strings once in the toString method. Returns true if the string still has room for further
+ * appends before it hits its max limit.
*/
def append(s: String): Unit = {
if (s != null) {
@@ -56,8 +55,8 @@ class StringConcat(val maxLength: Int = ByteArrayUtils.MAX_ROUNDED_ARRAY_LENGTH)
}
/**
- * The method allocates memory for all appended strings, writes them to the memory and
- * returns concatenated string.
+ * The method allocates memory for all appended strings, writes them to the memory and returns
+ * concatenated string.
*/
override def toString: String = {
val finalLength = if (atLimit) maxLength else length
@@ -68,6 +67,7 @@ class StringConcat(val maxLength: Int = ByteArrayUtils.MAX_ROUNDED_ARRAY_LENGTH)
}
object SparkStringUtils extends Logging {
+
/** Whether we have warned about plan string truncation yet. */
private val truncationWarningPrinted = new AtomicBoolean(false)
@@ -75,7 +75,8 @@ object SparkStringUtils extends Logging {
* Format a sequence with semantics similar to calling .mkString(). Any elements beyond
* maxNumToStringFields will be dropped and replaced by a "... N more fields" placeholder.
*
- * @return the trimmed and formatted string.
+ * @return
+ * the trimmed and formatted string.
*/
def truncatedString[T](
seq: Seq[T],
@@ -90,8 +91,9 @@ object SparkStringUtils extends Logging {
s"behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.")
}
val numFields = math.max(0, maxFields - 1)
- seq.take(numFields).mkString(
- start, sep, sep + "... " + (seq.length - numFields) + " more fields" + end)
+ seq
+ .take(numFields)
+ .mkString(start, sep, sep + "... " + (seq.length - numFields) + " more fields" + end)
} else {
seq.mkString(start, sep, end)
}
@@ -106,8 +108,8 @@ object SparkStringUtils extends Logging {
HexFormat.of().withDelimiter(" ").withUpperCase()
/**
- * Returns a pretty string of the byte array which prints each byte as a hex digit and add spaces
- * between them. For example, [1A C0].
+ * Returns a pretty string of the byte array which prints each byte as a hex digit and add
+ * spaces between them. For example, [1A C0].
*/
def getHexString(bytes: Array[Byte]): String = {
s"[${SPACE_DELIMITED_UPPERCASE_HEX.formatHex(bytes)}]"
@@ -122,8 +124,8 @@ object SparkStringUtils extends Logging {
val leftPadded = left ++ Seq.fill(math.max(right.size - left.size, 0))("")
val rightPadded = right ++ Seq.fill(math.max(left.size - right.size, 0))("")
- leftPadded.zip(rightPadded).map {
- case (l, r) => (if (l == r) " " else "!") + l + (" " * ((maxLeftSize - l.length) + 3)) + r
+ leftPadded.zip(rightPadded).map { case (l, r) =>
+ (if (l == r) " " else "!") + l + (" " * ((maxLeftSize - l.length) + 3)) + r
}
}
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 79d627b493fd8..4fcb84daf187d 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -41,14 +41,20 @@ import org.apache.spark.sql.types.{Decimal, TimestampNTZType}
import org.apache.spark.unsafe.types.UTF8String
sealed trait TimestampFormatter extends Serializable {
+
/**
* Parses a timestamp in a string and converts it to microseconds.
*
- * @param s - string with timestamp to parse
- * @return microseconds since epoch.
- * @throws ParseException can be thrown by legacy parser
- * @throws DateTimeParseException can be thrown by new parser
- * @throws DateTimeException unable to obtain local date or time
+ * @param s
+ * \- string with timestamp to parse
+ * @return
+ * microseconds since epoch.
+ * @throws ParseException
+ * can be thrown by legacy parser
+ * @throws DateTimeParseException
+ * can be thrown by new parser
+ * @throws DateTimeException
+ * unable to obtain local date or time
*/
@throws(classOf[ParseException])
@throws(classOf[DateTimeParseException])
@@ -58,11 +64,16 @@ sealed trait TimestampFormatter extends Serializable {
/**
* Parses a timestamp in a string and converts it to an optional number of microseconds.
*
- * @param s - string with timestamp to parse
- * @return An optional number of microseconds since epoch. The result is None on invalid input.
- * @throws ParseException can be thrown by legacy parser
- * @throws DateTimeParseException can be thrown by new parser
- * @throws DateTimeException unable to obtain local date or time
+ * @param s
+ * \- string with timestamp to parse
+ * @return
+ * An optional number of microseconds since epoch. The result is None on invalid input.
+ * @throws ParseException
+ * can be thrown by legacy parser
+ * @throws DateTimeParseException
+ * can be thrown by new parser
+ * @throws DateTimeException
+ * unable to obtain local date or time
*/
@throws(classOf[ParseException])
@throws(classOf[DateTimeParseException])
@@ -75,16 +86,24 @@ sealed trait TimestampFormatter extends Serializable {
}
/**
- * Parses a timestamp in a string and converts it to microseconds since Unix Epoch in local time.
+ * Parses a timestamp in a string and converts it to microseconds since Unix Epoch in local
+ * time.
*
- * @param s - string with timestamp to parse
- * @param allowTimeZone - indicates strict parsing of timezone
- * @return microseconds since epoch.
- * @throws ParseException can be thrown by legacy parser
- * @throws DateTimeParseException can be thrown by new parser
- * @throws DateTimeException unable to obtain local date or time
- * @throws IllegalStateException The formatter for timestamp without time zone should always
- * implement this method. The exception should never be hit.
+ * @param s
+ * \- string with timestamp to parse
+ * @param allowTimeZone
+ * \- indicates strict parsing of timezone
+ * @return
+ * microseconds since epoch.
+ * @throws ParseException
+ * can be thrown by legacy parser
+ * @throws DateTimeParseException
+ * can be thrown by new parser
+ * @throws DateTimeException
+ * unable to obtain local date or time
+ * @throws IllegalStateException
+ * The formatter for timestamp without time zone should always implement this method. The
+ * exception should never be hit.
*/
@throws(classOf[ParseException])
@throws(classOf[DateTimeParseException])
@@ -99,14 +118,21 @@ sealed trait TimestampFormatter extends Serializable {
* Parses a timestamp in a string and converts it to an optional number of microseconds since
* Unix Epoch in local time.
*
- * @param s - string with timestamp to parse
- * @param allowTimeZone - indicates strict parsing of timezone
- * @return An optional number of microseconds since epoch. The result is None on invalid input.
- * @throws ParseException can be thrown by legacy parser
- * @throws DateTimeParseException can be thrown by new parser
- * @throws DateTimeException unable to obtain local date or time
- * @throws IllegalStateException The formatter for timestamp without time zone should always
- * implement this method. The exception should never be hit.
+ * @param s
+ * \- string with timestamp to parse
+ * @param allowTimeZone
+ * \- indicates strict parsing of timezone
+ * @return
+ * An optional number of microseconds since epoch. The result is None on invalid input.
+ * @throws ParseException
+ * can be thrown by legacy parser
+ * @throws DateTimeParseException
+ * can be thrown by new parser
+ * @throws DateTimeException
+ * unable to obtain local date or time
+ * @throws IllegalStateException
+ * The formatter for timestamp without time zone should always implement this method. The
+ * exception should never be hit.
*/
@throws(classOf[ParseException])
@throws(classOf[DateTimeParseException])
@@ -120,8 +146,8 @@ sealed trait TimestampFormatter extends Serializable {
}
/**
- * Parses a timestamp in a string and converts it to microseconds since Unix Epoch in local time.
- * Zone-id and zone-offset components are ignored.
+ * Parses a timestamp in a string and converts it to microseconds since Unix Epoch in local
+ * time. Zone-id and zone-offset components are ignored.
*/
@throws(classOf[ParseException])
@throws(classOf[DateTimeParseException])
@@ -144,9 +170,10 @@ sealed trait TimestampFormatter extends Serializable {
/**
* Validates the pattern string.
- * @param checkLegacy if true and the pattern is invalid, check whether the pattern is valid for
- * legacy formatters and show hints for using legacy formatter.
- * Otherwise, simply check the pattern string.
+ * @param checkLegacy
+ * if true and the pattern is invalid, check whether the pattern is valid for legacy
+ * formatters and show hints for using legacy formatter. Otherwise, simply check the pattern
+ * string.
*/
def validatePatternString(checkLegacy: Boolean): Unit
}
@@ -157,7 +184,8 @@ class Iso8601TimestampFormatter(
locale: Locale,
legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
isParsing: Boolean)
- extends TimestampFormatter with DateTimeFormatterHelper {
+ extends TimestampFormatter
+ with DateTimeFormatterHelper {
@transient
protected lazy val formatter: DateTimeFormatter =
getOrCreateFormatter(pattern, locale, isParsing)
@@ -166,8 +194,8 @@ class Iso8601TimestampFormatter(
private lazy val zonedFormatter: DateTimeFormatter = formatter.withZone(zoneId)
@transient
- protected lazy val legacyFormatter = TimestampFormatter.getLegacyFormatter(
- pattern, zoneId, locale, legacyFormat)
+ protected lazy val legacyFormatter =
+ TimestampFormatter.getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
override def parseOptional(s: String): Option[Long] = {
try {
@@ -235,8 +263,8 @@ class Iso8601TimestampFormatter(
override def format(instant: Instant): String = {
try {
zonedFormatter.format(instant)
- } catch checkFormattedDiff(toJavaTimestamp(instantToMicros(instant)),
- (t: Timestamp) => format(t))
+ } catch
+ checkFormattedDiff(toJavaTimestamp(instantToMicros(instant)), (t: Timestamp) => format(t))
}
override def format(us: Long): String = {
@@ -256,8 +284,8 @@ class Iso8601TimestampFormatter(
if (checkLegacy) {
try {
formatter
- } catch checkLegacyFormatter(pattern,
- legacyFormatter.validatePatternString(checkLegacy = true))
+ } catch
+ checkLegacyFormatter(pattern, legacyFormatter.validatePatternString(checkLegacy = true))
()
} else {
try {
@@ -268,22 +296,30 @@ class Iso8601TimestampFormatter(
}
/**
- * The formatter for timestamps which doesn't require users to specify a pattern. While formatting,
- * it uses the default pattern [[TimestampFormatter.defaultPattern()]]. In parsing, it follows
- * the CAST logic in conversion of strings to Catalyst's TimestampType.
+ * The formatter for timestamps which doesn't require users to specify a pattern. While
+ * formatting, it uses the default pattern [[TimestampFormatter.defaultPattern()]]. In parsing, it
+ * follows the CAST logic in conversion of strings to Catalyst's TimestampType.
*
- * @param zoneId The time zone ID in which timestamps should be formatted or parsed.
- * @param locale The locale overrides the system locale and is used in formatting.
- * @param legacyFormat Defines the formatter used for legacy timestamps.
- * @param isParsing Whether the formatter is used for parsing (`true`) or for formatting (`false`).
+ * @param zoneId
+ * The time zone ID in which timestamps should be formatted or parsed.
+ * @param locale
+ * The locale overrides the system locale and is used in formatting.
+ * @param legacyFormat
+ * Defines the formatter used for legacy timestamps.
+ * @param isParsing
+ * Whether the formatter is used for parsing (`true`) or for formatting (`false`).
*/
class DefaultTimestampFormatter(
zoneId: ZoneId,
locale: Locale,
legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
isParsing: Boolean)
- extends Iso8601TimestampFormatter(
- TimestampFormatter.defaultPattern(), zoneId, locale, legacyFormat, isParsing) {
+ extends Iso8601TimestampFormatter(
+ TimestampFormatter.defaultPattern(),
+ zoneId,
+ locale,
+ legacyFormat,
+ isParsing) {
override def parse(s: String): Long = {
try {
@@ -299,7 +335,9 @@ class DefaultTimestampFormatter(
val utf8Value = UTF8String.fromString(s)
SparkDateTimeUtils.stringToTimestampWithoutTimeZone(utf8Value, allowTimeZone).getOrElse {
throw ExecutionErrors.cannotParseStringAsDataTypeError(
- TimestampFormatter.defaultPattern(), s, TimestampNTZType)
+ TimestampFormatter.defaultPattern(),
+ s,
+ TimestampNTZType)
}
} catch checkParsedDiff(s, legacyFormatter.parse)
}
@@ -311,20 +349,21 @@ class DefaultTimestampFormatter(
}
/**
- * The formatter parses/formats timestamps according to the pattern `yyyy-MM-dd HH:mm:ss.[..fff..]`
- * where `[..fff..]` is a fraction of second up to microsecond resolution. The formatter does not
- * output trailing zeros in the fraction. For example, the timestamp `2019-03-05 15:00:01.123400` is
- * formatted as the string `2019-03-05 15:00:01.1234`.
+ * The formatter parses/formats timestamps according to the pattern `yyyy-MM-dd
+ * HH:mm:ss.[..fff..]` where `[..fff..]` is a fraction of second up to microsecond resolution. The
+ * formatter does not output trailing zeros in the fraction. For example, the timestamp
+ * `2019-03-05 15:00:01.123400` is formatted as the string `2019-03-05 15:00:01.1234`.
*
- * @param zoneId the time zone identifier in which the formatter parses or format timestamps
+ * @param zoneId
+ * the time zone identifier in which the formatter parses or format timestamps
*/
class FractionTimestampFormatter(zoneId: ZoneId)
- extends Iso8601TimestampFormatter(
- TimestampFormatter.defaultPattern(),
- zoneId,
- TimestampFormatter.defaultLocale,
- LegacyDateFormats.FAST_DATE_FORMAT,
- isParsing = false) {
+ extends Iso8601TimestampFormatter(
+ TimestampFormatter.defaultPattern(),
+ zoneId,
+ TimestampFormatter.defaultLocale,
+ LegacyDateFormats.FAST_DATE_FORMAT,
+ isParsing = false) {
@transient
override protected lazy val formatter = DateTimeFormatterHelper.fractionFormatter
@@ -366,16 +405,14 @@ class FractionTimestampFormatter(zoneId: ZoneId)
}
/**
- * The custom sub-class of `GregorianCalendar` is needed to get access to
- * protected `fields` immediately after parsing. We cannot use
- * the `get()` method because it performs normalization of the fraction
- * part. Accordingly, the `MILLISECOND` field doesn't contain original value.
+ * The custom sub-class of `GregorianCalendar` is needed to get access to protected `fields`
+ * immediately after parsing. We cannot use the `get()` method because it performs normalization
+ * of the fraction part. Accordingly, the `MILLISECOND` field doesn't contain original value.
*
- * Also this class allows to set raw value to the `MILLISECOND` field
- * directly before formatting.
+ * Also this class allows to set raw value to the `MILLISECOND` field directly before formatting.
*/
class MicrosCalendar(tz: TimeZone, digitsInFraction: Int)
- extends GregorianCalendar(tz, Locale.US) {
+ extends GregorianCalendar(tz, Locale.US) {
// Converts parsed `MILLISECOND` field to seconds fraction in microsecond precision.
// For example if the fraction pattern is `SSSS` then `digitsInFraction` = 4, and
// if the `MILLISECOND` field was parsed to `1234`.
@@ -397,16 +434,13 @@ class MicrosCalendar(tz: TimeZone, digitsInFraction: Int)
}
}
-class LegacyFastTimestampFormatter(
- pattern: String,
- zoneId: ZoneId,
- locale: Locale) extends TimestampFormatter {
+class LegacyFastTimestampFormatter(pattern: String, zoneId: ZoneId, locale: Locale)
+ extends TimestampFormatter {
@transient private lazy val fastDateFormat =
FastDateFormat.getInstance(pattern, TimeZone.getTimeZone(zoneId), locale)
- @transient private lazy val cal = new MicrosCalendar(
- fastDateFormat.getTimeZone,
- fastDateFormat.getPattern.count(_ == 'S'))
+ @transient private lazy val cal =
+ new MicrosCalendar(fastDateFormat.getTimeZone, fastDateFormat.getPattern.count(_ == 'S'))
override def parse(s: String): Long = {
cal.clear() // Clear the calendar because it can be re-used many times
@@ -464,7 +498,8 @@ class LegacySimpleTimestampFormatter(
pattern: String,
zoneId: ZoneId,
locale: Locale,
- lenient: Boolean = true) extends TimestampFormatter {
+ lenient: Boolean = true)
+ extends TimestampFormatter {
@transient private lazy val sdf = {
val formatter = new SimpleDateFormat(pattern, locale)
formatter.setTimeZone(TimeZone.getTimeZone(zoneId))
@@ -586,14 +621,10 @@ object TimestampFormatter {
legacyFormat: LegacyDateFormat,
isParsing: Boolean,
forTimestampNTZ: Boolean): TimestampFormatter = {
- getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, isParsing,
- forTimestampNTZ)
+ getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, isParsing, forTimestampNTZ)
}
- def apply(
- format: String,
- zoneId: ZoneId,
- isParsing: Boolean): TimestampFormatter = {
+ def apply(format: String, zoneId: ZoneId, isParsing: Boolean): TimestampFormatter = {
getFormatter(Some(format), zoneId, isParsing = isParsing)
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/UDTUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/UDTUtils.scala
index a98aa26d02ef7..73ab43f04a5a0 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/UDTUtils.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/UDTUtils.scala
@@ -28,11 +28,12 @@ import org.apache.spark.util.SparkClassUtils
* with catalyst because they (amongst others) require access to Spark SQLs internal data
* representation.
*
- * This interface and its companion object provide an escape hatch for working with UDTs from within
- * the api project (e.g. Row.toJSON). The companion will try to bind to an implementation of the
- * interface in catalyst, if none is found it will bind to [[DefaultUDTUtils]].
+ * This interface and its companion object provide an escape hatch for working with UDTs from
+ * within the api project (e.g. Row.toJSON). The companion will try to bind to an implementation
+ * of the interface in catalyst, if none is found it will bind to [[DefaultUDTUtils]].
*/
private[sql] trait UDTUtils {
+
/**
* Convert the UDT instance to something that is compatible with [[org.apache.spark.sql.Row]].
* The returned value must conform to the schema of the UDT.
@@ -41,13 +42,14 @@ private[sql] trait UDTUtils {
}
private[sql] object UDTUtils extends UDTUtils {
- private val delegate = try {
- val cls = SparkClassUtils.classForName("org.apache.spark.sql.catalyst.util.UDTUtilsImpl")
- cls.getConstructor().newInstance().asInstanceOf[UDTUtils]
- } catch {
- case NonFatal(_) =>
- DefaultUDTUtils
- }
+ private val delegate =
+ try {
+ val cls = SparkClassUtils.classForName("org.apache.spark.sql.catalyst.util.UDTUtilsImpl")
+ cls.getConstructor().newInstance().asInstanceOf[UDTUtils]
+ } catch {
+ case NonFatal(_) =>
+ DefaultUDTUtils
+ }
override def toRow(value: Any, udt: UserDefinedType[Any]): Any = delegate.toRow(value, udt)
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala b/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala
index 6034c41906313..3e63b8281f739 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala
@@ -23,9 +23,7 @@ private[sql] trait CompilationErrors extends DataTypeErrorsBase {
def ambiguousColumnOrFieldError(name: Seq[String], numMatches: Int): AnalysisException = {
new AnalysisException(
errorClass = "AMBIGUOUS_COLUMN_OR_FIELD",
- messageParameters = Map(
- "name" -> toSQLId(name),
- "n" -> numMatches.toString))
+ messageParameters = Map("name" -> toSQLId(name), "n" -> numMatches.toString))
}
def columnNotFoundError(colName: String): AnalysisException = {
@@ -51,9 +49,7 @@ private[sql] trait CompilationErrors extends DataTypeErrorsBase {
}
def usingUntypedScalaUDFError(): Throwable = {
- new AnalysisException(
- errorClass = "UNTYPED_SCALA_UDF",
- messageParameters = Map.empty)
+ new AnalysisException(errorClass = "UNTYPED_SCALA_UDF", messageParameters = Map.empty)
}
def invalidBoundaryStartError(start: Long): Throwable = {
@@ -81,14 +77,11 @@ private[sql] trait CompilationErrors extends DataTypeErrorsBase {
def invalidSaveModeError(saveMode: String): Throwable = {
new AnalysisException(
errorClass = "INVALID_SAVE_MODE",
- messageParameters = Map("mode" -> toDSOption(saveMode))
- )
+ messageParameters = Map("mode" -> toDSOption(saveMode)))
}
def sortByWithoutBucketingError(): Throwable = {
- new AnalysisException(
- errorClass = "SORT_BY_WITHOUT_BUCKETING",
- messageParameters = Map.empty)
+ new AnalysisException(errorClass = "SORT_BY_WITHOUT_BUCKETING", messageParameters = Map.empty)
}
def bucketByUnsupportedByOperationError(operation: String): Throwable = {
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/errors/DataTypeErrors.scala b/sql/api/src/main/scala/org/apache/spark/sql/errors/DataTypeErrors.scala
index 32b4198dc1a63..388a98569258b 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/errors/DataTypeErrors.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/errors/DataTypeErrors.scala
@@ -25,9 +25,9 @@ import org.apache.spark.sql.types.{DataType, Decimal, StringType}
import org.apache.spark.unsafe.types.UTF8String
/**
- * Object for grouping error messages from (most) exceptions thrown during query execution.
- * This does not include exceptions thrown during the eager execution of commands, which are
- * grouped into [[CompilationErrors]].
+ * Object for grouping error messages from (most) exceptions thrown during query execution. This
+ * does not include exceptions thrown during the eager execution of commands, which are grouped
+ * into [[CompilationErrors]].
*/
private[sql] object DataTypeErrors extends DataTypeErrorsBase {
def unsupportedOperationExceptionError(): SparkUnsupportedOperationException = {
@@ -35,13 +35,12 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase {
}
def decimalPrecisionExceedsMaxPrecisionError(
- precision: Int, maxPrecision: Int): SparkArithmeticException = {
+ precision: Int,
+ maxPrecision: Int): SparkArithmeticException = {
new SparkArithmeticException(
errorClass = "DECIMAL_PRECISION_EXCEEDS_MAX_PRECISION",
- messageParameters = Map(
- "precision" -> precision.toString,
- "maxPrecision" -> maxPrecision.toString
- ),
+ messageParameters =
+ Map("precision" -> precision.toString, "maxPrecision" -> maxPrecision.toString),
context = Array.empty,
summary = "")
}
@@ -53,8 +52,7 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase {
def outOfDecimalTypeRangeError(str: UTF8String): SparkArithmeticException = {
new SparkArithmeticException(
errorClass = "NUMERIC_OUT_OF_SUPPORTED_RANGE",
- messageParameters = Map(
- "value" -> str.toString),
+ messageParameters = Map("value" -> str.toString),
context = Array.empty,
summary = "")
}
@@ -68,25 +66,20 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase {
def nullLiteralsCannotBeCastedError(name: String): SparkUnsupportedOperationException = {
new SparkUnsupportedOperationException(
errorClass = "_LEGACY_ERROR_TEMP_2226",
- messageParameters = Map(
- "name" -> name))
+ messageParameters = Map("name" -> name))
}
def notUserDefinedTypeError(name: String, userClass: String): Throwable = {
new SparkException(
errorClass = "_LEGACY_ERROR_TEMP_2227",
- messageParameters = Map(
- "name" -> name,
- "userClass" -> userClass),
+ messageParameters = Map("name" -> name, "userClass" -> userClass),
cause = null)
}
def cannotLoadUserDefinedTypeError(name: String, userClass: String): Throwable = {
new SparkException(
errorClass = "_LEGACY_ERROR_TEMP_2228",
- messageParameters = Map(
- "name" -> name,
- "userClass" -> userClass),
+ messageParameters = Map("name" -> name, "userClass" -> userClass),
cause = null)
}
@@ -99,50 +92,42 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase {
def schemaFailToParseError(schema: String, e: Throwable): Throwable = {
new AnalysisException(
errorClass = "INVALID_SCHEMA.PARSE_ERROR",
- messageParameters = Map(
- "inputSchema" -> toSQLSchema(schema),
- "reason" -> e.getMessage
- ),
+ messageParameters = Map("inputSchema" -> toSQLSchema(schema), "reason" -> e.getMessage),
cause = Some(e))
}
def invalidDayTimeIntervalType(startFieldName: String, endFieldName: String): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1224",
- messageParameters = Map(
- "startFieldName" -> startFieldName,
- "endFieldName" -> endFieldName))
+ messageParameters = Map("startFieldName" -> startFieldName, "endFieldName" -> endFieldName))
}
def invalidDayTimeField(field: Byte, supportedIds: Seq[String]): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1223",
- messageParameters = Map(
- "field" -> field.toString,
- "supportedIds" -> supportedIds.mkString(", ")))
+ messageParameters =
+ Map("field" -> field.toString, "supportedIds" -> supportedIds.mkString(", ")))
}
def invalidYearMonthField(field: Byte, supportedIds: Seq[String]): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1225",
- messageParameters = Map(
- "field" -> field.toString,
- "supportedIds" -> supportedIds.mkString(", ")))
+ messageParameters =
+ Map("field" -> field.toString, "supportedIds" -> supportedIds.mkString(", ")))
}
def decimalCannotGreaterThanPrecisionError(scale: Int, precision: Int): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1228",
- messageParameters = Map(
- "scale" -> scale.toString,
- "precision" -> precision.toString))
+ messageParameters = Map("scale" -> scale.toString, "precision" -> precision.toString))
}
def negativeScaleNotAllowedError(scale: Int): Throwable = {
val sqlConf = QuotingUtils.toSQLConf("spark.sql.legacy.allowNegativeScaleOfDecimal")
- SparkException.internalError(s"Negative scale is not allowed: ${scale.toString}." +
- s" Set the config ${sqlConf}" +
- " to \"true\" to allow it.")
+ SparkException.internalError(
+ s"Negative scale is not allowed: ${scale.toString}." +
+ s" Set the config ${sqlConf}" +
+ " to \"true\" to allow it.")
}
def attributeNameSyntaxError(name: String): Throwable = {
@@ -154,19 +139,17 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase {
def cannotMergeIncompatibleDataTypesError(left: DataType, right: DataType): Throwable = {
new SparkException(
errorClass = "CANNOT_MERGE_INCOMPATIBLE_DATA_TYPE",
- messageParameters = Map(
- "left" -> toSQLType(left),
- "right" -> toSQLType(right)),
+ messageParameters = Map("left" -> toSQLType(left), "right" -> toSQLType(right)),
cause = null)
}
def cannotMergeDecimalTypesWithIncompatibleScaleError(
- leftScale: Int, rightScale: Int): Throwable = {
+ leftScale: Int,
+ rightScale: Int): Throwable = {
new SparkException(
errorClass = "_LEGACY_ERROR_TEMP_2124",
- messageParameters = Map(
- "leftScale" -> leftScale.toString(),
- "rightScale" -> rightScale.toString()),
+ messageParameters =
+ Map("leftScale" -> leftScale.toString(), "rightScale" -> rightScale.toString()),
cause = null)
}
@@ -179,9 +162,7 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase {
def invalidFieldName(fieldName: Seq[String], path: Seq[String], context: Origin): Throwable = {
new AnalysisException(
errorClass = "INVALID_FIELD_NAME",
- messageParameters = Map(
- "fieldName" -> toSQLId(fieldName),
- "path" -> toSQLId(path)),
+ messageParameters = Map("fieldName" -> toSQLId(fieldName), "path" -> toSQLId(path)),
origin = context)
}
@@ -227,30 +208,26 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase {
messageParameters = Map(
"expression" -> convertedValueStr,
"sourceType" -> toSQLType(StringType),
- "targetType" -> toSQLType(to),
- "ansiConfig" -> toSQLConf("spark.sql.ansi.enabled")),
+ "targetType" -> toSQLType(to)),
context = getQueryContext(context),
summary = getSummary(context))
}
def ambiguousColumnOrFieldError(
- name: Seq[String], numMatches: Int, context: Origin): Throwable = {
+ name: Seq[String],
+ numMatches: Int,
+ context: Origin): Throwable = {
new AnalysisException(
errorClass = "AMBIGUOUS_COLUMN_OR_FIELD",
- messageParameters = Map(
- "name" -> toSQLId(name),
- "n" -> numMatches.toString),
+ messageParameters = Map("name" -> toSQLId(name), "n" -> numMatches.toString),
origin = context)
}
def castingCauseOverflowError(t: String, from: DataType, to: DataType): ArithmeticException = {
new SparkArithmeticException(
errorClass = "CAST_OVERFLOW",
- messageParameters = Map(
- "value" -> t,
- "sourceType" -> toSQLType(from),
- "targetType" -> toSQLType(to),
- "ansiConfig" -> toSQLConf("spark.sql.ansi.enabled")),
+ messageParameters =
+ Map("value" -> t, "sourceType" -> toSQLType(from), "targetType" -> toSQLType(to)),
context = Array.empty,
summary = "")
}
@@ -267,15 +244,13 @@ private[sql] object DataTypeErrors extends DataTypeErrorsBase {
messageParameters = Map(
"methodName" -> "fieldIndex",
"className" -> "Row",
- "fieldName" -> toSQLId(fieldName))
- )
+ "fieldName" -> toSQLId(fieldName)))
}
def valueIsNullError(index: Int): Throwable = {
new SparkRuntimeException(
errorClass = "ROW_VALUE_IS_NULL",
- messageParameters = Map(
- "index" -> index.toString),
+ messageParameters = Map("index" -> index.toString),
cause = null)
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/errors/ExecutionErrors.scala b/sql/api/src/main/scala/org/apache/spark/sql/errors/ExecutionErrors.scala
index 7910c386fcf14..698a7b096e1a5 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/errors/ExecutionErrors.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/errors/ExecutionErrors.scala
@@ -60,7 +60,8 @@ private[sql] trait ExecutionErrors extends DataTypeErrorsBase {
}
def failToRecognizePatternAfterUpgradeError(
- pattern: String, e: Throwable): SparkUpgradeException = {
+ pattern: String,
+ e: Throwable): SparkUpgradeException = {
new SparkUpgradeException(
errorClass = "INCONSISTENT_BEHAVIOR_CROSS_VERSION.DATETIME_PATTERN_RECOGNITION",
messageParameters = Map(
@@ -73,9 +74,8 @@ private[sql] trait ExecutionErrors extends DataTypeErrorsBase {
def failToRecognizePatternError(pattern: String, e: Throwable): SparkRuntimeException = {
new SparkRuntimeException(
errorClass = "_LEGACY_ERROR_TEMP_2130",
- messageParameters = Map(
- "pattern" -> toSQLValue(pattern),
- "docroot" -> SparkBuildInfo.spark_doc_root),
+ messageParameters =
+ Map("pattern" -> toSQLValue(pattern), "docroot" -> SparkBuildInfo.spark_doc_root),
cause = e)
}
@@ -93,9 +93,9 @@ private[sql] trait ExecutionErrors extends DataTypeErrorsBase {
}
def invalidInputInCastToDatetimeError(
- value: Double,
- to: DataType,
- context: QueryContext): SparkDateTimeException = {
+ value: Double,
+ to: DataType,
+ context: QueryContext): SparkDateTimeException = {
invalidInputInCastToDatetimeErrorInternal(toSQLValue(value), DoubleType, to, context)
}
@@ -109,8 +109,7 @@ private[sql] trait ExecutionErrors extends DataTypeErrorsBase {
messageParameters = Map(
"expression" -> sqlValue,
"sourceType" -> toSQLType(from),
- "targetType" -> toSQLType(to),
- "ansiConfig" -> toSQLConf(SqlApiConf.ANSI_ENABLED_KEY)),
+ "targetType" -> toSQLType(to)),
context = getQueryContext(context),
summary = getSummary(context))
}
@@ -132,8 +131,10 @@ private[sql] trait ExecutionErrors extends DataTypeErrorsBase {
summary = getSummary(context))
}
- def cannotParseStringAsDataTypeError(pattern: String, value: String, dataType: DataType)
- : Throwable = {
+ def cannotParseStringAsDataTypeError(
+ pattern: String,
+ value: String,
+ dataType: DataType): Throwable = {
SparkException.internalError(
s"Cannot parse field value ${toSQLValue(value)} for pattern ${toSQLValue(pattern)} " +
s"as the target spark data type ${toSQLType(dataType)}.")
@@ -161,17 +162,14 @@ private[sql] trait ExecutionErrors extends DataTypeErrorsBase {
def userDefinedTypeNotAnnotatedAndRegisteredError(udt: UserDefinedType[_]): Throwable = {
new SparkException(
errorClass = "_LEGACY_ERROR_TEMP_2155",
- messageParameters = Map(
- "userClass" -> udt.userClass.getName),
+ messageParameters = Map("userClass" -> udt.userClass.getName),
cause = null)
}
def cannotFindEncoderForTypeError(typeName: String): SparkUnsupportedOperationException = {
new SparkUnsupportedOperationException(
errorClass = "ENCODER_NOT_FOUND",
- messageParameters = Map(
- "typeName" -> typeName,
- "docroot" -> SparkBuildInfo.spark_doc_root))
+ messageParameters = Map("typeName" -> typeName, "docroot" -> SparkBuildInfo.spark_doc_root))
}
def cannotHaveCircularReferencesInBeanClassError(
@@ -184,8 +182,7 @@ private[sql] trait ExecutionErrors extends DataTypeErrorsBase {
def cannotFindConstructorForTypeError(tpe: String): SparkUnsupportedOperationException = {
new SparkUnsupportedOperationException(
errorClass = "_LEGACY_ERROR_TEMP_2144",
- messageParameters = Map(
- "tpe" -> tpe))
+ messageParameters = Map("tpe" -> tpe))
}
def cannotHaveCircularReferencesInClassError(t: String): SparkUnsupportedOperationException = {
@@ -195,12 +192,12 @@ private[sql] trait ExecutionErrors extends DataTypeErrorsBase {
}
def cannotUseInvalidJavaIdentifierAsFieldNameError(
- fieldName: String, walkedTypePath: WalkedTypePath): SparkUnsupportedOperationException = {
+ fieldName: String,
+ walkedTypePath: WalkedTypePath): SparkUnsupportedOperationException = {
new SparkUnsupportedOperationException(
errorClass = "_LEGACY_ERROR_TEMP_2140",
- messageParameters = Map(
- "fieldName" -> fieldName,
- "walkedTypePath" -> walkedTypePath.toString))
+ messageParameters =
+ Map("fieldName" -> fieldName, "walkedTypePath" -> walkedTypePath.toString))
}
def primaryConstructorNotFoundError(cls: Class[_]): SparkRuntimeException = {
@@ -213,8 +210,33 @@ private[sql] trait ExecutionErrors extends DataTypeErrorsBase {
def cannotGetOuterPointerForInnerClassError(innerCls: Class[_]): SparkRuntimeException = {
new SparkRuntimeException(
errorClass = "_LEGACY_ERROR_TEMP_2154",
+ messageParameters = Map("innerCls" -> innerCls.getName))
+ }
+
+ def cannotUseKryoSerialization(): SparkRuntimeException = {
+ new SparkRuntimeException(errorClass = "CANNOT_USE_KRYO", messageParameters = Map.empty)
+ }
+
+ def notPublicClassError(name: String): SparkUnsupportedOperationException = {
+ new SparkUnsupportedOperationException(
+ errorClass = "_LEGACY_ERROR_TEMP_2229",
+ messageParameters = Map("name" -> name))
+ }
+
+ def primitiveTypesNotSupportedError(): SparkUnsupportedOperationException = {
+ new SparkUnsupportedOperationException(errorClass = "_LEGACY_ERROR_TEMP_2230")
+ }
+
+ def elementsOfTupleExceedLimitError(): SparkUnsupportedOperationException = {
+ new SparkUnsupportedOperationException("_LEGACY_ERROR_TEMP_2150")
+ }
+
+ def invalidAgnosticEncoderError(encoder: AnyRef): Throwable = {
+ new SparkRuntimeException(
+ errorClass = "INVALID_AGNOSTIC_ENCODER",
messageParameters = Map(
- "innerCls" -> innerCls.getName))
+ "encoderType" -> encoder.getClass.getName,
+ "docroot" -> SparkBuildInfo.spark_doc_root))
}
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala b/sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala
index e7ae9f2bfb7bb..b19607a28f06c 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala
@@ -27,8 +27,8 @@ import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
import org.apache.spark.sql.catalyst.trees.Origin
/**
- * Object for grouping all error messages of the query parsing.
- * Currently it includes all ParseException.
+ * Object for grouping all error messages of the query parsing. Currently it includes all
+ * ParseException.
*/
private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
@@ -37,9 +37,7 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def parserStackOverflow(parserRuleContext: ParserRuleContext): Throwable = {
- throw new ParseException(
- errorClass = "FAILED_TO_PARSE_TOO_COMPLEX",
- ctx = parserRuleContext)
+ throw new ParseException(errorClass = "FAILED_TO_PARSE_TOO_COMPLEX", ctx = parserRuleContext)
}
def insertOverwriteDirectoryUnsupportedError(): Throwable = {
@@ -160,7 +158,9 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def incompatibleJoinTypesError(
- joinType1: String, joinType2: String, ctx: ParserRuleContext): Throwable = {
+ joinType1: String,
+ joinType2: String,
+ ctx: ParserRuleContext): Throwable = {
new ParseException(
errorClass = "INCOMPATIBLE_JOIN_TYPES",
messageParameters = Map(
@@ -209,13 +209,12 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def cannotParseValueTypeError(
- valueType: String, value: String, ctx: TypeConstructorContext): Throwable = {
+ valueType: String,
+ value: String,
+ ctx: TypeConstructorContext): Throwable = {
new ParseException(
errorClass = "INVALID_TYPED_LITERAL",
- messageParameters = Map(
- "valueType" -> toSQLType(valueType),
- "value" -> toSQLValue(value)
- ),
+ messageParameters = Map("valueType" -> toSQLType(valueType), "value" -> toSQLValue(value)),
ctx)
}
@@ -231,8 +230,12 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
ctx)
}
- def invalidNumericLiteralRangeError(rawStrippedQualifier: String, minValue: BigDecimal,
- maxValue: BigDecimal, typeName: String, ctx: NumberContext): Throwable = {
+ def invalidNumericLiteralRangeError(
+ rawStrippedQualifier: String,
+ minValue: BigDecimal,
+ maxValue: BigDecimal,
+ typeName: String,
+ ctx: NumberContext): Throwable = {
new ParseException(
errorClass = "INVALID_NUMERIC_LITERAL_RANGE",
messageParameters = Map(
@@ -259,7 +262,9 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def fromToIntervalUnsupportedError(
- from: String, to: String, ctx: ParserRuleContext): Throwable = {
+ from: String,
+ to: String,
+ ctx: ParserRuleContext): Throwable = {
new ParseException(
errorClass = "_LEGACY_ERROR_TEMP_0028",
messageParameters = Map("from" -> from, "to" -> to),
@@ -288,7 +293,8 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def nestedTypeMissingElementTypeError(
- dataType: String, ctx: PrimitiveDataTypeContext): Throwable = {
+ dataType: String,
+ ctx: PrimitiveDataTypeContext): Throwable = {
dataType.toUpperCase(Locale.ROOT) match {
case "ARRAY" =>
new ParseException(
@@ -309,23 +315,25 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def partitionTransformNotExpectedError(
- name: String, expr: String, ctx: ApplyTransformContext): Throwable = {
+ name: String,
+ expr: String,
+ ctx: ApplyTransformContext): Throwable = {
new ParseException(
errorClass = "INVALID_SQL_SYNTAX.INVALID_COLUMN_REFERENCE",
- messageParameters = Map(
- "transform" -> toSQLId(name),
- "expr" -> expr),
+ messageParameters = Map("transform" -> toSQLId(name), "expr" -> expr),
ctx)
}
def wrongNumberArgumentsForTransformError(
- name: String, actualNum: Int, ctx: ApplyTransformContext): Throwable = {
+ name: String,
+ actualNum: Int,
+ ctx: ApplyTransformContext): Throwable = {
new ParseException(
errorClass = "INVALID_SQL_SYNTAX.TRANSFORM_WRONG_NUM_ARGS",
messageParameters = Map(
"transform" -> toSQLId(name),
- "expectedNum" -> "1",
- "actualNum" -> actualNum.toString),
+ "expectedNum" -> "1",
+ "actualNum" -> actualNum.toString),
ctx)
}
@@ -337,7 +345,9 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def cannotCleanReservedNamespacePropertyError(
- property: String, ctx: ParserRuleContext, msg: String): Throwable = {
+ property: String,
+ ctx: ParserRuleContext,
+ msg: String): Throwable = {
new ParseException(
errorClass = "UNSUPPORTED_FEATURE.SET_NAMESPACE_PROPERTY",
messageParameters = Map("property" -> property, "msg" -> msg),
@@ -348,12 +358,13 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
new ParseException(
errorClass = "UNSUPPORTED_FEATURE.SET_PROPERTIES_AND_DBPROPERTIES",
messageParameters = Map.empty,
- ctx
- )
+ ctx)
}
def cannotCleanReservedTablePropertyError(
- property: String, ctx: ParserRuleContext, msg: String): Throwable = {
+ property: String,
+ ctx: ParserRuleContext,
+ msg: String): Throwable = {
new ParseException(
errorClass = "UNSUPPORTED_FEATURE.SET_TABLE_PROPERTY",
messageParameters = Map("property" -> property, "msg" -> msg),
@@ -361,12 +372,12 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def duplicatedTablePathsFoundError(
- pathOne: String, pathTwo: String, ctx: ParserRuleContext): Throwable = {
+ pathOne: String,
+ pathTwo: String,
+ ctx: ParserRuleContext): Throwable = {
new ParseException(
errorClass = "_LEGACY_ERROR_TEMP_0032",
- messageParameters = Map(
- "pathOne" -> pathOne,
- "pathTwo" -> pathTwo),
+ messageParameters = Map("pathOne" -> pathOne, "pathTwo" -> pathTwo),
ctx)
}
@@ -374,15 +385,17 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
new ParseException(errorClass = "_LEGACY_ERROR_TEMP_0033", ctx)
}
- def operationInHiveStyleCommandUnsupportedError(operation: String,
- command: String, ctx: StatementContext, msgOpt: Option[String] = None): Throwable = {
+ def operationInHiveStyleCommandUnsupportedError(
+ operation: String,
+ command: String,
+ ctx: StatementContext,
+ msgOpt: Option[String] = None): Throwable = {
new ParseException(
errorClass = "_LEGACY_ERROR_TEMP_0034",
messageParameters = Map(
"operation" -> operation,
"command" -> command,
- "msg" -> msgOpt.map(m => s", $m").getOrElse("")
- ),
+ "msg" -> msgOpt.map(m => s", $m").getOrElse("")),
ctx)
}
@@ -415,7 +428,8 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def addCatalogInCacheTableAsSelectNotAllowedError(
- quoted: String, ctx: CacheTableContext): Throwable = {
+ quoted: String,
+ ctx: CacheTableContext): Throwable = {
new ParseException(
errorClass = "_LEGACY_ERROR_TEMP_0037",
messageParameters = Map("quoted" -> quoted),
@@ -479,22 +493,22 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def invalidPropertyKeyForSetQuotedConfigurationError(
- keyCandidate: String, valueStr: String, ctx: ParserRuleContext): Throwable = {
+ keyCandidate: String,
+ valueStr: String,
+ ctx: ParserRuleContext): Throwable = {
new ParseException(
errorClass = "INVALID_PROPERTY_KEY",
- messageParameters = Map(
- "key" -> toSQLConf(keyCandidate),
- "value" -> toSQLConf(valueStr)),
+ messageParameters = Map("key" -> toSQLConf(keyCandidate), "value" -> toSQLConf(valueStr)),
ctx)
}
def invalidPropertyValueForSetQuotedConfigurationError(
- valueCandidate: String, keyStr: String, ctx: ParserRuleContext): Throwable = {
+ valueCandidate: String,
+ keyStr: String,
+ ctx: ParserRuleContext): Throwable = {
new ParseException(
errorClass = "INVALID_PROPERTY_VALUE",
- messageParameters = Map(
- "value" -> toSQLConf(valueCandidate),
- "key" -> toSQLConf(keyStr)),
+ messageParameters = Map("value" -> toSQLConf(valueCandidate), "key" -> toSQLConf(keyStr)),
ctx)
}
@@ -542,12 +556,32 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
ctx)
}
+ def identityColumnUnsupportedDataType(
+ ctx: IdentityColumnContext,
+ dataType: String): Throwable = {
+ new ParseException("IDENTITY_COLUMNS_UNSUPPORTED_DATA_TYPE", Map("dataType" -> dataType), ctx)
+ }
+
+ def identityColumnIllegalStep(ctx: IdentityColSpecContext): Throwable = {
+ new ParseException("IDENTITY_COLUMNS_ILLEGAL_STEP", Map.empty, ctx)
+ }
+
+ def identityColumnDuplicatedSequenceGeneratorOption(
+ ctx: IdentityColSpecContext,
+ sequenceGeneratorOption: String): Throwable = {
+ new ParseException(
+ "IDENTITY_COLUMNS_DUPLICATED_SEQUENCE_GENERATOR_OPTION",
+ Map("sequenceGeneratorOption" -> sequenceGeneratorOption),
+ ctx)
+ }
+
def createViewWithBothIfNotExistsAndReplaceError(ctx: CreateViewContext): Throwable = {
new ParseException(errorClass = "_LEGACY_ERROR_TEMP_0052", ctx)
}
def temporaryViewWithSchemaBindingMode(ctx: StatementContext): Throwable = {
- new ParseException(errorClass = "UNSUPPORTED_FEATURE.TEMPORARY_VIEW_WITH_SCHEMA_BINDING_MODE",
+ new ParseException(
+ errorClass = "UNSUPPORTED_FEATURE.TEMPORARY_VIEW_WITH_SCHEMA_BINDING_MODE",
messageParameters = Map.empty,
ctx)
}
@@ -581,9 +615,7 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
def defineTempFuncWithIfNotExistsError(ctx: ParserRuleContext): Throwable = {
- new ParseException(
- errorClass = "INVALID_SQL_SYNTAX.CREATE_TEMP_FUNC_WITH_IF_NOT_EXISTS",
- ctx)
+ new ParseException(errorClass = "INVALID_SQL_SYNTAX.CREATE_TEMP_FUNC_WITH_IF_NOT_EXISTS", ctx)
}
def unsupportedFunctionNameError(funcName: Seq[String], ctx: ParserRuleContext): Throwable = {
@@ -632,9 +664,8 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
def invalidNameForDropTempFunc(name: Seq[String], ctx: ParserRuleContext): Throwable = {
new ParseException(
errorClass = "INVALID_SQL_SYNTAX.MULTI_PART_NAME",
- messageParameters = Map(
- "statement" -> toSQLStmt("DROP TEMPORARY FUNCTION"),
- "funcName" -> toSQLId(name)),
+ messageParameters =
+ Map("statement" -> toSQLStmt("DROP TEMPORARY FUNCTION"), "funcName" -> toSQLId(name)),
ctx)
}
@@ -650,9 +681,7 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
new ParseException(errorClass = "REF_DEFAULT_VALUE_IS_NOT_ALLOWED_IN_PARTITION", ctx)
}
- def duplicateArgumentNamesError(
- arguments: Seq[String],
- ctx: ParserRuleContext): Throwable = {
+ def duplicateArgumentNamesError(arguments: Seq[String], ctx: ParserRuleContext): Throwable = {
new ParseException(
errorClass = "EXEC_IMMEDIATE_DUPLICATE_ARGUMENT_ALIASES",
messageParameters = Map("aliases" -> arguments.map(toSQLId).mkString(", ")),
@@ -679,12 +708,9 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
}
new ParseException(
errorClass = errorClass,
- messageParameters = alterTypeMap ++ Map(
- "columnName" -> columnName,
- "optionName" -> optionName
- ),
- ctx
- )
+ messageParameters =
+ alterTypeMap ++ Map("columnName" -> columnName, "optionName" -> optionName),
+ ctx)
}
def invalidDatetimeUnitError(
@@ -697,19 +723,17 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
"functionName" -> toSQLId(functionName),
"parameter" -> toSQLId("unit"),
"invalidValue" -> invalidValue),
- ctx
- )
+ ctx)
}
def invalidTableFunctionIdentifierArgumentMissingParentheses(
- ctx: ParserRuleContext, argumentName: String): Throwable = {
+ ctx: ParserRuleContext,
+ argumentName: String): Throwable = {
new ParseException(
errorClass =
"INVALID_SQL_SYNTAX.INVALID_TABLE_FUNCTION_IDENTIFIER_ARGUMENT_MISSING_PARENTHESES",
- messageParameters = Map(
- "argumentName" -> toSQLId(argumentName)),
- ctx
- )
+ messageParameters = Map("argumentName" -> toSQLId(argumentName)),
+ ctx)
}
def clusterByWithPartitionedBy(ctx: ParserRuleContext): Throwable = {
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala b/sql/api/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
index 146012b4266dd..7d8b33aa5e228 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
@@ -88,8 +88,8 @@ object ProcessingTimeTrigger {
}
/**
- * A [[Trigger]] that continuously processes streaming data, asynchronously checkpointing at
- * the specified interval.
+ * A [[Trigger]] that continuously processes streaming data, asynchronously checkpointing at the
+ * specified interval.
*/
case class ContinuousTrigger(intervalMs: Long) extends Trigger {
Triggers.validate(intervalMs)
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala b/sql/api/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
index 1a2fbdc1fd116..5bdaebe3b073a 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
@@ -43,9 +43,12 @@ import org.apache.spark.sql.internal.{InvokeInlineUserDefinedFunction, UserDefin
*
* Based loosely on Aggregator from Algebird: https://github.com/twitter/algebird
*
- * @tparam IN The input type for the aggregation.
- * @tparam BUF The type of the intermediate value of the reduction.
- * @tparam OUT The type of the final output result.
+ * @tparam IN
+ * The input type for the aggregation.
+ * @tparam BUF
+ * The type of the intermediate value of the reduction.
+ * @tparam OUT
+ * The type of the final output result.
* @since 1.6.0
*/
@SerialVersionUID(2093413866369130093L)
@@ -58,7 +61,7 @@ abstract class Aggregator[-IN, BUF, OUT] extends Serializable with UserDefinedFu
def zero: BUF
/**
- * Combine two values to produce a new value. For performance, the function may modify `b` and
+ * Combine two values to produce a new value. For performance, the function may modify `b` and
* return it instead of constructing new object for b.
* @since 1.6.0
*/
@@ -93,8 +96,6 @@ abstract class Aggregator[-IN, BUF, OUT] extends Serializable with UserDefinedFu
* @since 1.6.0
*/
def toColumn: TypedColumn[IN, OUT] = {
- new TypedColumn[IN, OUT](
- InvokeInlineUserDefinedFunction(this, Nil),
- outputEncoder)
+ new TypedColumn[IN, OUT](InvokeInlineUserDefinedFunction(this, Nil), outputEncoder)
}
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala b/sql/api/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
index c9e0e366a7447..6a22cbfaf351e 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
@@ -52,8 +52,8 @@ sealed abstract class UserDefinedFunction extends UserDefinedFunctionLike {
def nullable: Boolean
/**
- * Returns true iff the UDF is deterministic, i.e. the UDF produces the same output given the same
- * input.
+ * Returns true iff the UDF is deterministic, i.e. the UDF produces the same output given the
+ * same input.
*
* @since 2.3.0
*/
@@ -98,7 +98,8 @@ private[spark] case class SparkUserDefinedFunction(
outputEncoder: Option[Encoder[_]] = None,
givenName: Option[String] = None,
nullable: Boolean = true,
- deterministic: Boolean = true) extends UserDefinedFunction {
+ deterministic: Boolean = true)
+ extends UserDefinedFunction {
override def withName(name: String): SparkUserDefinedFunction = {
copy(givenName = Option(name))
@@ -169,7 +170,8 @@ private[sql] case class UserDefinedAggregator[IN, BUF, OUT](
inputEncoder: Encoder[IN],
givenName: Option[String] = None,
nullable: Boolean = true,
- deterministic: Boolean = true) extends UserDefinedFunction {
+ deterministic: Boolean = true)
+ extends UserDefinedFunction {
override def withName(name: String): UserDefinedAggregator[IN, BUF, OUT] = {
copy(givenName = Option(name))
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/expressions/Window.scala b/sql/api/src/main/scala/org/apache/spark/sql/expressions/Window.scala
index 9c4499ee243f5..dbe2da8f97341 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/expressions/Window.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/expressions/Window.scala
@@ -32,9 +32,10 @@ import org.apache.spark.sql.Column
* Window.partitionBy("country").orderBy("date").rowsBetween(-3, 3)
* }}}
*
- * @note When ordering is not defined, an unbounded window frame (rowFrame, unboundedPreceding,
- * unboundedFollowing) is used by default. When ordering is defined, a growing window frame
- * (rangeFrame, unboundedPreceding, currentRow) is used by default.
+ * @note
+ * When ordering is not defined, an unbounded window frame (rowFrame, unboundedPreceding,
+ * unboundedFollowing) is used by default. When ordering is defined, a growing window frame
+ * (rangeFrame, unboundedPreceding, currentRow) is used by default.
*
* @since 1.4.0
*/
@@ -47,7 +48,7 @@ object Window {
*/
@scala.annotation.varargs
def partitionBy(colName: String, colNames: String*): WindowSpec = {
- spec.partitionBy(colName, colNames : _*)
+ spec.partitionBy(colName, colNames: _*)
}
/**
@@ -56,7 +57,7 @@ object Window {
*/
@scala.annotation.varargs
def partitionBy(cols: Column*): WindowSpec = {
- spec.partitionBy(cols : _*)
+ spec.partitionBy(cols: _*)
}
/**
@@ -65,7 +66,7 @@ object Window {
*/
@scala.annotation.varargs
def orderBy(colName: String, colNames: String*): WindowSpec = {
- spec.orderBy(colName, colNames : _*)
+ spec.orderBy(colName, colNames: _*)
}
/**
@@ -74,12 +75,12 @@ object Window {
*/
@scala.annotation.varargs
def orderBy(cols: Column*): WindowSpec = {
- spec.orderBy(cols : _*)
+ spec.orderBy(cols: _*)
}
/**
- * Value representing the first row in the partition, equivalent to "UNBOUNDED PRECEDING" in SQL.
- * This can be used to specify the frame boundaries:
+ * Value representing the first row in the partition, equivalent to "UNBOUNDED PRECEDING" in
+ * SQL. This can be used to specify the frame boundaries:
*
* {{{
* Window.rowsBetween(Window.unboundedPreceding, Window.currentRow)
@@ -113,22 +114,22 @@ object Window {
def currentRow: Long = 0
/**
- * Creates a [[WindowSpec]] with the frame boundaries defined,
- * from `start` (inclusive) to `end` (inclusive).
+ * Creates a [[WindowSpec]] with the frame boundaries defined, from `start` (inclusive) to `end`
+ * (inclusive).
*
* Both `start` and `end` are positions relative to the current row. For example, "0" means
* "current row", while "-1" means the row before the current row, and "5" means the fifth row
* after the current row.
*
- * We recommend users use `Window.unboundedPreceding`, `Window.unboundedFollowing`,
- * and `Window.currentRow` to specify special boundary values, rather than using integral
- * values directly.
+ * We recommend users use `Window.unboundedPreceding`, `Window.unboundedFollowing`, and
+ * `Window.currentRow` to specify special boundary values, rather than using integral values
+ * directly.
*
- * A row based boundary is based on the position of the row within the partition.
- * An offset indicates the number of rows above or below the current row, the frame for the
- * current row starts or ends. For instance, given a row based sliding frame with a lower bound
- * offset of -1 and a upper bound offset of +2. The frame for row with index 5 would range from
- * index 4 to index 7.
+ * A row based boundary is based on the position of the row within the partition. An offset
+ * indicates the number of rows above or below the current row, the frame for the current row
+ * starts or ends. For instance, given a row based sliding frame with a lower bound offset of -1
+ * and a upper bound offset of +2. The frame for row with index 5 would range from index 4 to
+ * index 7.
*
* {{{
* import org.apache.spark.sql.expressions.Window
@@ -150,10 +151,12 @@ object Window {
* +---+--------+---+
* }}}
*
- * @param start boundary start, inclusive. The frame is unbounded if this is
- * the minimum long value (`Window.unboundedPreceding`).
- * @param end boundary end, inclusive. The frame is unbounded if this is the
- * maximum long value (`Window.unboundedFollowing`).
+ * @param start
+ * boundary start, inclusive. The frame is unbounded if this is the minimum long value
+ * (`Window.unboundedPreceding`).
+ * @param end
+ * boundary end, inclusive. The frame is unbounded if this is the maximum long value
+ * (`Window.unboundedFollowing`).
* @since 2.1.0
*/
// Note: when updating the doc for this method, also update WindowSpec.rowsBetween.
@@ -162,25 +165,24 @@ object Window {
}
/**
- * Creates a [[WindowSpec]] with the frame boundaries defined,
- * from `start` (inclusive) to `end` (inclusive).
+ * Creates a [[WindowSpec]] with the frame boundaries defined, from `start` (inclusive) to `end`
+ * (inclusive).
*
* Both `start` and `end` are relative to the current row. For example, "0" means "current row",
- * while "-1" means one off before the current row, and "5" means the five off after the
- * current row.
+ * while "-1" means one off before the current row, and "5" means the five off after the current
+ * row.
*
- * We recommend users use `Window.unboundedPreceding`, `Window.unboundedFollowing`,
- * and `Window.currentRow` to specify special boundary values, rather than using long values
+ * We recommend users use `Window.unboundedPreceding`, `Window.unboundedFollowing`, and
+ * `Window.currentRow` to specify special boundary values, rather than using long values
* directly.
*
- * A range-based boundary is based on the actual value of the ORDER BY
- * expression(s). An offset is used to alter the value of the ORDER BY expression,
- * for instance if the current ORDER BY expression has a value of 10 and the lower bound offset
- * is -3, the resulting lower bound for the current row will be 10 - 3 = 7. This however puts a
- * number of constraints on the ORDER BY expressions: there can be only one expression and this
- * expression must have a numerical data type. An exception can be made when the offset is
- * unbounded, because no value modification is needed, in this case multiple and non-numeric
- * ORDER BY expression are allowed.
+ * A range-based boundary is based on the actual value of the ORDER BY expression(s). An offset
+ * is used to alter the value of the ORDER BY expression, for instance if the current ORDER BY
+ * expression has a value of 10 and the lower bound offset is -3, the resulting lower bound for
+ * the current row will be 10 - 3 = 7. This however puts a number of constraints on the ORDER BY
+ * expressions: there can be only one expression and this expression must have a numerical data
+ * type. An exception can be made when the offset is unbounded, because no value modification is
+ * needed, in this case multiple and non-numeric ORDER BY expression are allowed.
*
* {{{
* import org.apache.spark.sql.expressions.Window
@@ -202,10 +204,12 @@ object Window {
* +---+--------+---+
* }}}
*
- * @param start boundary start, inclusive. The frame is unbounded if this is
- * the minimum long value (`Window.unboundedPreceding`).
- * @param end boundary end, inclusive. The frame is unbounded if this is the
- * maximum long value (`Window.unboundedFollowing`).
+ * @param start
+ * boundary start, inclusive. The frame is unbounded if this is the minimum long value
+ * (`Window.unboundedPreceding`).
+ * @param end
+ * boundary end, inclusive. The frame is unbounded if this is the maximum long value
+ * (`Window.unboundedFollowing`).
* @since 2.1.0
*/
// Note: when updating the doc for this method, also update WindowSpec.rangeBetween.
@@ -234,4 +238,4 @@ object Window {
* @since 1.4.0
*/
@Stable
-class Window private() // So we can see Window in JavaDoc.
+class Window private () // So we can see Window in JavaDoc.
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala b/sql/api/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala
index a888563d66a71..9abdee9c79ebc 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.internal.{ColumnNode, SortOrder, Window => EvalWindo
* @since 1.4.0
*/
@Stable
-class WindowSpec private[sql](
+class WindowSpec private[sql] (
partitionSpec: Seq[ColumnNode],
orderSpec: Seq[SortOrder],
frame: Option[WindowFrame]) {
@@ -78,15 +78,15 @@ class WindowSpec private[sql](
* "current row", while "-1" means the row before the current row, and "5" means the fifth row
* after the current row.
*
- * We recommend users use `Window.unboundedPreceding`, `Window.unboundedFollowing`,
- * and `Window.currentRow` to specify special boundary values, rather than using integral
- * values directly.
+ * We recommend users use `Window.unboundedPreceding`, `Window.unboundedFollowing`, and
+ * `Window.currentRow` to specify special boundary values, rather than using integral values
+ * directly.
*
- * A row based boundary is based on the position of the row within the partition.
- * An offset indicates the number of rows above or below the current row, the frame for the
- * current row starts or ends. For instance, given a row based sliding frame with a lower bound
- * offset of -1 and a upper bound offset of +2. The frame for row with index 5 would range from
- * index 4 to index 7.
+ * A row based boundary is based on the position of the row within the partition. An offset
+ * indicates the number of rows above or below the current row, the frame for the current row
+ * starts or ends. For instance, given a row based sliding frame with a lower bound offset of -1
+ * and a upper bound offset of +2. The frame for row with index 5 would range from index 4 to
+ * index 7.
*
* {{{
* import org.apache.spark.sql.expressions.Window
@@ -108,10 +108,12 @@ class WindowSpec private[sql](
* +---+--------+---+
* }}}
*
- * @param start boundary start, inclusive. The frame is unbounded if this is
- * the minimum long value (`Window.unboundedPreceding`).
- * @param end boundary end, inclusive. The frame is unbounded if this is the
- * maximum long value (`Window.unboundedFollowing`).
+ * @param start
+ * boundary start, inclusive. The frame is unbounded if this is the minimum long value
+ * (`Window.unboundedPreceding`).
+ * @param end
+ * boundary end, inclusive. The frame is unbounded if this is the maximum long value
+ * (`Window.unboundedFollowing`).
* @since 1.4.0
*/
// Note: when updating the doc for this method, also update Window.rowsBetween.
@@ -136,22 +138,21 @@ class WindowSpec private[sql](
/**
* Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).
*
- * Both `start` and `end` are relative from the current row. For example, "0" means
- * "current row", while "-1" means one off before the current row, and "5" means the five off
- * after the current row.
+ * Both `start` and `end` are relative from the current row. For example, "0" means "current
+ * row", while "-1" means one off before the current row, and "5" means the five off after the
+ * current row.
*
- * We recommend users use `Window.unboundedPreceding`, `Window.unboundedFollowing`,
- * and `Window.currentRow` to specify special boundary values, rather than using long values
+ * We recommend users use `Window.unboundedPreceding`, `Window.unboundedFollowing`, and
+ * `Window.currentRow` to specify special boundary values, rather than using long values
* directly.
*
- * A range-based boundary is based on the actual value of the ORDER BY
- * expression(s). An offset is used to alter the value of the ORDER BY expression, for
- * instance if the current order by expression has a value of 10 and the lower bound offset
- * is -3, the resulting lower bound for the current row will be 10 - 3 = 7. This however puts a
- * number of constraints on the ORDER BY expressions: there can be only one expression and this
- * expression must have a numerical data type. An exception can be made when the offset is
- * unbounded, because no value modification is needed, in this case multiple and non-numeric
- * ORDER BY expression are allowed.
+ * A range-based boundary is based on the actual value of the ORDER BY expression(s). An offset
+ * is used to alter the value of the ORDER BY expression, for instance if the current order by
+ * expression has a value of 10 and the lower bound offset is -3, the resulting lower bound for
+ * the current row will be 10 - 3 = 7. This however puts a number of constraints on the ORDER BY
+ * expressions: there can be only one expression and this expression must have a numerical data
+ * type. An exception can be made when the offset is unbounded, because no value modification is
+ * needed, in this case multiple and non-numeric ORDER BY expression are allowed.
*
* {{{
* import org.apache.spark.sql.expressions.Window
@@ -173,10 +174,12 @@ class WindowSpec private[sql](
* +---+--------+---+
* }}}
*
- * @param start boundary start, inclusive. The frame is unbounded if this is
- * the minimum long value (`Window.unboundedPreceding`).
- * @param end boundary end, inclusive. The frame is unbounded if this is the
- * maximum long value (`Window.unboundedFollowing`).
+ * @param start
+ * boundary start, inclusive. The frame is unbounded if this is the minimum long value
+ * (`Window.unboundedPreceding`).
+ * @param end
+ * boundary end, inclusive. The frame is unbounded if this is the maximum long value
+ * (`Window.unboundedFollowing`).
* @since 1.4.0
*/
// Note: when updating the doc for this method, also update Window.rangeBetween.
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/expressions/udaf.scala b/sql/api/src/main/scala/org/apache/spark/sql/expressions/udaf.scala
index a4aa9c312aff2..5e7c993fae414 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/expressions/udaf.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/expressions/udaf.scala
@@ -26,18 +26,21 @@ import org.apache.spark.sql.types._
* The base class for implementing user-defined aggregate functions (UDAF).
*
* @since 1.5.0
- * @deprecated UserDefinedAggregateFunction is deprecated.
- * Aggregator[IN, BUF, OUT] should now be registered as a UDF via the functions.udaf(agg) method.
+ * @deprecated
+ * UserDefinedAggregateFunction is deprecated. Aggregator[IN, BUF, OUT] should now be registered
+ * as a UDF via the functions.udaf(agg) method.
*/
@Stable
-@deprecated("Aggregator[IN, BUF, OUT] should now be registered as a UDF" +
- " via the functions.udaf(agg) method.", "3.0.0")
+@deprecated(
+ "Aggregator[IN, BUF, OUT] should now be registered as a UDF" +
+ " via the functions.udaf(agg) method.",
+ "3.0.0")
abstract class UserDefinedAggregateFunction extends Serializable with UserDefinedFunctionLike {
/**
- * A `StructType` represents data types of input arguments of this aggregate function.
- * For example, if a [[UserDefinedAggregateFunction]] expects two input arguments
- * with type of `DoubleType` and `LongType`, the returned `StructType` will look like
+ * A `StructType` represents data types of input arguments of this aggregate function. For
+ * example, if a [[UserDefinedAggregateFunction]] expects two input arguments with type of
+ * `DoubleType` and `LongType`, the returned `StructType` will look like
*
* ```
* new StructType()
@@ -45,18 +48,17 @@ abstract class UserDefinedAggregateFunction extends Serializable with UserDefine
* .add("longInput", LongType)
* ```
*
- * The name of a field of this `StructType` is only used to identify the corresponding
- * input argument. Users can choose names to identify the input arguments.
+ * The name of a field of this `StructType` is only used to identify the corresponding input
+ * argument. Users can choose names to identify the input arguments.
*
* @since 1.5.0
*/
def inputSchema: StructType
/**
- * A `StructType` represents data types of values in the aggregation buffer.
- * For example, if a [[UserDefinedAggregateFunction]]'s buffer has two values
- * (i.e. two intermediate values) with type of `DoubleType` and `LongType`,
- * the returned `StructType` will look like
+ * A `StructType` represents data types of values in the aggregation buffer. For example, if a
+ * [[UserDefinedAggregateFunction]]'s buffer has two values (i.e. two intermediate values) with
+ * type of `DoubleType` and `LongType`, the returned `StructType` will look like
*
* ```
* new StructType()
@@ -64,8 +66,8 @@ abstract class UserDefinedAggregateFunction extends Serializable with UserDefine
* .add("longInput", LongType)
* ```
*
- * The name of a field of this `StructType` is only used to identify the corresponding
- * buffer value. Users can choose names to identify the input arguments.
+ * The name of a field of this `StructType` is only used to identify the corresponding buffer
+ * value. Users can choose names to identify the input arguments.
*
* @since 1.5.0
*/
@@ -79,8 +81,8 @@ abstract class UserDefinedAggregateFunction extends Serializable with UserDefine
def dataType: DataType
/**
- * Returns true iff this function is deterministic, i.e. given the same input,
- * always return the same output.
+ * Returns true iff this function is deterministic, i.e. given the same input, always return the
+ * same output.
*
* @since 1.5.0
*/
@@ -90,8 +92,8 @@ abstract class UserDefinedAggregateFunction extends Serializable with UserDefine
* Initializes the given aggregation buffer, i.e. the zero value of the aggregation buffer.
*
* The contract should be that applying the merge function on two initial buffers should just
- * return the initial buffer itself, i.e.
- * `merge(initialBuffer, initialBuffer)` should equal `initialBuffer`.
+ * return the initial buffer itself, i.e. `merge(initialBuffer, initialBuffer)` should equal
+ * `initialBuffer`.
*
* @since 1.5.0
*/
@@ -134,8 +136,8 @@ abstract class UserDefinedAggregateFunction extends Serializable with UserDefine
}
/**
- * Creates a `Column` for this UDAF using the distinct values of the given
- * `Column`s as input arguments.
+ * Creates a `Column` for this UDAF using the distinct values of the given `Column`s as input
+ * arguments.
*
* @since 1.5.0
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index e46d6c95b31ae..0662b8f2b271f 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -104,9 +104,9 @@ object functions {
/**
* Creates a [[Column]] of literal value.
*
- * The passed in object is returned directly if it is already a [[Column]].
- * If the object is a Scala Symbol, it is converted into a [[Column]] also.
- * Otherwise, a new [[Column]] is created to represent the literal value.
+ * The passed in object is returned directly if it is already a [[Column]]. If the object is a
+ * Scala Symbol, it is converted into a [[Column]] also. Otherwise, a new [[Column]] is created
+ * to represent the literal value.
*
* @group normal_funcs
* @since 1.3.0
@@ -121,7 +121,7 @@ object functions {
// method, `typedLit[Any](literal)` will always fail and fallback to `Literal.apply`. Hence,
// we can just manually call `Literal.apply` to skip the expensive `ScalaReflection` code.
// This is significantly better when there are many threads calling `lit` concurrently.
- Column(internal.Literal(literal))
+ Column(internal.Literal(literal))
}
}
@@ -133,26 +133,26 @@ object functions {
* @group normal_funcs
* @since 2.2.0
*/
- def typedLit[T : TypeTag](literal: T): Column = {
+ def typedLit[T: TypeTag](literal: T): Column = {
typedlit(literal)
}
/**
* Creates a [[Column]] of literal value.
*
- * The passed in object is returned directly if it is already a [[Column]].
- * If the object is a Scala Symbol, it is converted into a [[Column]] also.
- * Otherwise, a new [[Column]] is created to represent the literal value.
- * The difference between this function and [[lit]] is that this function
- * can handle parameterized scala types e.g.: List, Seq and Map.
+ * The passed in object is returned directly if it is already a [[Column]]. If the object is a
+ * Scala Symbol, it is converted into a [[Column]] also. Otherwise, a new [[Column]] is created
+ * to represent the literal value. The difference between this function and [[lit]] is that this
+ * function can handle parameterized scala types e.g.: List, Seq and Map.
*
- * @note `typedlit` will call expensive Scala reflection APIs. `lit` is preferred if parameterized
- * Scala types are not used.
+ * @note
+ * `typedlit` will call expensive Scala reflection APIs. `lit` is preferred if parameterized
+ * Scala types are not used.
*
* @group normal_funcs
* @since 3.2.0
*/
- def typedlit[T : TypeTag](literal: T): Column = {
+ def typedlit[T: TypeTag](literal: T): Column = {
literal match {
case c: Column => c
case s: Symbol => new ColumnName(s.name)
@@ -178,8 +178,8 @@ object functions {
def asc(columnName: String): Column = Column(columnName).asc
/**
- * Returns a sort expression based on ascending order of the column,
- * and null values return before non-null values.
+ * Returns a sort expression based on ascending order of the column, and null values return
+ * before non-null values.
* {{{
* df.sort(asc_nulls_first("dept"), desc("age"))
* }}}
@@ -190,8 +190,8 @@ object functions {
def asc_nulls_first(columnName: String): Column = Column(columnName).asc_nulls_first
/**
- * Returns a sort expression based on ascending order of the column,
- * and null values appear after non-null values.
+ * Returns a sort expression based on ascending order of the column, and null values appear
+ * after non-null values.
* {{{
* df.sort(asc_nulls_last("dept"), desc("age"))
* }}}
@@ -213,8 +213,8 @@ object functions {
def desc(columnName: String): Column = Column(columnName).desc
/**
- * Returns a sort expression based on the descending order of the column,
- * and null values appear before non-null values.
+ * Returns a sort expression based on the descending order of the column, and null values appear
+ * before non-null values.
* {{{
* df.sort(asc("dept"), desc_nulls_first("age"))
* }}}
@@ -225,8 +225,8 @@ object functions {
def desc_nulls_first(columnName: String): Column = Column(columnName).desc_nulls_first
/**
- * Returns a sort expression based on the descending order of the column,
- * and null values appear after non-null values.
+ * Returns a sort expression based on the descending order of the column, and null values appear
+ * after non-null values.
* {{{
* df.sort(asc("dept"), desc_nulls_last("age"))
* }}}
@@ -236,7 +236,6 @@ object functions {
*/
def desc_nulls_last(columnName: String): Column = Column(columnName).desc_nulls_last
-
//////////////////////////////////////////////////////////////////////////////////////////////
// Aggregate functions
//////////////////////////////////////////////////////////////////////////////////////////////
@@ -285,12 +284,14 @@ object functions {
* @group agg_funcs
* @since 2.1.0
*/
- def approx_count_distinct(columnName: String): Column = approx_count_distinct(column(columnName))
+ def approx_count_distinct(columnName: String): Column = approx_count_distinct(
+ column(columnName))
/**
* Aggregate function: returns the approximate number of distinct items in a group.
*
- * @param rsd maximum relative standard deviation allowed (default = 0.05)
+ * @param rsd
+ * maximum relative standard deviation allowed (default = 0.05)
*
* @group agg_funcs
* @since 2.1.0
@@ -302,7 +303,8 @@ object functions {
/**
* Aggregate function: returns the approximate number of distinct items in a group.
*
- * @param rsd maximum relative standard deviation allowed (default = 0.05)
+ * @param rsd
+ * maximum relative standard deviation allowed (default = 0.05)
*
* @group agg_funcs
* @since 2.1.0
@@ -330,8 +332,9 @@ object functions {
/**
* Aggregate function: returns a list of objects with duplicates.
*
- * @note The function is non-deterministic because the order of collected results depends
- * on the order of the rows which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because the order of collected results depends on the
+ * order of the rows which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 1.6.0
@@ -341,8 +344,9 @@ object functions {
/**
* Aggregate function: returns a list of objects with duplicates.
*
- * @note The function is non-deterministic because the order of collected results depends
- * on the order of the rows which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because the order of collected results depends on the
+ * order of the rows which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 1.6.0
@@ -352,8 +356,9 @@ object functions {
/**
* Aggregate function: returns a set of objects with duplicate elements eliminated.
*
- * @note The function is non-deterministic because the order of collected results depends
- * on the order of the rows which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because the order of collected results depends on the
+ * order of the rows which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 1.6.0
@@ -363,8 +368,9 @@ object functions {
/**
* Aggregate function: returns a set of objects with duplicate elements eliminated.
*
- * @note The function is non-deterministic because the order of collected results depends
- * on the order of the rows which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because the order of collected results depends on the
+ * order of the rows which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 1.6.0
@@ -372,10 +378,10 @@ object functions {
def collect_set(columnName: String): Column = collect_set(Column(columnName))
/**
- * Returns a count-min sketch of a column with the given esp, confidence and seed. The result
- * is an array of bytes, which can be deserialized to a `CountMinSketch` before usage.
- * Count-min sketch is a probabilistic data structure used for cardinality estimation using
- * sub-linear space.
+ * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is
+ * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min
+ * sketch is a probabilistic data structure used for cardinality estimation using sub-linear
+ * space.
*
* @group agg_funcs
* @since 3.5.0
@@ -383,6 +389,18 @@ object functions {
def count_min_sketch(e: Column, eps: Column, confidence: Column, seed: Column): Column =
Column.fn("count_min_sketch", e, eps, confidence, seed)
+ /**
+ * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is
+ * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min
+ * sketch is a probabilistic data structure used for cardinality estimation using sub-linear
+ * space.
+ *
+ * @group agg_funcs
+ * @since 4.0.0
+ */
+ def count_min_sketch(e: Column, eps: Column, confidence: Column): Column =
+ count_min_sketch(e, eps, confidence, lit(SparkClassUtils.random.nextInt))
+
private[spark] def collect_top_k(e: Column, num: Int, reverse: Boolean): Column =
Column.internalFn("collect_top_k", e, lit(num), lit(reverse))
@@ -443,7 +461,7 @@ object functions {
*/
@scala.annotation.varargs
def countDistinct(columnName: String, columnNames: String*): Column =
- count_distinct(Column(columnName), columnNames.map(Column.apply) : _*)
+ count_distinct(Column(columnName), columnNames.map(Column.apply): _*)
/**
* Aggregate function: returns the number of distinct items in a group.
@@ -499,8 +517,9 @@ object functions {
* The function by default returns the first values it sees. It will return the first non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 2.0.0
@@ -514,8 +533,9 @@ object functions {
* The function by default returns the first values it sees. It will return the first non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 2.0.0
@@ -530,8 +550,9 @@ object functions {
* The function by default returns the first values it sees. It will return the first non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 1.3.0
@@ -544,8 +565,9 @@ object functions {
* The function by default returns the first values it sees. It will return the first non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 1.3.0
@@ -555,8 +577,9 @@ object functions {
/**
* Aggregate function: returns the first value in a group.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 3.5.0
@@ -569,8 +592,9 @@ object functions {
* The function by default returns the first values it sees. It will return the first non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 3.5.0
@@ -579,8 +603,8 @@ object functions {
Column.fn("first_value", e, ignoreNulls)
/**
- * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated
- * or not, returns 1 for aggregated or 0 for not aggregated in the result set.
+ * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or
+ * not, returns 1 for aggregated or 0 for not aggregated in the result set.
*
* @group agg_funcs
* @since 2.0.0
@@ -588,8 +612,8 @@ object functions {
def grouping(e: Column): Column = Column.fn("grouping", e)
/**
- * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated
- * or not, returns 1 for aggregated or 0 for not aggregated in the result set.
+ * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or
+ * not, returns 1 for aggregated or 0 for not aggregated in the result set.
*
* @group agg_funcs
* @since 2.0.0
@@ -603,8 +627,9 @@ object functions {
* (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn)
* }}}
*
- * @note The list of columns should match with grouping columns exactly, or empty (means all the
- * grouping columns).
+ * @note
+ * The list of columns should match with grouping columns exactly, or empty (means all the
+ * grouping columns).
*
* @group agg_funcs
* @since 2.0.0
@@ -618,18 +643,19 @@ object functions {
* (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn)
* }}}
*
- * @note The list of columns should match with grouping columns exactly.
+ * @note
+ * The list of columns should match with grouping columns exactly.
*
* @group agg_funcs
* @since 2.0.0
*/
def grouping_id(colName: String, colNames: String*): Column = {
- grouping_id((Seq(colName) ++ colNames).map(n => Column(n)) : _*)
+ grouping_id((Seq(colName) ++ colNames).map(n => Column(n)): _*)
}
/**
- * Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch configured with lgConfigK arg.
+ * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch
+ * configured with lgConfigK arg.
*
* @group agg_funcs
* @since 3.5.0
@@ -638,8 +664,8 @@ object functions {
Column.fn("hll_sketch_agg", e, lgConfigK)
/**
- * Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch configured with lgConfigK arg.
+ * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch
+ * configured with lgConfigK arg.
*
* @group agg_funcs
* @since 3.5.0
@@ -648,8 +674,8 @@ object functions {
Column.fn("hll_sketch_agg", e, lit(lgConfigK))
/**
- * Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch configured with lgConfigK arg.
+ * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch
+ * configured with lgConfigK arg.
*
* @group agg_funcs
* @since 3.5.0
@@ -659,8 +685,8 @@ object functions {
}
/**
- * Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch configured with default lgConfigK value.
+ * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch
+ * configured with default lgConfigK value.
*
* @group agg_funcs
* @since 3.5.0
@@ -669,8 +695,8 @@ object functions {
Column.fn("hll_sketch_agg", e)
/**
- * Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch configured with default lgConfigK value.
+ * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch
+ * configured with default lgConfigK value.
*
* @group agg_funcs
* @since 3.5.0
@@ -681,9 +707,9 @@ object functions {
/**
* Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch, generated by merging previously created Datasketches HllSketch instances
- * via a Datasketches Union instance. Throws an exception if sketches have different
- * lgConfigK values and allowDifferentLgConfigK is set to false.
+ * HllSketch, generated by merging previously created Datasketches HllSketch instances via a
+ * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values
+ * and allowDifferentLgConfigK is set to false.
*
* @group agg_funcs
* @since 3.5.0
@@ -693,9 +719,9 @@ object functions {
/**
* Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch, generated by merging previously created Datasketches HllSketch instances
- * via a Datasketches Union instance. Throws an exception if sketches have different
- * lgConfigK values and allowDifferentLgConfigK is set to false.
+ * HllSketch, generated by merging previously created Datasketches HllSketch instances via a
+ * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values
+ * and allowDifferentLgConfigK is set to false.
*
* @group agg_funcs
* @since 3.5.0
@@ -705,9 +731,9 @@ object functions {
/**
* Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch, generated by merging previously created Datasketches HllSketch instances
- * via a Datasketches Union instance. Throws an exception if sketches have different
- * lgConfigK values and allowDifferentLgConfigK is set to false.
+ * HllSketch, generated by merging previously created Datasketches HllSketch instances via a
+ * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values
+ * and allowDifferentLgConfigK is set to false.
*
* @group agg_funcs
* @since 3.5.0
@@ -718,9 +744,8 @@ object functions {
/**
* Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch, generated by merging previously created Datasketches HllSketch instances
- * via a Datasketches Union instance. Throws an exception if sketches have different
- * lgConfigK values.
+ * HllSketch, generated by merging previously created Datasketches HllSketch instances via a
+ * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values.
*
* @group agg_funcs
* @since 3.5.0
@@ -730,9 +755,8 @@ object functions {
/**
* Aggregate function: returns the updatable binary representation of the Datasketches
- * HllSketch, generated by merging previously created Datasketches HllSketch instances
- * via a Datasketches Union instance. Throws an exception if sketches have different
- * lgConfigK values.
+ * HllSketch, generated by merging previously created Datasketches HllSketch instances via a
+ * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values.
*
* @group agg_funcs
* @since 3.5.0
@@ -763,8 +787,9 @@ object functions {
* The function by default returns the last values it sees. It will return the last non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 2.0.0
@@ -778,8 +803,9 @@ object functions {
* The function by default returns the last values it sees. It will return the last non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 2.0.0
@@ -794,8 +820,9 @@ object functions {
* The function by default returns the last values it sees. It will return the last non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 1.3.0
@@ -808,8 +835,9 @@ object functions {
* The function by default returns the last values it sees. It will return the last non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 1.3.0
@@ -819,8 +847,9 @@ object functions {
/**
* Aggregate function: returns the last value in a group.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 3.5.0
@@ -833,8 +862,9 @@ object functions {
* The function by default returns the last values it sees. It will return the last non-null
* value it sees when ignoreNulls is set to true. If all values are null, then null is returned.
*
- * @note The function is non-deterministic because its results depends on the order of the rows
- * which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because its results depends on the order of the rows
+ * which may be non-deterministic after a shuffle.
*
* @group agg_funcs
* @since 3.5.0
@@ -881,8 +911,9 @@ object functions {
/**
* Aggregate function: returns the value associated with the maximum value of ord.
*
- * @note The function is non-deterministic so the output order can be different for
- * those associated the same values of `e`.
+ * @note
+ * The function is non-deterministic so the output order can be different for those associated
+ * the same values of `e`.
*
* @group agg_funcs
* @since 3.3.0
@@ -890,8 +921,7 @@ object functions {
def max_by(e: Column, ord: Column): Column = Column.fn("max_by", e, ord)
/**
- * Aggregate function: returns the average of the values in a group.
- * Alias for avg.
+ * Aggregate function: returns the average of the values in a group. Alias for avg.
*
* @group agg_funcs
* @since 1.4.0
@@ -899,8 +929,7 @@ object functions {
def mean(e: Column): Column = avg(e)
/**
- * Aggregate function: returns the average of the values in a group.
- * Alias for avg.
+ * Aggregate function: returns the average of the values in a group. Alias for avg.
*
* @group agg_funcs
* @since 1.4.0
@@ -934,8 +963,9 @@ object functions {
/**
* Aggregate function: returns the value associated with the minimum value of ord.
*
- * @note The function is non-deterministic so the output order can be different for
- * those associated the same values of `e`.
+ * @note
+ * The function is non-deterministic so the output order can be different for those associated
+ * the same values of `e`.
*
* @group agg_funcs
* @since 3.3.0
@@ -943,8 +973,8 @@ object functions {
def min_by(e: Column, ord: Column): Column = Column.fn("min_by", e, ord)
/**
- * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the
- * given percentage(s) with value range in [0.0, 1.0].
+ * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given
+ * percentage(s) with value range in [0.0, 1.0].
*
* @group agg_funcs
* @since 3.5.0
@@ -952,8 +982,8 @@ object functions {
def percentile(e: Column, percentage: Column): Column = Column.fn("percentile", e, percentage)
/**
- * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the
- * given percentage(s) with value range in [0.0, 1.0].
+ * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given
+ * percentage(s) with value range in [0.0, 1.0].
*
* @group agg_funcs
* @since 3.5.0
@@ -962,17 +992,16 @@ object functions {
Column.fn("percentile", e, percentage, frequency)
/**
- * Aggregate function: returns the approximate `percentile` of the numeric column `col` which
- * is the smallest value in the ordered `col` values (sorted from least to greatest) such that
- * no more than `percentage` of `col` values is less than the value or equal to that value.
+ * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is
+ * the smallest value in the ordered `col` values (sorted from least to greatest) such that no
+ * more than `percentage` of `col` values is less than the value or equal to that value.
*
- * If percentage is an array, each value must be between 0.0 and 1.0.
- * If it is a single floating point value, it must be between 0.0 and 1.0.
+ * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating
+ * point value, it must be between 0.0 and 1.0.
*
- * The accuracy parameter is a positive numeric literal
- * which controls approximation accuracy at the cost of memory.
- * Higher value of accuracy yields better accuracy, 1.0/accuracy
- * is the relative error of the approximation.
+ * The accuracy parameter is a positive numeric literal which controls approximation accuracy at
+ * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the
+ * relative error of the approximation.
*
* @group agg_funcs
* @since 3.1.0
@@ -981,17 +1010,16 @@ object functions {
Column.fn("percentile_approx", e, percentage, accuracy)
/**
- * Aggregate function: returns the approximate `percentile` of the numeric column `col` which
- * is the smallest value in the ordered `col` values (sorted from least to greatest) such that
- * no more than `percentage` of `col` values is less than the value or equal to that value.
+ * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is
+ * the smallest value in the ordered `col` values (sorted from least to greatest) such that no
+ * more than `percentage` of `col` values is less than the value or equal to that value.
*
- * If percentage is an array, each value must be between 0.0 and 1.0.
- * If it is a single floating point value, it must be between 0.0 and 1.0.
+ * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating
+ * point value, it must be between 0.0 and 1.0.
*
- * The accuracy parameter is a positive numeric literal
- * which controls approximation accuracy at the cost of memory.
- * Higher value of accuracy yields better accuracy, 1.0/accuracy
- * is the relative error of the approximation.
+ * The accuracy parameter is a positive numeric literal which controls approximation accuracy at
+ * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the
+ * relative error of the approximation.
*
* @group agg_funcs
* @since 3.5.0
@@ -1049,8 +1077,7 @@ object functions {
def stddev(columnName: String): Column = stddev(Column(columnName))
/**
- * Aggregate function: returns the sample standard deviation of
- * the expression in a group.
+ * Aggregate function: returns the sample standard deviation of the expression in a group.
*
* @group agg_funcs
* @since 1.6.0
@@ -1058,8 +1085,7 @@ object functions {
def stddev_samp(e: Column): Column = Column.fn("stddev_samp", e)
/**
- * Aggregate function: returns the sample standard deviation of
- * the expression in a group.
+ * Aggregate function: returns the sample standard deviation of the expression in a group.
*
* @group agg_funcs
* @since 1.6.0
@@ -1067,8 +1093,7 @@ object functions {
def stddev_samp(columnName: String): Column = stddev_samp(Column(columnName))
/**
- * Aggregate function: returns the population standard deviation of
- * the expression in a group.
+ * Aggregate function: returns the population standard deviation of the expression in a group.
*
* @group agg_funcs
* @since 1.6.0
@@ -1076,8 +1101,7 @@ object functions {
def stddev_pop(e: Column): Column = Column.fn("stddev_pop", e)
/**
- * Aggregate function: returns the population standard deviation of
- * the expression in a group.
+ * Aggregate function: returns the population standard deviation of the expression in a group.
*
* @group agg_funcs
* @since 1.6.0
@@ -1175,8 +1199,8 @@ object functions {
def var_pop(columnName: String): Column = var_pop(Column(columnName))
/**
- * Aggregate function: returns the average of the independent variable for non-null pairs
- * in a group, where `y` is the dependent variable and `x` is the independent variable.
+ * Aggregate function: returns the average of the independent variable for non-null pairs in a
+ * group, where `y` is the dependent variable and `x` is the independent variable.
*
* @group agg_funcs
* @since 3.5.0
@@ -1184,8 +1208,8 @@ object functions {
def regr_avgx(y: Column, x: Column): Column = Column.fn("regr_avgx", y, x)
/**
- * Aggregate function: returns the average of the independent variable for non-null pairs
- * in a group, where `y` is the dependent variable and `x` is the independent variable.
+ * Aggregate function: returns the average of the independent variable for non-null pairs in a
+ * group, where `y` is the dependent variable and `x` is the independent variable.
*
* @group agg_funcs
* @since 3.5.0
@@ -1193,8 +1217,8 @@ object functions {
def regr_avgy(y: Column, x: Column): Column = Column.fn("regr_avgy", y, x)
/**
- * Aggregate function: returns the number of non-null number pairs
- * in a group, where `y` is the dependent variable and `x` is the independent variable.
+ * Aggregate function: returns the number of non-null number pairs in a group, where `y` is the
+ * dependent variable and `x` is the independent variable.
*
* @group agg_funcs
* @since 3.5.0
@@ -1202,9 +1226,9 @@ object functions {
def regr_count(y: Column, x: Column): Column = Column.fn("regr_count", y, x)
/**
- * Aggregate function: returns the intercept of the univariate linear regression line
- * for non-null pairs in a group, where `y` is the dependent variable and
- * `x` is the independent variable.
+ * Aggregate function: returns the intercept of the univariate linear regression line for
+ * non-null pairs in a group, where `y` is the dependent variable and `x` is the independent
+ * variable.
*
* @group agg_funcs
* @since 3.5.0
@@ -1212,8 +1236,8 @@ object functions {
def regr_intercept(y: Column, x: Column): Column = Column.fn("regr_intercept", y, x)
/**
- * Aggregate function: returns the coefficient of determination for non-null pairs
- * in a group, where `y` is the dependent variable and `x` is the independent variable.
+ * Aggregate function: returns the coefficient of determination for non-null pairs in a group,
+ * where `y` is the dependent variable and `x` is the independent variable.
*
* @group agg_funcs
* @since 3.5.0
@@ -1221,8 +1245,8 @@ object functions {
def regr_r2(y: Column, x: Column): Column = Column.fn("regr_r2", y, x)
/**
- * Aggregate function: returns the slope of the linear regression line for non-null pairs
- * in a group, where `y` is the dependent variable and `x` is the independent variable.
+ * Aggregate function: returns the slope of the linear regression line for non-null pairs in a
+ * group, where `y` is the dependent variable and `x` is the independent variable.
*
* @group agg_funcs
* @since 3.5.0
@@ -1230,8 +1254,8 @@ object functions {
def regr_slope(y: Column, x: Column): Column = Column.fn("regr_slope", y, x)
/**
- * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs
- * in a group, where `y` is the dependent variable and `x` is the independent variable.
+ * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group,
+ * where `y` is the dependent variable and `x` is the independent variable.
*
* @group agg_funcs
* @since 3.5.0
@@ -1239,8 +1263,8 @@ object functions {
def regr_sxx(y: Column, x: Column): Column = Column.fn("regr_sxx", y, x)
/**
- * Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs
- * in a group, where `y` is the dependent variable and `x` is the independent variable.
+ * Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group,
+ * where `y` is the dependent variable and `x` is the independent variable.
*
* @group agg_funcs
* @since 3.5.0
@@ -1248,8 +1272,8 @@ object functions {
def regr_sxy(y: Column, x: Column): Column = Column.fn("regr_sxy", y, x)
/**
- * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs
- * in a group, where `y` is the dependent variable and `x` is the independent variable.
+ * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group,
+ * where `y` is the dependent variable and `x` is the independent variable.
*
* @group agg_funcs
* @since 3.5.0
@@ -1265,8 +1289,8 @@ object functions {
def any_value(e: Column): Column = Column.fn("any_value", e)
/**
- * Aggregate function: returns some value of `e` for a group of rows.
- * If `isIgnoreNull` is true, returns only non-null values.
+ * Aggregate function: returns some value of `e` for a group of rows. If `isIgnoreNull` is true,
+ * returns only non-null values.
*
* @group agg_funcs
* @since 3.5.0
@@ -1283,13 +1307,12 @@ object functions {
def count_if(e: Column): Column = Column.fn("count_if", e)
/**
- * Aggregate function: computes a histogram on numeric 'expr' using nb bins.
- * The return value is an array of (x,y) pairs representing the centers of the
- * histogram's bins. As the value of 'nb' is increased, the histogram approximation
- * gets finer-grained, but may yield artifacts around outliers. In practice, 20-40
- * histogram bins appear to work well, with more bins being required for skewed or
- * smaller datasets. Note that this function creates a histogram with non-uniform
- * bin widths. It offers no guarantees in terms of the mean-squared-error of the
+ * Aggregate function: computes a histogram on numeric 'expr' using nb bins. The return value is
+ * an array of (x,y) pairs representing the centers of the histogram's bins. As the value of
+ * 'nb' is increased, the histogram approximation gets finer-grained, but may yield artifacts
+ * around outliers. In practice, 20-40 histogram bins appear to work well, with more bins being
+ * required for skewed or smaller datasets. Note that this function creates a histogram with
+ * non-uniform bin widths. It offers no guarantees in terms of the mean-squared-error of the
* histogram, but in practice is comparable to the histograms produced by the R/S-Plus
* statistical computing packages. Note: the output type of the 'x' field in the return value is
* propagated from the input value consumed in the aggregate function.
@@ -1386,10 +1409,10 @@ object functions {
* Window function: returns the rank of rows within a window partition, without any gaps.
*
* The difference between rank and dense_rank is that denseRank leaves no gaps in ranking
- * sequence when there are ties. That is, if you were ranking a competition using dense_rank
- * and had three people tie for second place, you would say that all three were in second
- * place and that the next person came in third. Rank would give me sequential numbers, making
- * the person that came in third place (after the ties) would register as coming in fifth.
+ * sequence when there are ties. That is, if you were ranking a competition using dense_rank and
+ * had three people tie for second place, you would say that all three were in second place and
+ * that the next person came in third. Rank would give me sequential numbers, making the person
+ * that came in third place (after the ties) would register as coming in fifth.
*
* This is equivalent to the DENSE_RANK function in SQL.
*
@@ -1399,9 +1422,9 @@ object functions {
def dense_rank(): Column = Column.fn("dense_rank")
/**
- * Window function: returns the value that is `offset` rows before the current row, and
- * `null` if there is less than `offset` rows before the current row. For example,
- * an `offset` of one will return the previous row at any given point in the window partition.
+ * Window function: returns the value that is `offset` rows before the current row, and `null`
+ * if there is less than `offset` rows before the current row. For example, an `offset` of one
+ * will return the previous row at any given point in the window partition.
*
* This is equivalent to the LAG function in SQL.
*
@@ -1411,9 +1434,9 @@ object functions {
def lag(e: Column, offset: Int): Column = lag(e, offset, null)
/**
- * Window function: returns the value that is `offset` rows before the current row, and
- * `null` if there is less than `offset` rows before the current row. For example,
- * an `offset` of one will return the previous row at any given point in the window partition.
+ * Window function: returns the value that is `offset` rows before the current row, and `null`
+ * if there is less than `offset` rows before the current row. For example, an `offset` of one
+ * will return the previous row at any given point in the window partition.
*
* This is equivalent to the LAG function in SQL.
*
@@ -1424,8 +1447,8 @@ object functions {
/**
* Window function: returns the value that is `offset` rows before the current row, and
- * `defaultValue` if there is less than `offset` rows before the current row. For example,
- * an `offset` of one will return the previous row at any given point in the window partition.
+ * `defaultValue` if there is less than `offset` rows before the current row. For example, an
+ * `offset` of one will return the previous row at any given point in the window partition.
*
* This is equivalent to the LAG function in SQL.
*
@@ -1438,8 +1461,8 @@ object functions {
/**
* Window function: returns the value that is `offset` rows before the current row, and
- * `defaultValue` if there is less than `offset` rows before the current row. For example,
- * an `offset` of one will return the previous row at any given point in the window partition.
+ * `defaultValue` if there is less than `offset` rows before the current row. For example, an
+ * `offset` of one will return the previous row at any given point in the window partition.
*
* This is equivalent to the LAG function in SQL.
*
@@ -1453,9 +1476,9 @@ object functions {
/**
* Window function: returns the value that is `offset` rows before the current row, and
* `defaultValue` if there is less than `offset` rows before the current row. `ignoreNulls`
- * determines whether null values of row are included in or eliminated from the calculation.
- * For example, an `offset` of one will return the previous row at any given point in the
- * window partition.
+ * determines whether null values of row are included in or eliminated from the calculation. For
+ * example, an `offset` of one will return the previous row at any given point in the window
+ * partition.
*
* This is equivalent to the LAG function in SQL.
*
@@ -1466,9 +1489,9 @@ object functions {
Column.fn("lag", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls))
/**
- * Window function: returns the value that is `offset` rows after the current row, and
- * `null` if there is less than `offset` rows after the current row. For example,
- * an `offset` of one will return the next row at any given point in the window partition.
+ * Window function: returns the value that is `offset` rows after the current row, and `null` if
+ * there is less than `offset` rows after the current row. For example, an `offset` of one will
+ * return the next row at any given point in the window partition.
*
* This is equivalent to the LEAD function in SQL.
*
@@ -1478,9 +1501,9 @@ object functions {
def lead(columnName: String, offset: Int): Column = { lead(columnName, offset, null) }
/**
- * Window function: returns the value that is `offset` rows after the current row, and
- * `null` if there is less than `offset` rows after the current row. For example,
- * an `offset` of one will return the next row at any given point in the window partition.
+ * Window function: returns the value that is `offset` rows after the current row, and `null` if
+ * there is less than `offset` rows after the current row. For example, an `offset` of one will
+ * return the next row at any given point in the window partition.
*
* This is equivalent to the LEAD function in SQL.
*
@@ -1491,8 +1514,8 @@ object functions {
/**
* Window function: returns the value that is `offset` rows after the current row, and
- * `defaultValue` if there is less than `offset` rows after the current row. For example,
- * an `offset` of one will return the next row at any given point in the window partition.
+ * `defaultValue` if there is less than `offset` rows after the current row. For example, an
+ * `offset` of one will return the next row at any given point in the window partition.
*
* This is equivalent to the LEAD function in SQL.
*
@@ -1505,8 +1528,8 @@ object functions {
/**
* Window function: returns the value that is `offset` rows after the current row, and
- * `defaultValue` if there is less than `offset` rows after the current row. For example,
- * an `offset` of one will return the next row at any given point in the window partition.
+ * `defaultValue` if there is less than `offset` rows after the current row. For example, an
+ * `offset` of one will return the next row at any given point in the window partition.
*
* This is equivalent to the LEAD function in SQL.
*
@@ -1520,9 +1543,9 @@ object functions {
/**
* Window function: returns the value that is `offset` rows after the current row, and
* `defaultValue` if there is less than `offset` rows after the current row. `ignoreNulls`
- * determines whether null values of row are included in or eliminated from the calculation.
- * The default value of `ignoreNulls` is false. For example, an `offset` of one will return
- * the next row at any given point in the window partition.
+ * determines whether null values of row are included in or eliminated from the calculation. The
+ * default value of `ignoreNulls` is false. For example, an `offset` of one will return the next
+ * row at any given point in the window partition.
*
* This is equivalent to the LEAD function in SQL.
*
@@ -1533,11 +1556,11 @@ object functions {
Column.fn("lead", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls))
/**
- * Window function: returns the value that is the `offset`th row of the window frame
- * (counting from 1), and `null` if the size of window frame is less than `offset` rows.
+ * Window function: returns the value that is the `offset`th row of the window frame (counting
+ * from 1), and `null` if the size of window frame is less than `offset` rows.
*
- * It will return the `offset`th non-null value it sees when ignoreNulls is set to true.
- * If all values are null, then null is returned.
+ * It will return the `offset`th non-null value it sees when ignoreNulls is set to true. If all
+ * values are null, then null is returned.
*
* This is equivalent to the nth_value function in SQL.
*
@@ -1548,8 +1571,8 @@ object functions {
Column.fn("nth_value", false, e, lit(offset), lit(ignoreNulls))
/**
- * Window function: returns the value that is the `offset`th row of the window frame
- * (counting from 1), and `null` if the size of window frame is less than `offset` rows.
+ * Window function: returns the value that is the `offset`th row of the window frame (counting
+ * from 1), and `null` if the size of window frame is less than `offset` rows.
*
* This is equivalent to the nth_value function in SQL.
*
@@ -1560,8 +1583,8 @@ object functions {
/**
* Window function: returns the ntile group id (from 1 to `n` inclusive) in an ordered window
- * partition. For example, if `n` is 4, the first quarter of the rows will get value 1, the second
- * quarter will get 2, the third quarter will get 3, and the last quarter will get 4.
+ * partition. For example, if `n` is 4, the first quarter of the rows will get value 1, the
+ * second quarter will get 2, the third quarter will get 3, and the last quarter will get 4.
*
* This is equivalent to the NTILE function in SQL.
*
@@ -1571,7 +1594,8 @@ object functions {
def ntile(n: Int): Column = Column.fn("ntile", lit(n))
/**
- * Window function: returns the relative rank (i.e. percentile) of rows within a window partition.
+ * Window function: returns the relative rank (i.e. percentile) of rows within a window
+ * partition.
*
* This is computed by:
* {{{
@@ -1589,10 +1613,10 @@ object functions {
* Window function: returns the rank of rows within a window partition.
*
* The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking
- * sequence when there are ties. That is, if you were ranking a competition using dense_rank
- * and had three people tie for second place, you would say that all three were in second
- * place and that the next person came in third. Rank would give me sequential numbers, making
- * the person that came in third place (after the ties) would register as coming in fifth.
+ * sequence when there are ties. That is, if you were ranking a competition using dense_rank and
+ * had three people tie for second place, you would say that all three were in second place and
+ * that the next person came in third. Rank would give me sequential numbers, making the person
+ * that came in third place (after the ties) would register as coming in fifth.
*
* This is equivalent to the RANK function in SQL.
*
@@ -1630,13 +1654,13 @@ object functions {
*/
@scala.annotation.varargs
def array(colName: String, colNames: String*): Column = {
- array((colName +: colNames).map(col) : _*)
+ array((colName +: colNames).map(col): _*)
}
/**
- * Creates a new map column. The input columns must be grouped as key-value pairs, e.g.
- * (key1, value1, key2, value2, ...). The key columns must all have the same data type, and can't
- * be null. The value columns must all have the same data type.
+ * Creates a new map column. The input columns must be grouped as key-value pairs, e.g. (key1,
+ * value1, key2, value2, ...). The key columns must all have the same data type, and can't be
+ * null. The value columns must all have the same data type.
*
* @group map_funcs
* @since 2.0
@@ -1663,8 +1687,8 @@ object functions {
Column.fn("map_from_arrays", keys, values)
/**
- * Creates a map after splitting the text into key/value pairs using delimiters.
- * Both `pairDelim` and `keyValueDelim` are treated as regular expressions.
+ * Creates a map after splitting the text into key/value pairs using delimiters. Both
+ * `pairDelim` and `keyValueDelim` are treated as regular expressions.
*
* @group map_funcs
* @since 3.5.0
@@ -1673,8 +1697,8 @@ object functions {
Column.fn("str_to_map", text, pairDelim, keyValueDelim)
/**
- * Creates a map after splitting the text into key/value pairs using delimiters.
- * The `pairDelim` is treated as regular expressions.
+ * Creates a map after splitting the text into key/value pairs using delimiters. The `pairDelim`
+ * is treated as regular expressions.
*
* @group map_funcs
* @since 3.5.0
@@ -1702,15 +1726,15 @@ object functions {
* @group normal_funcs
* @since 1.5.0
*/
- def broadcast[DS[U] <: api.Dataset[U, DS]](df: DS[_]): df.type = {
+ def broadcast[DS[U] <: api.Dataset[U]](df: DS[_]): df.type = {
df.hint("broadcast").asInstanceOf[df.type]
}
/**
* Returns the first column that is not null, or null if all inputs are null.
*
- * For example, `coalesce(a, b, c)` will return a if a is not null,
- * or b if a is null and b is not null, or c if both a and b are null but c is not null.
+ * For example, `coalesce(a, b, c)` will return a if a is not null, or b if a is null and b is
+ * not null, or c if both a and b are null but c is not null.
*
* @group conditional_funcs
* @since 1.3.0
@@ -1745,13 +1769,13 @@ object functions {
/**
* A column expression that generates monotonically increasing 64-bit integers.
*
- * The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive.
- * The current implementation puts the partition ID in the upper 31 bits, and the record number
- * within each partition in the lower 33 bits. The assumption is that the data frame has
- * less than 1 billion partitions, and each partition has less than 8 billion records.
+ * The generated ID is guaranteed to be monotonically increasing and unique, but not
+ * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the
+ * record number within each partition in the lower 33 bits. The assumption is that the data
+ * frame has less than 1 billion partitions, and each partition has less than 8 billion records.
*
- * As an example, consider a `DataFrame` with two partitions, each with 3 records.
- * This expression would return the following IDs:
+ * As an example, consider a `DataFrame` with two partitions, each with 3 records. This
+ * expression would return the following IDs:
*
* {{{
* 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
@@ -1766,13 +1790,13 @@ object functions {
/**
* A column expression that generates monotonically increasing 64-bit integers.
*
- * The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive.
- * The current implementation puts the partition ID in the upper 31 bits, and the record number
- * within each partition in the lower 33 bits. The assumption is that the data frame has
- * less than 1 billion partitions, and each partition has less than 8 billion records.
+ * The generated ID is guaranteed to be monotonically increasing and unique, but not
+ * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the
+ * record number within each partition in the lower 33 bits. The assumption is that the data
+ * frame has less than 1 billion partitions, and each partition has less than 8 billion records.
*
- * As an example, consider a `DataFrame` with two partitions, each with 3 records.
- * This expression would return the following IDs:
+ * As an example, consider a `DataFrame` with two partitions, each with 3 records. This
+ * expression would return the following IDs:
*
* {{{
* 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
@@ -1828,7 +1852,8 @@ object functions {
* Generate a random column with independent and identically distributed (i.i.d.) samples
* uniformly distributed in [0.0, 1.0).
*
- * @note The function is non-deterministic in general case.
+ * @note
+ * The function is non-deterministic in general case.
*
* @group math_funcs
* @since 1.4.0
@@ -1839,7 +1864,8 @@ object functions {
* Generate a random column with independent and identically distributed (i.i.d.) samples
* uniformly distributed in [0.0, 1.0).
*
- * @note The function is non-deterministic in general case.
+ * @note
+ * The function is non-deterministic in general case.
*
* @group math_funcs
* @since 1.4.0
@@ -1847,10 +1873,11 @@ object functions {
def rand(): Column = rand(SparkClassUtils.random.nextLong)
/**
- * Generate a column with independent and identically distributed (i.i.d.) samples from
- * the standard normal distribution.
+ * Generate a column with independent and identically distributed (i.i.d.) samples from the
+ * standard normal distribution.
*
- * @note The function is non-deterministic in general case.
+ * @note
+ * The function is non-deterministic in general case.
*
* @group math_funcs
* @since 1.4.0
@@ -1858,10 +1885,11 @@ object functions {
def randn(seed: Long): Column = Column.fn("randn", lit(seed))
/**
- * Generate a column with independent and identically distributed (i.i.d.) samples from
- * the standard normal distribution.
+ * Generate a column with independent and identically distributed (i.i.d.) samples from the
+ * standard normal distribution.
*
- * @note The function is non-deterministic in general case.
+ * @note
+ * The function is non-deterministic in general case.
*
* @group math_funcs
* @since 1.4.0
@@ -1871,7 +1899,8 @@ object functions {
/**
* Partition ID.
*
- * @note This is non-deterministic because it depends on data partitioning and task scheduling.
+ * @note
+ * This is non-deterministic because it depends on data partitioning and task scheduling.
*
* @group misc_funcs
* @since 1.6.0
@@ -1921,8 +1950,7 @@ object functions {
def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right)
/**
- * Returns the remainder of `dividend``/``divisor`. Its result is
- * always null if `divisor` is 0.
+ * Returns the remainder of `dividend``/``divisor`. Its result is always null if `divisor` is 0.
*
* @group math_funcs
* @since 4.0.0
@@ -1956,11 +1984,10 @@ object functions {
def try_sum(e: Column): Column = Column.fn("try_sum", e)
/**
- * Creates a new struct column.
- * If the input column is a column in a `DataFrame`, or a derived column expression
- * that is named (i.e. aliased), its name would be retained as the StructField's name,
- * otherwise, the newly generated StructField's name would be auto generated as
- * `col` with a suffix `index + 1`, i.e. col1, col2, col3, ...
+ * Creates a new struct column. If the input column is a column in a `DataFrame`, or a derived
+ * column expression that is named (i.e. aliased), its name would be retained as the
+ * StructField's name, otherwise, the newly generated StructField's name would be auto generated
+ * as `col` with a suffix `index + 1`, i.e. col1, col2, col3, ...
*
* @group struct_funcs
* @since 1.4.0
@@ -1976,12 +2003,12 @@ object functions {
*/
@scala.annotation.varargs
def struct(colName: String, colNames: String*): Column = {
- struct((colName +: colNames).map(col) : _*)
+ struct((colName +: colNames).map(col): _*)
}
/**
- * Evaluates a list of conditions and returns one of multiple possible result expressions.
- * If otherwise is not defined at the end, null is returned for unmatched conditions.
+ * Evaluates a list of conditions and returns one of multiple possible result expressions. If
+ * otherwise is not defined at the end, null is returned for unmatched conditions.
*
* {{{
* // Example: encoding gender string column into integer.
@@ -2030,9 +2057,8 @@ object functions {
def bit_count(e: Column): Column = Column.fn("bit_count", e)
/**
- * Returns the value of the bit (0 or 1) at the specified position.
- * The positions are numbered from right to left, starting at zero.
- * The position argument cannot be negative.
+ * Returns the value of the bit (0 or 1) at the specified position. The positions are numbered
+ * from right to left, starting at zero. The position argument cannot be negative.
*
* @group bitwise_funcs
* @since 3.5.0
@@ -2040,9 +2066,8 @@ object functions {
def bit_get(e: Column, pos: Column): Column = Column.fn("bit_get", e, pos)
/**
- * Returns the value of the bit (0 or 1) at the specified position.
- * The positions are numbered from right to left, starting at zero.
- * The position argument cannot be negative.
+ * Returns the value of the bit (0 or 1) at the specified position. The positions are numbered
+ * from right to left, starting at zero. The position argument cannot be negative.
*
* @group bitwise_funcs
* @since 3.5.0
@@ -2074,7 +2099,8 @@ object functions {
def abs(e: Column): Column = Column.fn("abs", e)
/**
- * @return inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`
+ * @return
+ * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`
*
* @group math_funcs
* @since 1.4.0
@@ -2082,7 +2108,8 @@ object functions {
def acos(e: Column): Column = Column.fn("acos", e)
/**
- * @return inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`
+ * @return
+ * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`
*
* @group math_funcs
* @since 1.4.0
@@ -2090,7 +2117,8 @@ object functions {
def acos(columnName: String): Column = acos(Column(columnName))
/**
- * @return inverse hyperbolic cosine of `e`
+ * @return
+ * inverse hyperbolic cosine of `e`
*
* @group math_funcs
* @since 3.1.0
@@ -2098,7 +2126,8 @@ object functions {
def acosh(e: Column): Column = Column.fn("acosh", e)
/**
- * @return inverse hyperbolic cosine of `columnName`
+ * @return
+ * inverse hyperbolic cosine of `columnName`
*
* @group math_funcs
* @since 3.1.0
@@ -2106,7 +2135,8 @@ object functions {
def acosh(columnName: String): Column = acosh(Column(columnName))
/**
- * @return inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`
+ * @return
+ * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`
*
* @group math_funcs
* @since 1.4.0
@@ -2114,7 +2144,8 @@ object functions {
def asin(e: Column): Column = Column.fn("asin", e)
/**
- * @return inverse sine of `columnName`, as if computed by `java.lang.Math.asin`
+ * @return
+ * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`
*
* @group math_funcs
* @since 1.4.0
@@ -2122,7 +2153,8 @@ object functions {
def asin(columnName: String): Column = asin(Column(columnName))
/**
- * @return inverse hyperbolic sine of `e`
+ * @return
+ * inverse hyperbolic sine of `e`
*
* @group math_funcs
* @since 3.1.0
@@ -2130,7 +2162,8 @@ object functions {
def asinh(e: Column): Column = Column.fn("asinh", e)
/**
- * @return inverse hyperbolic sine of `columnName`
+ * @return
+ * inverse hyperbolic sine of `columnName`
*
* @group math_funcs
* @since 3.1.0
@@ -2138,7 +2171,8 @@ object functions {
def asinh(columnName: String): Column = asinh(Column(columnName))
/**
- * @return inverse tangent of `e` as if computed by `java.lang.Math.atan`
+ * @return
+ * inverse tangent of `e` as if computed by `java.lang.Math.atan`
*
* @group math_funcs
* @since 1.4.0
@@ -2146,7 +2180,8 @@ object functions {
def atan(e: Column): Column = Column.fn("atan", e)
/**
- * @return inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`
+ * @return
+ * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`
*
* @group math_funcs
* @since 1.4.0
@@ -2154,13 +2189,14 @@ object functions {
def atan(columnName: String): Column = atan(Column(columnName))
/**
- * @param y coordinate on y-axis
- * @param x coordinate on x-axis
- * @return the theta component of the point
- * (r, theta)
- * in polar coordinates that corresponds to the point
- * (x, y) in Cartesian coordinates,
- * as if computed by `java.lang.Math.atan2`
+ * @param y
+ * coordinate on y-axis
+ * @param x
+ * coordinate on x-axis
+ * @return
+ * the theta component of the point (r, theta) in polar coordinates that
+ * corresponds to the point (x, y) in Cartesian coordinates, as if computed by
+ * `java.lang.Math.atan2`
*
* @group math_funcs
* @since 1.4.0
@@ -2168,13 +2204,14 @@ object functions {
def atan2(y: Column, x: Column): Column = Column.fn("atan2", y, x)
/**
- * @param y coordinate on y-axis
- * @param xName coordinate on x-axis
- * @return the theta component of the point
- * (r, theta)
- * in polar coordinates that corresponds to the point
- * (x, y) in Cartesian coordinates,
- * as if computed by `java.lang.Math.atan2`
+ * @param y
+ * coordinate on y-axis
+ * @param xName
+ * coordinate on x-axis
+ * @return
+ * the theta component of the point (r, theta) in polar coordinates that
+ * corresponds to the point (x, y) in Cartesian coordinates, as if computed by
+ * `java.lang.Math.atan2`
*
* @group math_funcs
* @since 1.4.0
@@ -2182,13 +2219,14 @@ object functions {
def atan2(y: Column, xName: String): Column = atan2(y, Column(xName))
/**
- * @param yName coordinate on y-axis
- * @param x coordinate on x-axis
- * @return the theta component of the point
- * (r, theta)
- * in polar coordinates that corresponds to the point
- * (x, y) in Cartesian coordinates,
- * as if computed by `java.lang.Math.atan2`
+ * @param yName
+ * coordinate on y-axis
+ * @param x
+ * coordinate on x-axis
+ * @return
+ * the theta component of the point (r, theta) in polar coordinates that
+ * corresponds to the point (x, y) in Cartesian coordinates, as if computed by
+ * `java.lang.Math.atan2`
*
* @group math_funcs
* @since 1.4.0
@@ -2196,13 +2234,14 @@ object functions {
def atan2(yName: String, x: Column): Column = atan2(Column(yName), x)
/**
- * @param yName coordinate on y-axis
- * @param xName coordinate on x-axis
- * @return the theta component of the point
- * (r, theta)
- * in polar coordinates that corresponds to the point
- * (x, y) in Cartesian coordinates,
- * as if computed by `java.lang.Math.atan2`
+ * @param yName
+ * coordinate on y-axis
+ * @param xName
+ * coordinate on x-axis
+ * @return
+ * the theta component of the point (r, theta) in polar coordinates that
+ * corresponds to the point (x, y) in Cartesian coordinates, as if computed by
+ * `java.lang.Math.atan2`
*
* @group math_funcs
* @since 1.4.0
@@ -2211,13 +2250,14 @@ object functions {
atan2(Column(yName), Column(xName))
/**
- * @param y coordinate on y-axis
- * @param xValue coordinate on x-axis
- * @return the theta component of the point
- * (r, theta)
- * in polar coordinates that corresponds to the point
- * (x, y) in Cartesian coordinates,
- * as if computed by `java.lang.Math.atan2`
+ * @param y
+ * coordinate on y-axis
+ * @param xValue
+ * coordinate on x-axis
+ * @return
+ * the theta component of the point (r, theta) in polar coordinates that
+ * corresponds to the point (x, y) in Cartesian coordinates, as if computed by
+ * `java.lang.Math.atan2`
*
* @group math_funcs
* @since 1.4.0
@@ -2225,13 +2265,14 @@ object functions {
def atan2(y: Column, xValue: Double): Column = atan2(y, lit(xValue))
/**
- * @param yName coordinate on y-axis
- * @param xValue coordinate on x-axis
- * @return the theta component of the point
- * (r, theta)
- * in polar coordinates that corresponds to the point
- * (x, y) in Cartesian coordinates,
- * as if computed by `java.lang.Math.atan2`
+ * @param yName
+ * coordinate on y-axis
+ * @param xValue
+ * coordinate on x-axis
+ * @return
+ * the theta component of the point (r, theta) in polar coordinates that
+ * corresponds to the point (x, y) in Cartesian coordinates, as if computed by
+ * `java.lang.Math.atan2`
*
* @group math_funcs
* @since 1.4.0
@@ -2239,13 +2280,14 @@ object functions {
def atan2(yName: String, xValue: Double): Column = atan2(Column(yName), xValue)
/**
- * @param yValue coordinate on y-axis
- * @param x coordinate on x-axis
- * @return the theta component of the point
- * (r, theta)
- * in polar coordinates that corresponds to the point
- * (x, y) in Cartesian coordinates,
- * as if computed by `java.lang.Math.atan2`
+ * @param yValue
+ * coordinate on y-axis
+ * @param x
+ * coordinate on x-axis
+ * @return
+ * the theta component of the point (r, theta) in polar coordinates that
+ * corresponds to the point (x, y) in Cartesian coordinates, as if computed by
+ * `java.lang.Math.atan2`
*
* @group math_funcs
* @since 1.4.0
@@ -2253,13 +2295,14 @@ object functions {
def atan2(yValue: Double, x: Column): Column = atan2(lit(yValue), x)
/**
- * @param yValue coordinate on y-axis
- * @param xName coordinate on x-axis
- * @return the theta component of the point
- * (r, theta)
- * in polar coordinates that corresponds to the point
- * (x, y) in Cartesian coordinates,
- * as if computed by `java.lang.Math.atan2`
+ * @param yValue
+ * coordinate on y-axis
+ * @param xName
+ * coordinate on x-axis
+ * @return
+ * the theta component of the point (r, theta) in polar coordinates that
+ * corresponds to the point (x, y) in Cartesian coordinates, as if computed by
+ * `java.lang.Math.atan2`
*
* @group math_funcs
* @since 1.4.0
@@ -2267,7 +2310,8 @@ object functions {
def atan2(yValue: Double, xName: String): Column = atan2(yValue, Column(xName))
/**
- * @return inverse hyperbolic tangent of `e`
+ * @return
+ * inverse hyperbolic tangent of `e`
*
* @group math_funcs
* @since 3.1.0
@@ -2275,7 +2319,8 @@ object functions {
def atanh(e: Column): Column = Column.fn("atanh", e)
/**
- * @return inverse hyperbolic tangent of `columnName`
+ * @return
+ * inverse hyperbolic tangent of `columnName`
*
* @group math_funcs
* @since 3.1.0
@@ -2366,8 +2411,10 @@ object functions {
Column.fn("conv", num, lit(fromBase), lit(toBase))
/**
- * @param e angle in radians
- * @return cosine of the angle, as if computed by `java.lang.Math.cos`
+ * @param e
+ * angle in radians
+ * @return
+ * cosine of the angle, as if computed by `java.lang.Math.cos`
*
* @group math_funcs
* @since 1.4.0
@@ -2375,8 +2422,10 @@ object functions {
def cos(e: Column): Column = Column.fn("cos", e)
/**
- * @param columnName angle in radians
- * @return cosine of the angle, as if computed by `java.lang.Math.cos`
+ * @param columnName
+ * angle in radians
+ * @return
+ * cosine of the angle, as if computed by `java.lang.Math.cos`
*
* @group math_funcs
* @since 1.4.0
@@ -2384,8 +2433,10 @@ object functions {
def cos(columnName: String): Column = cos(Column(columnName))
/**
- * @param e hyperbolic angle
- * @return hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`
+ * @param e
+ * hyperbolic angle
+ * @return
+ * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`
*
* @group math_funcs
* @since 1.4.0
@@ -2393,8 +2444,10 @@ object functions {
def cosh(e: Column): Column = Column.fn("cosh", e)
/**
- * @param columnName hyperbolic angle
- * @return hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`
+ * @param columnName
+ * hyperbolic angle
+ * @return
+ * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`
*
* @group math_funcs
* @since 1.4.0
@@ -2402,8 +2455,10 @@ object functions {
def cosh(columnName: String): Column = cosh(Column(columnName))
/**
- * @param e angle in radians
- * @return cotangent of the angle
+ * @param e
+ * angle in radians
+ * @return
+ * cotangent of the angle
*
* @group math_funcs
* @since 3.3.0
@@ -2411,8 +2466,10 @@ object functions {
def cot(e: Column): Column = Column.fn("cot", e)
/**
- * @param e angle in radians
- * @return cosecant of the angle
+ * @param e
+ * angle in radians
+ * @return
+ * cosecant of the angle
*
* @group math_funcs
* @since 3.3.0
@@ -2492,8 +2549,8 @@ object functions {
def floor(columnName: String): Column = floor(Column(columnName))
/**
- * Returns the greatest value of the list of values, skipping null values.
- * This function takes at least 2 parameters. It will return null iff all parameters are null.
+ * Returns the greatest value of the list of values, skipping null values. This function takes
+ * at least 2 parameters. It will return null iff all parameters are null.
*
* @group math_funcs
* @since 1.5.0
@@ -2502,8 +2559,8 @@ object functions {
def greatest(exprs: Column*): Column = Column.fn("greatest", exprs: _*)
/**
- * Returns the greatest value of the list of column names, skipping null values.
- * This function takes at least 2 parameters. It will return null iff all parameters are null.
+ * Returns the greatest value of the list of column names, skipping null values. This function
+ * takes at least 2 parameters. It will return null iff all parameters are null.
*
* @group math_funcs
* @since 1.5.0
@@ -2522,8 +2579,8 @@ object functions {
def hex(column: Column): Column = Column.fn("hex", column)
/**
- * Inverse of hex. Interprets each pair of characters as a hexadecimal number
- * and converts to the byte representation of number.
+ * Inverse of hex. Interprets each pair of characters as a hexadecimal number and converts to
+ * the byte representation of number.
*
* @group math_funcs
* @since 1.5.0
@@ -2596,8 +2653,8 @@ object functions {
def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName))
/**
- * Returns the least value of the list of values, skipping null values.
- * This function takes at least 2 parameters. It will return null iff all parameters are null.
+ * Returns the least value of the list of values, skipping null values. This function takes at
+ * least 2 parameters. It will return null iff all parameters are null.
*
* @group math_funcs
* @since 1.5.0
@@ -2606,8 +2663,8 @@ object functions {
def least(exprs: Column*): Column = Column.fn("least", exprs: _*)
/**
- * Returns the least value of the list of column names, skipping null values.
- * This function takes at least 2 parameters. It will return null iff all parameters are null.
+ * Returns the least value of the list of column names, skipping null values. This function
+ * takes at least 2 parameters. It will return null iff all parameters are null.
*
* @group math_funcs
* @since 1.5.0
@@ -2810,8 +2867,8 @@ object functions {
def pmod(dividend: Column, divisor: Column): Column = Column.fn("pmod", dividend, divisor)
/**
- * Returns the double value that is closest in value to the argument and
- * is equal to a mathematical integer.
+ * Returns the double value that is closest in value to the argument and is equal to a
+ * mathematical integer.
*
* @group math_funcs
* @since 1.4.0
@@ -2819,8 +2876,8 @@ object functions {
def rint(e: Column): Column = Column.fn("rint", e)
/**
- * Returns the double value that is closest in value to the argument and
- * is equal to a mathematical integer.
+ * Returns the double value that is closest in value to the argument and is equal to a
+ * mathematical integer.
*
* @group math_funcs
* @since 1.4.0
@@ -2836,8 +2893,8 @@ object functions {
def round(e: Column): Column = round(e, 0)
/**
- * Round the value of `e` to `scale` decimal places with HALF_UP round mode
- * if `scale` is greater than or equal to 0 or at integral part when `scale` is less than 0.
+ * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is
+ * greater than or equal to 0 or at integral part when `scale` is less than 0.
*
* @group math_funcs
* @since 1.5.0
@@ -2845,8 +2902,8 @@ object functions {
def round(e: Column, scale: Int): Column = Column.fn("round", e, lit(scale))
/**
- * Round the value of `e` to `scale` decimal places with HALF_UP round mode
- * if `scale` is greater than or equal to 0 or at integral part when `scale` is less than 0.
+ * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is
+ * greater than or equal to 0 or at integral part when `scale` is less than 0.
*
* @group math_funcs
* @since 4.0.0
@@ -2862,8 +2919,8 @@ object functions {
def bround(e: Column): Column = bround(e, 0)
/**
- * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode
- * if `scale` is greater than or equal to 0 or at integral part when `scale` is less than 0.
+ * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is
+ * greater than or equal to 0 or at integral part when `scale` is less than 0.
*
* @group math_funcs
* @since 2.0.0
@@ -2871,8 +2928,8 @@ object functions {
def bround(e: Column, scale: Int): Column = Column.fn("bround", e, lit(scale))
/**
- * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode
- * if `scale` is greater than or equal to 0 or at integral part when `scale` is less than 0.
+ * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is
+ * greater than or equal to 0 or at integral part when `scale` is less than 0.
*
* @group math_funcs
* @since 4.0.0
@@ -2880,8 +2937,10 @@ object functions {
def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale)
/**
- * @param e angle in radians
- * @return secant of the angle
+ * @param e
+ * angle in radians
+ * @return
+ * secant of the angle
*
* @group math_funcs
* @since 3.3.0
@@ -2889,8 +2948,8 @@ object functions {
def sec(e: Column): Column = Column.fn("sec", e)
/**
- * Shift the given value numBits left. If the given value is a long value, this function
- * will return a long value else it will return an integer value.
+ * Shift the given value numBits left. If the given value is a long value, this function will
+ * return a long value else it will return an integer value.
*
* @group bitwise_funcs
* @since 1.5.0
@@ -2899,8 +2958,8 @@ object functions {
def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits)
/**
- * Shift the given value numBits left. If the given value is a long value, this function
- * will return a long value else it will return an integer value.
+ * Shift the given value numBits left. If the given value is a long value, this function will
+ * return a long value else it will return an integer value.
*
* @group bitwise_funcs
* @since 3.2.0
@@ -2927,8 +2986,8 @@ object functions {
def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits))
/**
- * Unsigned shift the given value numBits right. If the given value is a long value,
- * it will return a long value else it will return an integer value.
+ * Unsigned shift the given value numBits right. If the given value is a long value, it will
+ * return a long value else it will return an integer value.
*
* @group bitwise_funcs
* @since 1.5.0
@@ -2937,8 +2996,8 @@ object functions {
def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits)
/**
- * Unsigned shift the given value numBits right. If the given value is a long value,
- * it will return a long value else it will return an integer value.
+ * Unsigned shift the given value numBits right. If the given value is a long value, it will
+ * return a long value else it will return an integer value.
*
* @group bitwise_funcs
* @since 3.2.0
@@ -2971,8 +3030,10 @@ object functions {
def signum(columnName: String): Column = signum(Column(columnName))
/**
- * @param e angle in radians
- * @return sine of the angle, as if computed by `java.lang.Math.sin`
+ * @param e
+ * angle in radians
+ * @return
+ * sine of the angle, as if computed by `java.lang.Math.sin`
*
* @group math_funcs
* @since 1.4.0
@@ -2980,8 +3041,10 @@ object functions {
def sin(e: Column): Column = Column.fn("sin", e)
/**
- * @param columnName angle in radians
- * @return sine of the angle, as if computed by `java.lang.Math.sin`
+ * @param columnName
+ * angle in radians
+ * @return
+ * sine of the angle, as if computed by `java.lang.Math.sin`
*
* @group math_funcs
* @since 1.4.0
@@ -2989,8 +3052,10 @@ object functions {
def sin(columnName: String): Column = sin(Column(columnName))
/**
- * @param e hyperbolic angle
- * @return hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`
+ * @param e
+ * hyperbolic angle
+ * @return
+ * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`
*
* @group math_funcs
* @since 1.4.0
@@ -2998,8 +3063,10 @@ object functions {
def sinh(e: Column): Column = Column.fn("sinh", e)
/**
- * @param columnName hyperbolic angle
- * @return hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`
+ * @param columnName
+ * hyperbolic angle
+ * @return
+ * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`
*
* @group math_funcs
* @since 1.4.0
@@ -3007,8 +3074,10 @@ object functions {
def sinh(columnName: String): Column = sinh(Column(columnName))
/**
- * @param e angle in radians
- * @return tangent of the given value, as if computed by `java.lang.Math.tan`
+ * @param e
+ * angle in radians
+ * @return
+ * tangent of the given value, as if computed by `java.lang.Math.tan`
*
* @group math_funcs
* @since 1.4.0
@@ -3016,8 +3085,10 @@ object functions {
def tan(e: Column): Column = Column.fn("tan", e)
/**
- * @param columnName angle in radians
- * @return tangent of the given value, as if computed by `java.lang.Math.tan`
+ * @param columnName
+ * angle in radians
+ * @return
+ * tangent of the given value, as if computed by `java.lang.Math.tan`
*
* @group math_funcs
* @since 1.4.0
@@ -3025,8 +3096,10 @@ object functions {
def tan(columnName: String): Column = tan(Column(columnName))
/**
- * @param e hyperbolic angle
- * @return hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`
+ * @param e
+ * hyperbolic angle
+ * @return
+ * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`
*
* @group math_funcs
* @since 1.4.0
@@ -3034,8 +3107,10 @@ object functions {
def tanh(e: Column): Column = Column.fn("tanh", e)
/**
- * @param columnName hyperbolic angle
- * @return hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`
+ * @param columnName
+ * hyperbolic angle
+ * @return
+ * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`
*
* @group math_funcs
* @since 1.4.0
@@ -3057,10 +3132,13 @@ object functions {
def toDegrees(columnName: String): Column = degrees(Column(columnName))
/**
- * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
+ * Converts an angle measured in radians to an approximately equivalent angle measured in
+ * degrees.
*
- * @param e angle in radians
- * @return angle in degrees, as if computed by `java.lang.Math.toDegrees`
+ * @param e
+ * angle in radians
+ * @return
+ * angle in degrees, as if computed by `java.lang.Math.toDegrees`
*
* @group math_funcs
* @since 2.1.0
@@ -3068,10 +3146,13 @@ object functions {
def degrees(e: Column): Column = Column.fn("degrees", e)
/**
- * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
+ * Converts an angle measured in radians to an approximately equivalent angle measured in
+ * degrees.
*
- * @param columnName angle in radians
- * @return angle in degrees, as if computed by `java.lang.Math.toDegrees`
+ * @param columnName
+ * angle in radians
+ * @return
+ * angle in degrees, as if computed by `java.lang.Math.toDegrees`
*
* @group math_funcs
* @since 2.1.0
@@ -3093,10 +3174,13 @@ object functions {
def toRadians(columnName: String): Column = radians(Column(columnName))
/**
- * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
+ * Converts an angle measured in degrees to an approximately equivalent angle measured in
+ * radians.
*
- * @param e angle in degrees
- * @return angle in radians, as if computed by `java.lang.Math.toRadians`
+ * @param e
+ * angle in degrees
+ * @return
+ * angle in radians, as if computed by `java.lang.Math.toRadians`
*
* @group math_funcs
* @since 2.1.0
@@ -3104,10 +3188,13 @@ object functions {
def radians(e: Column): Column = Column.fn("radians", e)
/**
- * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
+ * Converts an angle measured in degrees to an approximately equivalent angle measured in
+ * radians.
*
- * @param columnName angle in degrees
- * @return angle in radians, as if computed by `java.lang.Math.toRadians`
+ * @param columnName
+ * angle in degrees
+ * @return
+ * angle in radians, as if computed by `java.lang.Math.toRadians`
*
* @group math_funcs
* @since 2.1.0
@@ -3115,15 +3202,20 @@ object functions {
def radians(columnName: String): Column = radians(Column(columnName))
/**
- * Returns the bucket number into which the value of this expression would fall
- * after being evaluated. Note that input arguments must follow conditions listed below;
- * otherwise, the method will return null.
+ * Returns the bucket number into which the value of this expression would fall after being
+ * evaluated. Note that input arguments must follow conditions listed below; otherwise, the
+ * method will return null.
*
- * @param v value to compute a bucket number in the histogram
- * @param min minimum value of the histogram
- * @param max maximum value of the histogram
- * @param numBucket the number of buckets
- * @return the bucket number into which the value would fall after being evaluated
+ * @param v
+ * value to compute a bucket number in the histogram
+ * @param min
+ * minimum value of the histogram
+ * @param max
+ * maximum value of the histogram
+ * @param numBucket
+ * the number of buckets
+ * @return
+ * the bucket number into which the value would fall after being evaluated
* @group math_funcs
* @since 3.5.0
*/
@@ -3167,8 +3259,8 @@ object functions {
def current_user(): Column = Column.fn("current_user")
/**
- * Calculates the MD5 digest of a binary column and returns the value
- * as a 32 character hex string.
+ * Calculates the MD5 digest of a binary column and returns the value as a 32 character hex
+ * string.
*
* @group hash_funcs
* @since 1.5.0
@@ -3176,8 +3268,8 @@ object functions {
def md5(e: Column): Column = Column.fn("md5", e)
/**
- * Calculates the SHA-1 digest of a binary column and returns the value
- * as a 40 character hex string.
+ * Calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex
+ * string.
*
* @group hash_funcs
* @since 1.5.0
@@ -3185,11 +3277,13 @@ object functions {
def sha1(e: Column): Column = Column.fn("sha1", e)
/**
- * Calculates the SHA-2 family of hash functions of a binary column and
- * returns the value as a hex string.
+ * Calculates the SHA-2 family of hash functions of a binary column and returns the value as a
+ * hex string.
*
- * @param e column to compute SHA-2 on.
- * @param numBits one of 224, 256, 384, or 512.
+ * @param e
+ * column to compute SHA-2 on.
+ * @param numBits
+ * one of 224, 256, 384, or 512.
*
* @group hash_funcs
* @since 1.5.0
@@ -3202,8 +3296,8 @@ object functions {
}
/**
- * Calculates the cyclic redundancy check value (CRC32) of a binary column and
- * returns the value as a bigint.
+ * Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value
+ * as a bigint.
*
* @group hash_funcs
* @since 1.5.0
@@ -3220,9 +3314,8 @@ object functions {
def hash(cols: Column*): Column = Column.fn("hash", cols: _*)
/**
- * Calculates the hash code of given columns using the 64-bit
- * variant of the xxHash algorithm, and returns the result as a long
- * column. The hash computation uses an initial seed of 42.
+ * Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm,
+ * and returns the result as a long column. The hash computation uses an initial seed of 42.
*
* @group hash_funcs
* @since 3.0.0
@@ -3255,8 +3348,8 @@ object functions {
def raise_error(c: Column): Column = Column.fn("raise_error", c)
/**
- * Returns the estimated number of unique values given the binary representation
- * of a Datasketches HllSketch.
+ * Returns the estimated number of unique values given the binary representation of a
+ * Datasketches HllSketch.
*
* @group misc_funcs
* @since 3.5.0
@@ -3264,8 +3357,8 @@ object functions {
def hll_sketch_estimate(c: Column): Column = Column.fn("hll_sketch_estimate", c)
/**
- * Returns the estimated number of unique values given the binary representation
- * of a Datasketches HllSketch.
+ * Returns the estimated number of unique values given the binary representation of a
+ * Datasketches HllSketch.
*
* @group misc_funcs
* @since 3.5.0
@@ -3275,9 +3368,8 @@ object functions {
}
/**
- * Merges two binary representations of Datasketches HllSketch objects, using a
- * Datasketches Union object. Throws an exception if sketches have different
- * lgConfigK values.
+ * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches
+ * Union object. Throws an exception if sketches have different lgConfigK values.
*
* @group misc_funcs
* @since 3.5.0
@@ -3286,9 +3378,8 @@ object functions {
Column.fn("hll_union", c1, c2)
/**
- * Merges two binary representations of Datasketches HllSketch objects, using a
- * Datasketches Union object. Throws an exception if sketches have different
- * lgConfigK values.
+ * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches
+ * Union object. Throws an exception if sketches have different lgConfigK values.
*
* @group misc_funcs
* @since 3.5.0
@@ -3298,9 +3389,9 @@ object functions {
}
/**
- * Merges two binary representations of Datasketches HllSketch objects, using a
- * Datasketches Union object. Throws an exception if sketches have different
- * lgConfigK values and allowDifferentLgConfigK is set to false.
+ * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches
+ * Union object. Throws an exception if sketches have different lgConfigK values and
+ * allowDifferentLgConfigK is set to false.
*
* @group misc_funcs
* @since 3.5.0
@@ -3309,15 +3400,17 @@ object functions {
Column.fn("hll_union", c1, c2, lit(allowDifferentLgConfigK))
/**
- * Merges two binary representations of Datasketches HllSketch objects, using a
- * Datasketches Union object. Throws an exception if sketches have different
- * lgConfigK values and allowDifferentLgConfigK is set to false.
+ * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches
+ * Union object. Throws an exception if sketches have different lgConfigK values and
+ * allowDifferentLgConfigK is set to false.
*
* @group misc_funcs
* @since 3.5.0
*/
- def hll_union(columnName1: String, columnName2: String, allowDifferentLgConfigK: Boolean):
- Column = {
+ def hll_union(
+ columnName1: String,
+ columnName2: String,
+ allowDifferentLgConfigK: Boolean): Column = {
hll_union(Column(columnName1), Column(columnName2), allowDifferentLgConfigK)
}
@@ -3684,8 +3777,8 @@ object functions {
Column.fn("bitmap_bucket_number", col)
/**
- * Returns a bitmap with the positions of the bits set from all the values from the input column.
- * The input column will most likely be bitmap_bit_position().
+ * Returns a bitmap with the positions of the bits set from all the values from the input
+ * column. The input column will most likely be bitmap_bit_position().
*
* @group agg_funcs
* @since 3.5.0
@@ -3702,8 +3795,8 @@ object functions {
def bitmap_count(col: Column): Column = Column.fn("bitmap_count", col)
/**
- * Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column.
- * The input column should be bitmaps created from bitmap_construct_agg().
+ * Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. The
+ * input column should be bitmaps created from bitmap_construct_agg().
*
* @group agg_funcs
* @since 3.5.0
@@ -3724,8 +3817,8 @@ object functions {
def ascii(e: Column): Column = Column.fn("ascii", e)
/**
- * Computes the BASE64 encoding of a binary column and returns it as a string column.
- * This is the reverse of unbase64.
+ * Computes the BASE64 encoding of a binary column and returns it as a string column. This is
+ * the reverse of unbase64.
*
* @group string_funcs
* @since 1.5.0
@@ -3741,10 +3834,11 @@ object functions {
def bit_length(e: Column): Column = Column.fn("bit_length", e)
/**
- * Concatenates multiple input string columns together into a single string column,
- * using the given separator.
+ * Concatenates multiple input string columns together into a single string column, using the
+ * given separator.
*
- * @note Input strings which are null are skipped.
+ * @note
+ * Input strings which are null are skipped.
*
* @group string_funcs
* @since 1.5.0
@@ -3754,9 +3848,9 @@ object functions {
Column.fn("concat_ws", lit(sep) +: exprs: _*)
/**
- * Computes the first argument into a string from a binary using the provided character set
- * (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32').
- * If either argument is null, the result will also be null.
+ * Computes the first argument into a string from a binary using the provided character set (one
+ * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either
+ * argument is null, the result will also be null.
*
* @group string_funcs
* @since 1.5.0
@@ -3765,9 +3859,9 @@ object functions {
Column.fn("decode", value, lit(charset))
/**
- * Computes the first argument into a binary from a string using the provided character set
- * (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32').
- * If either argument is null, the result will also be null.
+ * Computes the first argument into a binary from a string using the provided character set (one
+ * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either
+ * argument is null, the result will also be null.
*
* @group string_funcs
* @since 1.5.0
@@ -3776,11 +3870,11 @@ object functions {
Column.fn("encode", value, lit(charset))
/**
- * Formats numeric column x to a format like '#,###,###.##', rounded to d decimal places
- * with HALF_EVEN round mode, and returns the result as a string column.
+ * Formats numeric column x to a format like '#,###,###.##', rounded to d decimal places with
+ * HALF_EVEN round mode, and returns the result as a string column.
*
- * If d is 0, the result has no decimal point or fractional part.
- * If d is less than 0, the result will be null.
+ * If d is 0, the result has no decimal point or fractional part. If d is less than 0, the
+ * result will be null.
*
* @group string_funcs
* @since 1.5.0
@@ -3798,8 +3892,8 @@ object functions {
Column.fn("format_string", lit(format) +: arguments: _*)
/**
- * Returns a new string column by converting the first letter of each word to uppercase.
- * Words are delimited by whitespace.
+ * Returns a new string column by converting the first letter of each word to uppercase. Words
+ * are delimited by whitespace.
*
* For example, "hello world" will become "Hello World".
*
@@ -3809,11 +3903,12 @@ object functions {
def initcap(e: Column): Column = Column.fn("initcap", e)
/**
- * Locate the position of the first occurrence of substr column in the given string.
- * Returns null if either of the arguments are null.
+ * Locate the position of the first occurrence of substr column in the given string. Returns
+ * null if either of the arguments are null.
*
- * @note The position is not zero based, but 1 based index. Returns 0 if substr
- * could not be found in str.
+ * @note
+ * The position is not zero based, but 1 based index. Returns 0 if substr could not be found
+ * in str.
*
* @group string_funcs
* @since 1.5.0
@@ -3821,8 +3916,8 @@ object functions {
def instr(str: Column, substring: String): Column = Column.fn("instr", str, lit(substring))
/**
- * Computes the character length of a given string or number of bytes of a binary string.
- * The length of character strings include the trailing spaces. The length of binary strings
+ * Computes the character length of a given string or number of bytes of a binary string. The
+ * length of character strings include the trailing spaces. The length of binary strings
* includes binary zeros.
*
* @group string_funcs
@@ -3831,8 +3926,8 @@ object functions {
def length(e: Column): Column = Column.fn("length", e)
/**
- * Computes the character length of a given string or number of bytes of a binary string.
- * The length of character strings include the trailing spaces. The length of binary strings
+ * Computes the character length of a given string or number of bytes of a binary string. The
+ * length of character strings include the trailing spaces. The length of binary strings
* includes binary zeros.
*
* @group string_funcs
@@ -3849,9 +3944,10 @@ object functions {
def lower(e: Column): Column = Column.fn("lower", e)
/**
- * Computes the Levenshtein distance of the two given string columns if it's less than or
- * equal to a given threshold.
- * @return result distance, or -1
+ * Computes the Levenshtein distance of the two given string columns if it's less than or equal
+ * to a given threshold.
+ * @return
+ * result distance, or -1
* @group string_funcs
* @since 3.5.0
*/
@@ -3868,8 +3964,9 @@ object functions {
/**
* Locate the position of the first occurrence of substr.
*
- * @note The position is not zero based, but 1 based index. Returns 0 if substr
- * could not be found in str.
+ * @note
+ * The position is not zero based, but 1 based index. Returns 0 if substr could not be found
+ * in str.
*
* @group string_funcs
* @since 1.5.0
@@ -3879,8 +3976,9 @@ object functions {
/**
* Locate the position of the first occurrence of substr in a string column, after position pos.
*
- * @note The position is not zero based, but 1 based index. returns 0 if substr
- * could not be found in str.
+ * @note
+ * The position is not zero based, but 1 based index. returns 0 if substr could not be found
+ * in str.
*
* @group string_funcs
* @since 1.5.0
@@ -3889,8 +3987,8 @@ object functions {
Column.fn("locate", lit(substr), str, lit(pos))
/**
- * Left-pad the string column with pad to a length of len. If the string column is longer
- * than len, the return value is shortened to len characters.
+ * Left-pad the string column with pad to a length of len. If the string column is longer than
+ * len, the return value is shortened to len characters.
*
* @group string_funcs
* @since 1.5.0
@@ -3972,8 +4070,8 @@ object functions {
def regexp_like(str: Column, regexp: Column): Column = Column.fn("regexp_like", str, regexp)
/**
- * Returns a count of the number of times that the regular expression pattern `regexp`
- * is matched in the string `str`.
+ * Returns a count of the number of times that the regular expression pattern `regexp` is
+ * matched in the string `str`.
*
* @group string_funcs
* @since 3.5.0
@@ -3981,10 +4079,10 @@ object functions {
def regexp_count(str: Column, regexp: Column): Column = Column.fn("regexp_count", str, regexp)
/**
- * Extract a specific group matched by a Java regex, from the specified string column.
- * If the regex did not match, or the specified group did not match, an empty string is returned.
- * if the specified group index exceeds the group count of regex, an IllegalArgumentException
- * will be thrown.
+ * Extract a specific group matched by a Java regex, from the specified string column. If the
+ * regex did not match, or the specified group did not match, an empty string is returned. if
+ * the specified group index exceeds the group count of regex, an IllegalArgumentException will
+ * be thrown.
*
* @group string_funcs
* @since 1.5.0
@@ -3993,8 +4091,8 @@ object functions {
Column.fn("regexp_extract", e, lit(exp), lit(groupIdx))
/**
- * Extract all strings in the `str` that match the `regexp` expression and
- * corresponding to the first regex group index.
+ * Extract all strings in the `str` that match the `regexp` expression and corresponding to the
+ * first regex group index.
*
* @group string_funcs
* @since 3.5.0
@@ -4003,8 +4101,8 @@ object functions {
Column.fn("regexp_extract_all", str, regexp)
/**
- * Extract all strings in the `str` that match the `regexp` expression and
- * corresponding to the regex group index.
+ * Extract all strings in the `str` that match the `regexp` expression and corresponding to the
+ * regex group index.
*
* @group string_funcs
* @since 3.5.0
@@ -4040,9 +4138,9 @@ object functions {
def regexp_substr(str: Column, regexp: Column): Column = Column.fn("regexp_substr", str, regexp)
/**
- * Searches a string for a regular expression and returns an integer that indicates
- * the beginning position of the matched substring. Positions are 1-based, not 0-based.
- * If no match is found, returns 0.
+ * Searches a string for a regular expression and returns an integer that indicates the
+ * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match
+ * is found, returns 0.
*
* @group string_funcs
* @since 3.5.0
@@ -4050,9 +4148,9 @@ object functions {
def regexp_instr(str: Column, regexp: Column): Column = Column.fn("regexp_instr", str, regexp)
/**
- * Searches a string for a regular expression and returns an integer that indicates
- * the beginning position of the matched substring. Positions are 1-based, not 0-based.
- * If no match is found, returns 0.
+ * Searches a string for a regular expression and returns an integer that indicates the
+ * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match
+ * is found, returns 0.
*
* @group string_funcs
* @since 3.5.0
@@ -4061,8 +4159,8 @@ object functions {
Column.fn("regexp_instr", str, regexp, idx)
/**
- * Decodes a BASE64 encoded string column and returns it as a binary column.
- * This is the reverse of base64.
+ * Decodes a BASE64 encoded string column and returns it as a binary column. This is the reverse
+ * of base64.
*
* @group string_funcs
* @since 1.5.0
@@ -4070,8 +4168,8 @@ object functions {
def unbase64(e: Column): Column = Column.fn("unbase64", e)
/**
- * Right-pad the string column with pad to a length of len. If the string column is longer
- * than len, the return value is shortened to len characters.
+ * Right-pad the string column with pad to a length of len. If the string column is longer than
+ * len, the return value is shortened to len characters.
*
* @group string_funcs
* @since 1.5.0
@@ -4199,11 +4297,11 @@ object functions {
Column.fn("split", str, pattern, limit)
/**
- * Substring starts at `pos` and is of length `len` when str is String type or
- * returns the slice of byte array that starts at `pos` in byte and is of length `len`
- * when str is Binary type
+ * Substring starts at `pos` and is of length `len` when str is String type or returns the slice
+ * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type
*
- * @note The position is not zero based, but 1 based index.
+ * @note
+ * The position is not zero based, but 1 based index.
*
* @group string_funcs
* @since 1.5.0
@@ -4212,11 +4310,11 @@ object functions {
Column.fn("substring", str, lit(pos), lit(len))
/**
- * Substring starts at `pos` and is of length `len` when str is String type or
- * returns the slice of byte array that starts at `pos` in byte and is of length `len`
- * when str is Binary type
+ * Substring starts at `pos` and is of length `len` when str is String type or returns the slice
+ * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type
*
- * @note The position is not zero based, but 1 based index.
+ * @note
+ * The position is not zero based, but 1 based index.
*
* @group string_funcs
* @since 4.0.0
@@ -4225,8 +4323,8 @@ object functions {
Column.fn("substring", str, pos, len)
/**
- * Returns the substring from string str before count occurrences of the delimiter delim.
- * If count is positive, everything the left of the final delimiter (counting from left) is
+ * Returns the substring from string str before count occurrences of the delimiter delim. If
+ * count is positive, everything the left of the final delimiter (counting from left) is
* returned. If count is negative, every to the right of the final delimiter (counting from the
* right) is returned. substring_index performs a case-sensitive match when searching for delim.
*
@@ -4236,8 +4334,8 @@ object functions {
Column.fn("substring_index", str, lit(delim), lit(count))
/**
- * Overlay the specified portion of `src` with `replace`,
- * starting from byte position `pos` of `src` and proceeding for `len` bytes.
+ * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of
+ * `src` and proceeding for `len` bytes.
*
* @group string_funcs
* @since 3.0.0
@@ -4246,8 +4344,8 @@ object functions {
Column.fn("overlay", src, replace, pos, len)
/**
- * Overlay the specified portion of `src` with `replace`,
- * starting from byte position `pos` of `src`.
+ * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of
+ * `src`.
*
* @group string_funcs
* @since 3.0.0
@@ -4264,18 +4362,26 @@ object functions {
Column.fn("sentences", string, language, country)
/**
- * Splits a string into arrays of sentences, where each sentence is an array of words.
- * The default locale is used.
+ * Splits a string into arrays of sentences, where each sentence is an array of words. The
+ * default `country`('') is used.
+ * @group string_funcs
+ * @since 4.0.0
+ */
+ def sentences(string: Column, language: Column): Column =
+ Column.fn("sentences", string, language)
+
+ /**
+ * Splits a string into arrays of sentences, where each sentence is an array of words. The
+ * default locale is used.
* @group string_funcs
* @since 3.2.0
*/
def sentences(string: Column): Column = Column.fn("sentences", string)
/**
- * Translate any character in the src by a character in replaceString.
- * The characters in replaceString correspond to the characters in matchingString.
- * The translate will happen when any character in the string matches the character
- * in the `matchingString`.
+ * Translate any character in the src by a character in replaceString. The characters in
+ * replaceString correspond to the characters in matchingString. The translate will happen when
+ * any character in the string matches the character in the `matchingString`.
*
* @group string_funcs
* @since 1.5.0
@@ -4307,10 +4413,10 @@ object functions {
def upper(e: Column): Column = Column.fn("upper", e)
/**
- * Converts the input `e` to a binary value based on the supplied `format`.
- * The `format` can be a case-insensitive string literal of "hex", "utf-8", "utf8", or "base64".
- * By default, the binary format for conversion is "hex" if `format` is omitted.
- * The function returns NULL if at least one of the input parameters is NULL.
+ * Converts the input `e` to a binary value based on the supplied `format`. The `format` can be
+ * a case-insensitive string literal of "hex", "utf-8", "utf8", or "base64". By default, the
+ * binary format for conversion is "hex" if `format` is omitted. The function returns NULL if at
+ * least one of the input parameters is NULL.
*
* @group string_funcs
* @since 3.5.0
@@ -4318,8 +4424,8 @@ object functions {
def to_binary(e: Column, f: Column): Column = Column.fn("to_binary", e, f)
/**
- * Converts the input `e` to a binary value based on the default format "hex".
- * The function returns NULL if at least one of the input parameters is NULL.
+ * Converts the input `e` to a binary value based on the default format "hex". The function
+ * returns NULL if at least one of the input parameters is NULL.
*
* @group string_funcs
* @since 3.5.0
@@ -4328,32 +4434,27 @@ object functions {
// scalastyle:off line.size.limit
/**
- * Convert `e` to a string based on the `format`.
- * Throws an exception if the conversion fails. The format can consist of the following
- * characters, case insensitive:
- * '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format
- * string matches a sequence of digits in the input value, generating a result string of the
- * same length as the corresponding sequence in the format string. The result string is
- * left-padded with zeros if the 0/9 sequence comprises more digits than the matching part of
- * the decimal value, starts with 0, and is before the decimal point. Otherwise, it is
- * padded with spaces.
- * '.' or 'D': Specifies the position of the decimal point (optional, only allowed once).
- * ',' or 'G': Specifies the position of the grouping (thousands) separator (,). There must be
- * a 0 or 9 to the left and right of each grouping separator.
- * '$': Specifies the location of the $ currency sign. This character may only be specified
- * once.
- * 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at
- * the beginning or end of the format string). Note that 'S' prints '+' for positive values
- * but 'MI' prints a space.
- * 'PR': Only allowed at the end of the format string; specifies that the result string will be
- * wrapped by angle brackets if the input value is negative.
- *
- * If `e` is a datetime, `format` shall be a valid datetime pattern, see
- * Datetime Patterns.
- * If `e` is a binary, it is converted to a string in one of the formats:
- * 'base64': a base 64 string.
- * 'hex': a string in the hexadecimal format.
- * 'utf-8': the input binary is decoded to UTF-8 string.
+ * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails.
+ * The format can consist of the following characters, case insensitive: '0' or '9': Specifies
+ * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a
+ * sequence of digits in the input value, generating a result string of the same length as the
+ * corresponding sequence in the format string. The result string is left-padded with zeros if
+ * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts
+ * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D':
+ * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G':
+ * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to
+ * the left and right of each grouping separator. '$': Specifies the location of the $ currency
+ * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-'
+ * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note
+ * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the
+ * end of the format string; specifies that the result string will be wrapped by angle brackets
+ * if the input value is negative.
+ *
+ * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime
+ * Patterns. If `e` is a binary, it is converted to a string in one of the formats:
+ * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input
+ * binary is decoded to UTF-8 string.
*
* @group string_funcs
* @since 3.5.0
@@ -4363,32 +4464,27 @@ object functions {
// scalastyle:off line.size.limit
/**
- * Convert `e` to a string based on the `format`.
- * Throws an exception if the conversion fails. The format can consist of the following
- * characters, case insensitive:
- * '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format
- * string matches a sequence of digits in the input value, generating a result string of the
- * same length as the corresponding sequence in the format string. The result string is
- * left-padded with zeros if the 0/9 sequence comprises more digits than the matching part of
- * the decimal value, starts with 0, and is before the decimal point. Otherwise, it is
- * padded with spaces.
- * '.' or 'D': Specifies the position of the decimal point (optional, only allowed once).
- * ',' or 'G': Specifies the position of the grouping (thousands) separator (,). There must be
- * a 0 or 9 to the left and right of each grouping separator.
- * '$': Specifies the location of the $ currency sign. This character may only be specified
- * once.
- * 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at
- * the beginning or end of the format string). Note that 'S' prints '+' for positive values
- * but 'MI' prints a space.
- * 'PR': Only allowed at the end of the format string; specifies that the result string will be
- * wrapped by angle brackets if the input value is negative.
- *
- * If `e` is a datetime, `format` shall be a valid datetime pattern, see
- * Datetime Patterns.
- * If `e` is a binary, it is converted to a string in one of the formats:
- * 'base64': a base 64 string.
- * 'hex': a string in the hexadecimal format.
- * 'utf-8': the input binary is decoded to UTF-8 string.
+ * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails.
+ * The format can consist of the following characters, case insensitive: '0' or '9': Specifies
+ * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a
+ * sequence of digits in the input value, generating a result string of the same length as the
+ * corresponding sequence in the format string. The result string is left-padded with zeros if
+ * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts
+ * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D':
+ * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G':
+ * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to
+ * the left and right of each grouping separator. '$': Specifies the location of the $ currency
+ * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-'
+ * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note
+ * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the
+ * end of the format string; specifies that the result string will be wrapped by angle brackets
+ * if the input value is negative.
+ *
+ * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime
+ * Patterns. If `e` is a binary, it is converted to a string in one of the formats:
+ * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input
+ * binary is decoded to UTF-8 string.
*
* @group string_funcs
* @since 3.5.0
@@ -4397,24 +4493,21 @@ object functions {
def to_varchar(e: Column, format: Column): Column = Column.fn("to_varchar", e, format)
/**
- * Convert string 'e' to a number based on the string format 'format'.
- * Throws an exception if the conversion fails. The format can consist of the following
- * characters, case insensitive:
- * '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format
- * string matches a sequence of digits in the input string. If the 0/9 sequence starts with
- * 0 and is before the decimal point, it can only match a digit sequence of the same size.
- * Otherwise, if the sequence starts with 9 or is after the decimal point, it can match a
- * digit sequence that has the same or smaller size.
- * '.' or 'D': Specifies the position of the decimal point (optional, only allowed once).
- * ',' or 'G': Specifies the position of the grouping (thousands) separator (,). There must be
- * a 0 or 9 to the left and right of each grouping separator. 'expr' must match the
- * grouping separator relevant for the size of the number.
- * '$': Specifies the location of the $ currency sign. This character may only be specified
- * once.
- * 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at
- * the beginning or end of the format string). Note that 'S' allows '-' but 'MI' does not.
- * 'PR': Only allowed at the end of the format string; specifies that 'expr' indicates a
- * negative number with wrapping angled brackets.
+ * Convert string 'e' to a number based on the string format 'format'. Throws an exception if
+ * the conversion fails. The format can consist of the following characters, case insensitive:
+ * '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format
+ * string matches a sequence of digits in the input string. If the 0/9 sequence starts with 0
+ * and is before the decimal point, it can only match a digit sequence of the same size.
+ * Otherwise, if the sequence starts with 9 or is after the decimal point, it can match a digit
+ * sequence that has the same or smaller size. '.' or 'D': Specifies the position of the decimal
+ * point (optional, only allowed once). ',' or 'G': Specifies the position of the grouping
+ * (thousands) separator (,). There must be a 0 or 9 to the left and right of each grouping
+ * separator. 'expr' must match the grouping separator relevant for the size of the number. '$':
+ * Specifies the location of the $ currency sign. This character may only be specified once. 'S'
+ * or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at the
+ * beginning or end of the format string). Note that 'S' allows '-' but 'MI' does not. 'PR':
+ * Only allowed at the end of the format string; specifies that 'expr' indicates a negative
+ * number with wrapping angled brackets.
*
* @group string_funcs
* @since 3.5.0
@@ -4452,11 +4545,10 @@ object functions {
def replace(src: Column, search: Column): Column = Column.fn("replace", src, search)
/**
- * Splits `str` by delimiter and return requested part of the split (1-based).
- * If any input is null, returns null. if `partNum` is out of range of split parts,
- * returns empty string. If `partNum` is 0, throws an error. If `partNum` is negative,
- * the parts are counted backward from the end of the string.
- * If the `delimiter` is an empty string, the `str` is not split.
+ * Splits `str` by delimiter and return requested part of the split (1-based). If any input is
+ * null, returns null. if `partNum` is out of range of split parts, returns empty string. If
+ * `partNum` is 0, throws an error. If `partNum` is negative, the parts are counted backward
+ * from the end of the string. If the `delimiter` is an empty string, the `str` is not split.
*
* @group string_funcs
* @since 3.5.0
@@ -4465,8 +4557,8 @@ object functions {
Column.fn("split_part", str, delimiter, partNum)
/**
- * Returns the substring of `str` that starts at `pos` and is of length `len`,
- * or the slice of byte array that starts at `pos` and is of length `len`.
+ * Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of
+ * byte array that starts at `pos` and is of length `len`.
*
* @group string_funcs
* @since 3.5.0
@@ -4475,8 +4567,8 @@ object functions {
Column.fn("substr", str, pos, len)
/**
- * Returns the substring of `str` that starts at `pos`,
- * or the slice of byte array that starts at `pos`.
+ * Returns the substring of `str` that starts at `pos`, or the slice of byte array that starts
+ * at `pos`.
*
* @group string_funcs
* @since 3.5.0
@@ -4511,8 +4603,8 @@ object functions {
Column.fn("printf", (format +: arguments): _*)
/**
- * Decodes a `str` in 'application/x-www-form-urlencoded' format
- * using a specific encoding scheme.
+ * Decodes a `str` in 'application/x-www-form-urlencoded' format using a specific encoding
+ * scheme.
*
* @group url_funcs
* @since 3.5.0
@@ -4520,8 +4612,8 @@ object functions {
def url_decode(str: Column): Column = Column.fn("url_decode", str)
/**
- * This is a special version of `url_decode` that performs the same operation, but returns
- * a NULL value instead of raising an error if the decoding cannot be performed.
+ * This is a special version of `url_decode` that performs the same operation, but returns a
+ * NULL value instead of raising an error if the decoding cannot be performed.
*
* @group url_funcs
* @since 4.0.0
@@ -4529,8 +4621,8 @@ object functions {
def try_url_decode(str: Column): Column = Column.fn("try_url_decode", str)
/**
- * Translates a string into 'application/x-www-form-urlencoded' format
- * using a specific encoding scheme.
+ * Translates a string into 'application/x-www-form-urlencoded' format using a specific encoding
+ * scheme.
*
* @group url_funcs
* @since 3.5.0
@@ -4538,8 +4630,8 @@ object functions {
def url_encode(str: Column): Column = Column.fn("url_encode", str)
/**
- * Returns the position of the first occurrence of `substr` in `str` after position `start`.
- * The given `start` and return value are 1-based.
+ * Returns the position of the first occurrence of `substr` in `str` after position `start`. The
+ * given `start` and return value are 1-based.
*
* @group string_funcs
* @since 3.5.0
@@ -4548,8 +4640,8 @@ object functions {
Column.fn("position", substr, str, start)
/**
- * Returns the position of the first occurrence of `substr` in `str` after position `1`.
- * The return value are 1-based.
+ * Returns the position of the first occurrence of `substr` in `str` after position `1`. The
+ * return value are 1-based.
*
* @group string_funcs
* @since 3.5.0
@@ -4558,9 +4650,9 @@ object functions {
Column.fn("position", substr, str)
/**
- * Returns a boolean. The value is True if str ends with suffix.
- * Returns NULL if either input expression is NULL. Otherwise, returns False.
- * Both str or suffix must be of STRING or BINARY type.
+ * Returns a boolean. The value is True if str ends with suffix. Returns NULL if either input
+ * expression is NULL. Otherwise, returns False. Both str or suffix must be of STRING or BINARY
+ * type.
*
* @group string_funcs
* @since 3.5.0
@@ -4569,9 +4661,9 @@ object functions {
Column.fn("endswith", str, suffix)
/**
- * Returns a boolean. The value is True if str starts with prefix.
- * Returns NULL if either input expression is NULL. Otherwise, returns False.
- * Both str or prefix must be of STRING or BINARY type.
+ * Returns a boolean. The value is True if str starts with prefix. Returns NULL if either input
+ * expression is NULL. Otherwise, returns False. Both str or prefix must be of STRING or BINARY
+ * type.
*
* @group string_funcs
* @since 3.5.0
@@ -4580,8 +4672,8 @@ object functions {
Column.fn("startswith", str, prefix)
/**
- * Returns the ASCII character having the binary equivalent to `n`.
- * If n is larger than 256 the result is equivalent to char(n % 256)
+ * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the
+ * result is equivalent to char(n % 256)
*
* @group string_funcs
* @since 3.5.0
@@ -4633,9 +4725,8 @@ object functions {
def try_to_number(e: Column, format: Column): Column = Column.fn("try_to_number", e, format)
/**
- * Returns the character length of string data or number of bytes of binary data.
- * The length of string data includes the trailing spaces.
- * The length of binary data includes binary zeros.
+ * Returns the character length of string data or number of bytes of binary data. The length of
+ * string data includes the trailing spaces. The length of binary data includes binary zeros.
*
* @group string_funcs
* @since 3.5.0
@@ -4643,9 +4734,8 @@ object functions {
def char_length(str: Column): Column = Column.fn("char_length", str)
/**
- * Returns the character length of string data or number of bytes of binary data.
- * The length of string data includes the trailing spaces.
- * The length of binary data includes binary zeros.
+ * Returns the character length of string data or number of bytes of binary data. The length of
+ * string data includes the trailing spaces. The length of binary data includes binary zeros.
*
* @group string_funcs
* @since 3.5.0
@@ -4653,8 +4743,8 @@ object functions {
def character_length(str: Column): Column = Column.fn("character_length", str)
/**
- * Returns the ASCII character having the binary equivalent to `n`.
- * If n is larger than 256 the result is equivalent to chr(n % 256)
+ * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the
+ * result is equivalent to chr(n % 256)
*
* @group string_funcs
* @since 3.5.0
@@ -4662,9 +4752,9 @@ object functions {
def chr(n: Column): Column = Column.fn("chr", n)
/**
- * Returns a boolean. The value is True if right is found inside left.
- * Returns NULL if either input expression is NULL. Otherwise, returns False.
- * Both left or right must be of STRING or BINARY type.
+ * Returns a boolean. The value is True if right is found inside left. Returns NULL if either
+ * input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or
+ * BINARY type.
*
* @group string_funcs
* @since 3.5.0
@@ -4672,10 +4762,10 @@ object functions {
def contains(left: Column, right: Column): Column = Column.fn("contains", left, right)
/**
- * Returns the `n`-th input, e.g., returns `input2` when `n` is 2.
- * The function returns NULL if the index exceeds the length of the array
- * and `spark.sql.ansi.enabled` is set to false. If `spark.sql.ansi.enabled` is set to true,
- * it throws ArrayIndexOutOfBoundsException for invalid indices.
+ * Returns the `n`-th input, e.g., returns `input2` when `n` is 2. The function returns NULL if
+ * the index exceeds the length of the array and `spark.sql.ansi.enabled` is set to false. If
+ * `spark.sql.ansi.enabled` is set to true, it throws ArrayIndexOutOfBoundsException for invalid
+ * indices.
*
* @group string_funcs
* @since 3.5.0
@@ -4684,9 +4774,9 @@ object functions {
def elt(inputs: Column*): Column = Column.fn("elt", inputs: _*)
/**
- * Returns the index (1-based) of the given string (`str`) in the comma-delimited
- * list (`strArray`). Returns 0, if the string was not found or if the given string (`str`)
- * contains a comma.
+ * Returns the index (1-based) of the given string (`str`) in the comma-delimited list
+ * (`strArray`). Returns 0, if the string was not found or if the given string (`str`) contains
+ * a comma.
*
* @group string_funcs
* @since 3.5.0
@@ -4748,8 +4838,8 @@ object functions {
def ucase(str: Column): Column = Column.fn("ucase", str)
/**
- * Returns the leftmost `len`(`len` can be string type) characters from the string `str`,
- * if `len` is less or equal than 0 the result is an empty string.
+ * Returns the leftmost `len`(`len` can be string type) characters from the string `str`, if
+ * `len` is less or equal than 0 the result is an empty string.
*
* @group string_funcs
* @since 3.5.0
@@ -4757,8 +4847,8 @@ object functions {
def left(str: Column, len: Column): Column = Column.fn("left", str, len)
/**
- * Returns the rightmost `len`(`len` can be string type) characters from the string `str`,
- * if `len` is less or equal than 0 the result is an empty string.
+ * Returns the rightmost `len`(`len` can be string type) characters from the string `str`, if
+ * `len` is less or equal than 0 the result is an empty string.
*
* @group string_funcs
* @since 3.5.0
@@ -4772,23 +4862,29 @@ object functions {
/**
* Returns the date that is `numMonths` after `startDate`.
*
- * @param startDate A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param numMonths The number of months to add to `startDate`, can be negative to subtract months
- * @return A date, or null if `startDate` was a string that could not be cast to a date
+ * @param startDate
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param numMonths
+ * The number of months to add to `startDate`, can be negative to subtract months
+ * @return
+ * A date, or null if `startDate` was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
- def add_months(startDate: Column, numMonths: Int): Column = add_months(startDate, lit(numMonths))
+ def add_months(startDate: Column, numMonths: Int): Column =
+ add_months(startDate, lit(numMonths))
/**
* Returns the date that is `numMonths` after `startDate`.
*
- * @param startDate A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param numMonths A column of the number of months to add to `startDate`, can be negative to
- * subtract months
- * @return A date, or null if `startDate` was a string that could not be cast to a date
+ * @param startDate
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param numMonths
+ * A column of the number of months to add to `startDate`, can be negative to subtract months
+ * @return
+ * A date, or null if `startDate` was a string that could not be cast to a date
* @group datetime_funcs
* @since 3.0.0
*/
@@ -4796,8 +4892,8 @@ object functions {
Column.fn("add_months", startDate, numMonths)
/**
- * Returns the current date at the start of query evaluation as a date column.
- * All calls of current_date within the same query return the same value.
+ * Returns the current date at the start of query evaluation as a date column. All calls of
+ * current_date within the same query return the same value.
*
* @group datetime_funcs
* @since 3.5.0
@@ -4805,8 +4901,8 @@ object functions {
def curdate(): Column = Column.fn("curdate")
/**
- * Returns the current date at the start of query evaluation as a date column.
- * All calls of current_date within the same query return the same value.
+ * Returns the current date at the start of query evaluation as a date column. All calls of
+ * current_date within the same query return the same value.
*
* @group datetime_funcs
* @since 1.5.0
@@ -4822,8 +4918,8 @@ object functions {
def current_timezone(): Column = Column.fn("current_timezone")
/**
- * Returns the current timestamp at the start of query evaluation as a timestamp column.
- * All calls of current_timestamp within the same query return the same value.
+ * Returns the current timestamp at the start of query evaluation as a timestamp column. All
+ * calls of current_timestamp within the same query return the same value.
*
* @group datetime_funcs
* @since 1.5.0
@@ -4839,9 +4935,9 @@ object functions {
def now(): Column = Column.fn("now")
/**
- * Returns the current timestamp without time zone at the start of query evaluation
- * as a timestamp without time zone column.
- * All calls of localtimestamp within the same query return the same value.
+ * Returns the current timestamp without time zone at the start of query evaluation as a
+ * timestamp without time zone column. All calls of localtimestamp within the same query return
+ * the same value.
*
* @group datetime_funcs
* @since 3.3.0
@@ -4852,17 +4948,21 @@ object functions {
* Converts a date/timestamp/string to a value of string in the format specified by the date
* format given by the second argument.
*
- * See
- * Datetime Patterns
- * for valid date and time format patterns
- *
- * @param dateExpr A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param format A pattern `dd.MM.yyyy` would return a string like `18.03.1993`
- * @return A string, or null if `dateExpr` was a string that could not be cast to a timestamp
- * @note Use specialized functions like [[year]] whenever possible as they benefit from a
- * specialized implementation.
- * @throws IllegalArgumentException if the `format` pattern is invalid
+ * See Datetime
+ * Patterns for valid date and time format patterns
+ *
+ * @param dateExpr
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param format
+ * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`
+ * @return
+ * A string, or null if `dateExpr` was a string that could not be cast to a timestamp
+ * @note
+ * Use specialized functions like [[year]] whenever possible as they benefit from a
+ * specialized implementation.
+ * @throws IllegalArgumentException
+ * if the `format` pattern is invalid
* @group datetime_funcs
* @since 1.5.0
*/
@@ -4872,10 +4972,13 @@ object functions {
/**
* Returns the date that is `days` days after `start`
*
- * @param start A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param days The number of days to add to `start`, can be negative to subtract days
- * @return A date, or null if `start` was a string that could not be cast to a date
+ * @param start
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param days
+ * The number of days to add to `start`, can be negative to subtract days
+ * @return
+ * A date, or null if `start` was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -4884,10 +4987,13 @@ object functions {
/**
* Returns the date that is `days` days after `start`
*
- * @param start A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param days A column of the number of days to add to `start`, can be negative to subtract days
- * @return A date, or null if `start` was a string that could not be cast to a date
+ * @param start
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param days
+ * A column of the number of days to add to `start`, can be negative to subtract days
+ * @return
+ * A date, or null if `start` was a string that could not be cast to a date
* @group datetime_funcs
* @since 3.0.0
*/
@@ -4896,10 +5002,13 @@ object functions {
/**
* Returns the date that is `days` days after `start`
*
- * @param start A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param days A column of the number of days to add to `start`, can be negative to subtract days
- * @return A date, or null if `start` was a string that could not be cast to a date
+ * @param start
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param days
+ * A column of the number of days to add to `start`, can be negative to subtract days
+ * @return
+ * A date, or null if `start` was a string that could not be cast to a date
* @group datetime_funcs
* @since 3.5.0
*/
@@ -4908,10 +5017,13 @@ object functions {
/**
* Returns the date that is `days` days before `start`
*
- * @param start A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param days The number of days to subtract from `start`, can be negative to add days
- * @return A date, or null if `start` was a string that could not be cast to a date
+ * @param start
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param days
+ * The number of days to subtract from `start`, can be negative to add days
+ * @return
+ * A date, or null if `start` was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -4920,11 +5032,13 @@ object functions {
/**
* Returns the date that is `days` days before `start`
*
- * @param start A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param days A column of the number of days to subtract from `start`, can be negative to add
- * days
- * @return A date, or null if `start` was a string that could not be cast to a date
+ * @param start
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param days
+ * A column of the number of days to subtract from `start`, can be negative to add days
+ * @return
+ * A date, or null if `start` was a string that could not be cast to a date
* @group datetime_funcs
* @since 3.0.0
*/
@@ -4940,12 +5054,15 @@ object functions {
* // returns 1
* }}}
*
- * @param end A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param start A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @return An integer, or null if either `end` or `start` were strings that could not be cast to
- * a date. Negative if `end` is before `start`
+ * @param end
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param start
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @return
+ * An integer, or null if either `end` or `start` were strings that could not be cast to a
+ * date. Negative if `end` is before `start`
* @group datetime_funcs
* @since 1.5.0
*/
@@ -4960,12 +5077,15 @@ object functions {
* // returns 1
* }}}
*
- * @param end A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param start A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @return An integer, or null if either `end` or `start` were strings that could not be cast to
- * a date. Negative if `end` is before `start`
+ * @param end
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param start
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @return
+ * An integer, or null if either `end` or `start` were strings that could not be cast to a
+ * date. Negative if `end` is before `start`
* @group datetime_funcs
* @since 3.5.0
*/
@@ -4981,7 +5101,8 @@ object functions {
/**
* Extracts the year as an integer from a given date/timestamp/string.
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -4989,7 +5110,8 @@ object functions {
/**
* Extracts the quarter as an integer from a given date/timestamp/string.
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -4997,16 +5119,18 @@ object functions {
/**
* Extracts the month as an integer from a given date/timestamp/string.
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
def month(e: Column): Column = Column.fn("month", e)
/**
- * Extracts the day of the week as an integer from a given date/timestamp/string.
- * Ranges from 1 for a Sunday through to 7 for a Saturday
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1
+ * for a Sunday through to 7 for a Saturday
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 2.3.0
*/
@@ -5014,7 +5138,8 @@ object functions {
/**
* Extracts the day of the month as an integer from a given date/timestamp/string.
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5022,7 +5147,8 @@ object functions {
/**
* Extracts the day of the month as an integer from a given date/timestamp/string.
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 3.5.0
*/
@@ -5030,7 +5156,8 @@ object functions {
/**
* Extracts the day of the year as an integer from a given date/timestamp/string.
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5038,7 +5165,8 @@ object functions {
/**
* Extracts the hours as an integer from a given date/timestamp/string.
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5047,9 +5175,12 @@ object functions {
/**
* Extracts a part of the date/timestamp or interval source.
*
- * @param field selects which part of the source should be extracted.
- * @param source a date/timestamp or interval column from where `field` should be extracted.
- * @return a part of the date/timestamp or interval source
+ * @param field
+ * selects which part of the source should be extracted.
+ * @param source
+ * a date/timestamp or interval column from where `field` should be extracted.
+ * @return
+ * a part of the date/timestamp or interval source
* @group datetime_funcs
* @since 3.5.0
*/
@@ -5060,10 +5191,13 @@ object functions {
/**
* Extracts a part of the date/timestamp or interval source.
*
- * @param field selects which part of the source should be extracted, and supported string values
- * are as same as the fields of the equivalent function `extract`.
- * @param source a date/timestamp or interval column from where `field` should be extracted.
- * @return a part of the date/timestamp or interval source
+ * @param field
+ * selects which part of the source should be extracted, and supported string values are as
+ * same as the fields of the equivalent function `extract`.
+ * @param source
+ * a date/timestamp or interval column from where `field` should be extracted.
+ * @return
+ * a part of the date/timestamp or interval source
* @group datetime_funcs
* @since 3.5.0
*/
@@ -5074,10 +5208,13 @@ object functions {
/**
* Extracts a part of the date/timestamp or interval source.
*
- * @param field selects which part of the source should be extracted, and supported string values
- * are as same as the fields of the equivalent function `EXTRACT`.
- * @param source a date/timestamp or interval column from where `field` should be extracted.
- * @return a part of the date/timestamp or interval source
+ * @param field
+ * selects which part of the source should be extracted, and supported string values are as
+ * same as the fields of the equivalent function `EXTRACT`.
+ * @param source
+ * a date/timestamp or interval column from where `field` should be extracted.
+ * @return
+ * a part of the date/timestamp or interval source
* @group datetime_funcs
* @since 3.5.0
*/
@@ -5086,13 +5223,14 @@ object functions {
}
/**
- * Returns the last day of the month which the given date belongs to.
- * For example, input "2015-07-27" returns "2015-07-31" since July 31 is the last day of the
- * month in July 2015.
+ * Returns the last day of the month which the given date belongs to. For example, input
+ * "2015-07-27" returns "2015-07-31" since July 31 is the last day of the month in July 2015.
*
- * @param e A date, timestamp or string. If a string, the data must be in a format that can be
- * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @return A date, or null if the input was a string that could not be cast to a date
+ * @param e
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @return
+ * A date, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5100,7 +5238,8 @@ object functions {
/**
* Extracts the minutes as an integer from a given date/timestamp/string.
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5115,7 +5254,8 @@ object functions {
def weekday(e: Column): Column = Column.fn("weekday", e)
/**
- * @return A date created from year, month and day fields.
+ * @return
+ * A date created from year, month and day fields.
* @group datetime_funcs
* @since 3.3.0
*/
@@ -5126,7 +5266,8 @@ object functions {
* Returns number of months between dates `start` and `end`.
*
* A whole number is returned if both inputs have the same day of month or both are the last day
- * of their respective months. Otherwise, the difference is calculated assuming 31 days per month.
+ * of their respective months. Otherwise, the difference is calculated assuming 31 days per
+ * month.
*
* For example:
* {{{
@@ -5135,12 +5276,15 @@ object functions {
* months_between("2017-06-01", "2017-06-16 12:00:00") // returns -0.5
* }}}
*
- * @param end A date, timestamp or string. If a string, the data must be in a format that can
- * be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param start A date, timestamp or string. If a string, the data must be in a format that can
- * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @return A double, or null if either `end` or `start` were strings that could not be cast to a
- * timestamp. Negative if `end` is before `start`
+ * @param end
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param start
+ * A date, timestamp or string. If a string, the data must be in a format that can cast to a
+ * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @return
+ * A double, or null if either `end` or `start` were strings that could not be cast to a
+ * timestamp. Negative if `end` is before `start`
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5163,11 +5307,14 @@ object functions {
* For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first
* Sunday after 2015-07-27.
*
- * @param date A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param dayOfWeek Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"
- * @return A date, or null if `date` was a string that could not be cast to a date or if
- * `dayOfWeek` was an invalid value
+ * @param date
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param dayOfWeek
+ * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"
+ * @return
+ * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek`
+ * was an invalid value
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5180,12 +5327,15 @@ object functions {
* For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first
* Sunday after 2015-07-27.
*
- * @param date A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param dayOfWeek A column of the day of week. Case insensitive, and accepts: "Mon", "Tue",
- * "Wed", "Thu", "Fri", "Sat", "Sun"
- * @return A date, or null if `date` was a string that could not be cast to a date or if
- * `dayOfWeek` was an invalid value
+ * @param date
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param dayOfWeek
+ * A column of the day of week. Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu",
+ * "Fri", "Sat", "Sun"
+ * @return
+ * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek`
+ * was an invalid value
* @group datetime_funcs
* @since 3.2.0
*/
@@ -5194,7 +5344,8 @@ object functions {
/**
* Extracts the seconds as an integer from a given date/timestamp/string.
- * @return An integer, or null if the input was a string that could not be cast to a timestamp
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a timestamp
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5206,7 +5357,8 @@ object functions {
* A week is considered to start on a Monday and week 1 is the first week with more than 3 days,
* as defined by ISO 8601
*
- * @return An integer, or null if the input was a string that could not be cast to a date
+ * @return
+ * An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5214,12 +5366,14 @@ object functions {
/**
* Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string
- * representing the timestamp of that moment in the current system time zone in the
- * yyyy-MM-dd HH:mm:ss format.
- *
- * @param ut A number of a type that is castable to a long, such as string or integer. Can be
- * negative for timestamps before the unix epoch
- * @return A string, or null if the input was a string that could not be cast to a long
+ * representing the timestamp of that moment in the current system time zone in the yyyy-MM-dd
+ * HH:mm:ss format.
+ *
+ * @param ut
+ * A number of a type that is castable to a long, such as string or integer. Can be negative
+ * for timestamps before the unix epoch
+ * @return
+ * A string, or null if the input was a string that could not be cast to a long
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5230,15 +5384,17 @@ object functions {
* representing the timestamp of that moment in the current system time zone in the given
* format.
*
- * See
- * Datetime Patterns
- * for valid date and time format patterns
- *
- * @param ut A number of a type that is castable to a long, such as string or integer. Can be
- * negative for timestamps before the unix epoch
- * @param f A date time pattern that the input will be formatted to
- * @return A string, or null if `ut` was a string that could not be cast to a long or `f` was
- * an invalid date time pattern
+ * See Datetime
+ * Patterns for valid date and time format patterns
+ *
+ * @param ut
+ * A number of a type that is castable to a long, such as string or integer. Can be negative
+ * for timestamps before the unix epoch
+ * @param f
+ * A date time pattern that the input will be formatted to
+ * @return
+ * A string, or null if `ut` was a string that could not be cast to a long or `f` was an
+ * invalid date time pattern
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5248,8 +5404,9 @@ object functions {
/**
* Returns the current Unix timestamp (in seconds) as a long.
*
- * @note All calls of `unix_timestamp` within the same query return the same value
- * (i.e. the current timestamp is calculated at the start of query evaluation).
+ * @note
+ * All calls of `unix_timestamp` within the same query return the same value (i.e. the current
+ * timestamp is calculated at the start of query evaluation).
*
* @group datetime_funcs
* @since 1.5.0
@@ -5257,12 +5414,14 @@ object functions {
def unix_timestamp(): Column = unix_timestamp(current_timestamp())
/**
- * Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds),
- * using the default timezone and the default locale.
+ * Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds), using the
+ * default timezone and the default locale.
*
- * @param s A date, timestamp or string. If a string, the data must be in the
- * `yyyy-MM-dd HH:mm:ss` format
- * @return A long, or null if the input was a string not of the correct format
+ * @param s
+ * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss`
+ * format
+ * @return
+ * A long, or null if the input was a string not of the correct format
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5271,15 +5430,17 @@ object functions {
/**
* Converts time string with given pattern to Unix timestamp (in seconds).
*
- * See
- * Datetime Patterns
- * for valid date and time format patterns
- *
- * @param s A date, timestamp or string. If a string, the data must be in a format that can be
- * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param p A date time pattern detailing the format of `s` when `s` is a string
- * @return A long, or null if `s` was a string that could not be cast to a date or `p` was
- * an invalid format
+ * See Datetime
+ * Patterns for valid date and time format patterns
+ *
+ * @param s
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param p
+ * A date time pattern detailing the format of `s` when `s` is a string
+ * @return
+ * A long, or null if `s` was a string that could not be cast to a date or `p` was an invalid
+ * format
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5289,9 +5450,11 @@ object functions {
/**
* Converts to a timestamp by casting rules to `TimestampType`.
*
- * @param s A date, timestamp or string. If a string, the data must be in a format that can be
- * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @return A timestamp, or null if the input was a string that could not be cast to a timestamp
+ * @param s
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @return
+ * A timestamp, or null if the input was a string that could not be cast to a timestamp
* @group datetime_funcs
* @since 2.2.0
*/
@@ -5300,15 +5463,17 @@ object functions {
/**
* Converts time string with the given pattern to timestamp.
*
- * See
- * Datetime Patterns
- * for valid date and time format patterns
- *
- * @param s A date, timestamp or string. If a string, the data must be in a format that can be
- * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param fmt A date time pattern detailing the format of `s` when `s` is a string
- * @return A timestamp, or null if `s` was a string that could not be cast to a timestamp or
- * `fmt` was an invalid format
+ * See Datetime
+ * Patterns for valid date and time format patterns
+ *
+ * @param s
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param fmt
+ * A date time pattern detailing the format of `s` when `s` is a string
+ * @return
+ * A timestamp, or null if `s` was a string that could not be cast to a timestamp or `fmt` was
+ * an invalid format
* @group datetime_funcs
* @since 2.2.0
*/
@@ -5326,9 +5491,9 @@ object functions {
Column.fn("try_to_timestamp", s, format)
/**
- * Parses the `s` to a timestamp. The function always returns null on an invalid
- * input with`/`without ANSI SQL mode enabled. It follows casting rules to a timestamp. The
- * result data type is consistent with the value of configuration `spark.sql.timestampType`.
+ * Parses the `s` to a timestamp. The function always returns null on an invalid input
+ * with`/`without ANSI SQL mode enabled. It follows casting rules to a timestamp. The result
+ * data type is consistent with the value of configuration `spark.sql.timestampType`.
*
* @group datetime_funcs
* @since 3.5.0
@@ -5346,15 +5511,17 @@ object functions {
/**
* Converts the column into a `DateType` with a specified format
*
- * See
- * Datetime Patterns
- * for valid date and time format patterns
- *
- * @param e A date, timestamp or string. If a string, the data must be in a format that can be
- * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param fmt A date time pattern detailing the format of `e` when `e`is a string
- * @return A date, or null if `e` was a string that could not be cast to a date or `fmt` was an
- * invalid format
+ * See Datetime
+ * Patterns for valid date and time format patterns
+ *
+ * @param e
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param fmt
+ * A date time pattern detailing the format of `e` when `e`is a string
+ * @return
+ * A date, or null if `e` was a string that could not be cast to a date or `fmt` was an
+ * invalid format
* @group datetime_funcs
* @since 2.2.0
*/
@@ -5377,8 +5544,8 @@ object functions {
def unix_micros(e: Column): Column = Column.fn("unix_micros", e)
/**
- * Returns the number of milliseconds since 1970-01-01 00:00:00 UTC.
- * Truncates higher levels of precision.
+ * Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of
+ * precision.
*
* @group datetime_funcs
* @since 3.5.0
@@ -5386,8 +5553,8 @@ object functions {
def unix_millis(e: Column): Column = Column.fn("unix_millis", e)
/**
- * Returns the number of seconds since 1970-01-01 00:00:00 UTC.
- * Truncates higher levels of precision.
+ * Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of
+ * precision.
*
* @group datetime_funcs
* @since 3.5.0
@@ -5399,14 +5566,16 @@ object functions {
*
* For example, `trunc("2018-11-19 12:01:19", "year")` returns 2018-01-01
*
- * @param date A date, timestamp or string. If a string, the data must be in a format that can be
- * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param format: 'year', 'yyyy', 'yy' to truncate by year,
- * or 'month', 'mon', 'mm' to truncate by month
- * Other options are: 'week', 'quarter'
+ * @param date
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param format:
+ * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month
+ * Other options are: 'week', 'quarter'
*
- * @return A date, or null if `date` was a string that could not be cast to a date or `format`
- * was an invalid value
+ * @return
+ * A date, or null if `date` was a string that could not be cast to a date or `format` was an
+ * invalid value
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5417,15 +5586,16 @@ object functions {
*
* For example, `date_trunc("year", "2018-11-19 12:01:19")` returns 2018-01-01 00:00:00
*
- * @param format: 'year', 'yyyy', 'yy' to truncate by year,
- * 'month', 'mon', 'mm' to truncate by month,
- * 'day', 'dd' to truncate by day,
- * Other options are:
- * 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'
- * @param timestamp A date, timestamp or string. If a string, the data must be in a format that
- * can be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @return A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp
- * or `format` was an invalid value
+ * @param format:
+ * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day',
+ * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second',
+ * 'minute', 'hour', 'week', 'quarter'
+ * @param timestamp
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @return
+ * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or
+ * `format` was an invalid value
* @group datetime_funcs
* @since 2.3.0
*/
@@ -5434,19 +5604,21 @@ object functions {
/**
* Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders
- * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield
- * '2017-07-14 03:40:00.0'.
- *
- * @param ts A date, timestamp or string. If a string, the data must be in a format that can be
- * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param tz A string detailing the time zone ID that the input should be adjusted to. It should
- * be in the format of either region-based zone IDs or zone offsets. Region IDs must
- * have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
- * the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
- * supported as aliases of '+00:00'. Other short names are not recommended to use
- * because they can be ambiguous.
- * @return A timestamp, or null if `ts` was a string that could not be cast to a timestamp or
- * `tz` was an invalid value
+ * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14
+ * 03:40:00.0'.
+ *
+ * @param ts
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param tz
+ * A string detailing the time zone ID that the input should be adjusted to. It should be in
+ * the format of either region-based zone IDs or zone offsets. Region IDs must have the form
+ * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format
+ * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases
+ * of '+00:00'. Other short names are not recommended to use because they can be ambiguous.
+ * @return
+ * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was
+ * an invalid value
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5454,8 +5626,8 @@ object functions {
/**
* Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders
- * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield
- * '2017-07-14 03:40:00.0'.
+ * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14
+ * 03:40:00.0'.
* @group datetime_funcs
* @since 2.4.0
*/
@@ -5467,16 +5639,18 @@ object functions {
* zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield
* '2017-07-14 01:40:00.0'.
*
- * @param ts A date, timestamp or string. If a string, the data must be in a format that can be
- * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
- * @param tz A string detailing the time zone ID that the input should be adjusted to. It should
- * be in the format of either region-based zone IDs or zone offsets. Region IDs must
- * have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
- * the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
- * supported as aliases of '+00:00'. Other short names are not recommended to use
- * because they can be ambiguous.
- * @return A timestamp, or null if `ts` was a string that could not be cast to a timestamp or
- * `tz` was an invalid value
+ * @param ts
+ * A date, timestamp or string. If a string, the data must be in a format that can be cast to
+ * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+ * @param tz
+ * A string detailing the time zone ID that the input should be adjusted to. It should be in
+ * the format of either region-based zone IDs or zone offsets. Region IDs must have the form
+ * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format
+ * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases
+ * of '+00:00'. Other short names are not recommended to use because they can be ambiguous.
+ * @return
+ * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was
+ * an invalid value
* @group datetime_funcs
* @since 1.5.0
*/
@@ -5495,8 +5669,8 @@ object functions {
* Bucketize rows into one or more time windows given a timestamp specifying column. Window
* starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window
* [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in
- * the order of months are not supported. The following example takes the average stock price for
- * a one minute window every 10 seconds starting 5 seconds after the hour:
+ * the order of months are not supported. The following example takes the average stock price
+ * for a one minute window every 10 seconds starting 5 seconds after the hour:
*
* {{{
* val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType
@@ -5515,23 +5689,23 @@ object functions {
* For a streaming query, you may use the function `current_timestamp` to generate windows on
* processing time.
*
- * @param timeColumn The column or the expression to use as the timestamp for windowing by time.
- * The time column must be of TimestampType or TimestampNTZType.
- * @param windowDuration A string specifying the width of the window, e.g. `10 minutes`,
- * `1 second`. Check `org.apache.spark.unsafe.types.CalendarInterval` for
- * valid duration identifiers. Note that the duration is a fixed length of
- * time, and does not vary over time according to a calendar. For example,
- * `1 day` always means 86,400,000 milliseconds, not a calendar day.
- * @param slideDuration A string specifying the sliding interval of the window, e.g. `1 minute`.
- * A new window will be generated every `slideDuration`. Must be less than
- * or equal to the `windowDuration`. Check
- * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration
- * identifiers. This duration is likewise absolute, and does not vary
- * according to a calendar.
- * @param startTime The offset with respect to 1970-01-01 00:00:00 UTC with which to start
- * window intervals. For example, in order to have hourly tumbling windows that
- * start 15 minutes past the hour, e.g. 12:15-13:15, 13:15-14:15... provide
- * `startTime` as `15 minutes`.
+ * @param timeColumn
+ * The column or the expression to use as the timestamp for windowing by time. The time column
+ * must be of TimestampType or TimestampNTZType.
+ * @param windowDuration
+ * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check
+ * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that
+ * the duration is a fixed length of time, and does not vary over time according to a
+ * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day.
+ * @param slideDuration
+ * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will
+ * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`.
+ * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This
+ * duration is likewise absolute, and does not vary according to a calendar.
+ * @param startTime
+ * The offset with respect to 1970-01-01 00:00:00 UTC with which to start window intervals.
+ * For example, in order to have hourly tumbling windows that start 15 minutes past the hour,
+ * e.g. 12:15-13:15, 13:15-14:15... provide `startTime` as `15 minutes`.
*
* @group datetime_funcs
* @since 2.0.0
@@ -5547,8 +5721,9 @@ object functions {
* Bucketize rows into one or more time windows given a timestamp specifying column. Window
* starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window
* [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in
- * the order of months are not supported. The windows start beginning at 1970-01-01 00:00:00 UTC.
- * The following example takes the average stock price for a one minute window every 10 seconds:
+ * the order of months are not supported. The windows start beginning at 1970-01-01 00:00:00
+ * UTC. The following example takes the average stock price for a one minute window every 10
+ * seconds:
*
* {{{
* val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType
@@ -5567,19 +5742,19 @@ object functions {
* For a streaming query, you may use the function `current_timestamp` to generate windows on
* processing time.
*
- * @param timeColumn The column or the expression to use as the timestamp for windowing by time.
- * The time column must be of TimestampType or TimestampNTZType.
- * @param windowDuration A string specifying the width of the window, e.g. `10 minutes`,
- * `1 second`. Check `org.apache.spark.unsafe.types.CalendarInterval` for
- * valid duration identifiers. Note that the duration is a fixed length of
- * time, and does not vary over time according to a calendar. For example,
- * `1 day` always means 86,400,000 milliseconds, not a calendar day.
- * @param slideDuration A string specifying the sliding interval of the window, e.g. `1 minute`.
- * A new window will be generated every `slideDuration`. Must be less than
- * or equal to the `windowDuration`. Check
- * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration
- * identifiers. This duration is likewise absolute, and does not vary
- * according to a calendar.
+ * @param timeColumn
+ * The column or the expression to use as the timestamp for windowing by time. The time column
+ * must be of TimestampType or TimestampNTZType.
+ * @param windowDuration
+ * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check
+ * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that
+ * the duration is a fixed length of time, and does not vary over time according to a
+ * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day.
+ * @param slideDuration
+ * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will
+ * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`.
+ * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This
+ * duration is likewise absolute, and does not vary according to a calendar.
*
* @group datetime_funcs
* @since 2.0.0
@@ -5589,11 +5764,11 @@ object functions {
}
/**
- * Generates tumbling time windows given a timestamp specifying column. Window
- * starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window
- * [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in
- * the order of months are not supported. The windows start beginning at 1970-01-01 00:00:00 UTC.
- * The following example takes the average stock price for a one minute tumbling window:
+ * Generates tumbling time windows given a timestamp specifying column. Window starts are
+ * inclusive but the window ends are exclusive, e.g. 12:05 will be in the window [12:05,12:10)
+ * but not in [12:00,12:05). Windows can support microsecond precision. Windows in the order of
+ * months are not supported. The windows start beginning at 1970-01-01 00:00:00 UTC. The
+ * following example takes the average stock price for a one minute tumbling window:
*
* {{{
* val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType
@@ -5612,11 +5787,12 @@ object functions {
* For a streaming query, you may use the function `current_timestamp` to generate windows on
* processing time.
*
- * @param timeColumn The column or the expression to use as the timestamp for windowing by time.
- * The time column must be of TimestampType or TimestampNTZType.
- * @param windowDuration A string specifying the width of the window, e.g. `10 minutes`,
- * `1 second`. Check `org.apache.spark.unsafe.types.CalendarInterval` for
- * valid duration identifiers.
+ * @param timeColumn
+ * The column or the expression to use as the timestamp for windowing by time. The time column
+ * must be of TimestampType or TimestampNTZType.
+ * @param windowDuration
+ * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check
+ * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers.
*
* @group datetime_funcs
* @since 2.0.0
@@ -5632,8 +5808,9 @@ object functions {
* inclusive and end is exclusive. Since event time can support microsecond precision,
* window_time(window) = window.end - 1 microsecond.
*
- * @param windowColumn The window column (typically produced by window aggregation) of type
- * StructType { start: Timestamp, end: Timestamp }
+ * @param windowColumn
+ * The window column (typically produced by window aggregation) of type StructType { start:
+ * Timestamp, end: Timestamp }
*
* @group datetime_funcs
* @since 3.4.0
@@ -5644,10 +5821,9 @@ object functions {
* Generates session window given a timestamp specifying column.
*
* Session window is one of dynamic windows, which means the length of window is varying
- * according to the given inputs. The length of session window is defined as "the timestamp
- * of latest input of the session + gap duration", so when the new inputs are bound to the
- * current session window, the end time of session window can be expanded according to the new
- * inputs.
+ * according to the given inputs. The length of session window is defined as "the timestamp of
+ * latest input of the session + gap duration", so when the new inputs are bound to the current
+ * session window, the end time of session window can be expanded according to the new inputs.
*
* Windows can support microsecond precision. gapDuration in the order of months are not
* supported.
@@ -5655,11 +5831,12 @@ object functions {
* For a streaming query, you may use the function `current_timestamp` to generate windows on
* processing time.
*
- * @param timeColumn The column or the expression to use as the timestamp for windowing by time.
- * The time column must be of TimestampType or TimestampNTZType.
- * @param gapDuration A string specifying the timeout of the session, e.g. `10 minutes`,
- * `1 second`. Check `org.apache.spark.unsafe.types.CalendarInterval` for
- * valid duration identifiers.
+ * @param timeColumn
+ * The column or the expression to use as the timestamp for windowing by time. The time column
+ * must be of TimestampType or TimestampNTZType.
+ * @param gapDuration
+ * A string specifying the timeout of the session, e.g. `10 minutes`, `1 second`. Check
+ * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers.
*
* @group datetime_funcs
* @since 3.2.0
@@ -5671,17 +5848,17 @@ object functions {
* Generates session window given a timestamp specifying column.
*
* Session window is one of dynamic windows, which means the length of window is varying
- * according to the given inputs. For static gap duration, the length of session window
- * is defined as "the timestamp of latest input of the session + gap duration", so when
- * the new inputs are bound to the current session window, the end time of session window
- * can be expanded according to the new inputs.
- *
- * Besides a static gap duration value, users can also provide an expression to specify
- * gap duration dynamically based on the input row. With dynamic gap duration, the closing
- * of a session window does not depend on the latest input anymore. A session window's range
- * is the union of all events' ranges which are determined by event start time and evaluated
- * gap duration during the query execution. Note that the rows with negative or zero gap
- * duration will be filtered out from the aggregation.
+ * according to the given inputs. For static gap duration, the length of session window is
+ * defined as "the timestamp of latest input of the session + gap duration", so when the new
+ * inputs are bound to the current session window, the end time of session window can be
+ * expanded according to the new inputs.
+ *
+ * Besides a static gap duration value, users can also provide an expression to specify gap
+ * duration dynamically based on the input row. With dynamic gap duration, the closing of a
+ * session window does not depend on the latest input anymore. A session window's range is the
+ * union of all events' ranges which are determined by event start time and evaluated gap
+ * duration during the query execution. Note that the rows with negative or zero gap duration
+ * will be filtered out from the aggregation.
*
* Windows can support microsecond precision. gapDuration in the order of months are not
* supported.
@@ -5689,11 +5866,13 @@ object functions {
* For a streaming query, you may use the function `current_timestamp` to generate windows on
* processing time.
*
- * @param timeColumn The column or the expression to use as the timestamp for windowing by time.
- * The time column must be of TimestampType or TimestampNTZType.
- * @param gapDuration A column specifying the timeout of the session. It could be static value,
- * e.g. `10 minutes`, `1 second`, or an expression/UDF that specifies gap
- * duration dynamically based on the input row.
+ * @param timeColumn
+ * The column or the expression to use as the timestamp for windowing by time. The time column
+ * must be of TimestampType or TimestampNTZType.
+ * @param gapDuration
+ * A column specifying the timeout of the session. It could be static value, e.g. `10
+ * minutes`, `1 second`, or an expression/UDF that specifies gap duration dynamically based on
+ * the input row.
*
* @group datetime_funcs
* @since 3.2.0
@@ -5702,8 +5881,7 @@ object functions {
Column.fn("session_window", timeColumn, gapDuration)
/**
- * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z)
- * to a timestamp.
+ * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z) to a timestamp.
* @group datetime_funcs
* @since 3.1.0
*/
@@ -5726,8 +5904,8 @@ object functions {
def timestamp_micros(e: Column): Column = Column.fn("timestamp_micros", e)
/**
- * Gets the difference between the timestamps in the specified units by truncating
- * the fraction part.
+ * Gets the difference between the timestamps in the specified units by truncating the fraction
+ * part.
*
* @group datetime_funcs
* @since 4.0.0
@@ -5745,8 +5923,8 @@ object functions {
Column.internalFn("timestampadd", lit(unit), quantity, ts)
/**
- * Parses the `timestamp` expression with the `format` expression
- * to a timestamp without time zone. Returns null with invalid input.
+ * Parses the `timestamp` expression with the `format` expression to a timestamp without time
+ * zone. Returns null with invalid input.
*
* @group datetime_funcs
* @since 3.5.0
@@ -5765,8 +5943,8 @@ object functions {
Column.fn("to_timestamp_ltz", timestamp)
/**
- * Parses the `timestamp_str` expression with the `format` expression
- * to a timestamp without time zone. Returns null with invalid input.
+ * Parses the `timestamp_str` expression with the `format` expression to a timestamp without
+ * time zone. Returns null with invalid input.
*
* @group datetime_funcs
* @since 3.5.0
@@ -5855,9 +6033,12 @@ object functions {
* Returns an array containing all the elements in `x` from index `start` (or starting from the
* end if `start` is negative) with the specified `length`.
*
- * @param x the array column to be sliced
- * @param start the starting index
- * @param length the length of the slice
+ * @param x
+ * the array column to be sliced
+ * @param start
+ * the starting index
+ * @param length
+ * the length of the slice
*
* @group array_funcs
* @since 2.4.0
@@ -5869,9 +6050,12 @@ object functions {
* Returns an array containing all the elements in `x` from index `start` (or starting from the
* end if `start` is negative) with the specified `length`.
*
- * @param x the array column to be sliced
- * @param start the starting index
- * @param length the length of the slice
+ * @param x
+ * the array column to be sliced
+ * @param start
+ * the starting index
+ * @param length
+ * the length of the slice
*
* @group array_funcs
* @since 3.1.0
@@ -5897,10 +6081,11 @@ object functions {
Column.fn("array_join", column, lit(delimiter))
/**
- * Concatenates multiple input columns together into a single column.
- * The function works with strings, binary and compatible array columns.
+ * Concatenates multiple input columns together into a single column. The function works with
+ * strings, binary and compatible array columns.
*
- * @note Returns null if any of the input columns are null.
+ * @note
+ * Returns null if any of the input columns are null.
*
* @group collection_funcs
* @since 1.5.0
@@ -5909,11 +6094,12 @@ object functions {
def concat(exprs: Column*): Column = Column.fn("concat", exprs: _*)
/**
- * Locates the position of the first occurrence of the value in the given array as long.
- * Returns null if either of the arguments are null.
+ * Locates the position of the first occurrence of the value in the given array as long. Returns
+ * null if either of the arguments are null.
*
- * @note The position is not zero based, but 1 based index. Returns 0 if value
- * could not be found in array.
+ * @note
+ * The position is not zero based, but 1 based index. Returns 0 if value could not be found in
+ * array.
*
* @group array_funcs
* @since 2.4.0
@@ -5922,8 +6108,8 @@ object functions {
Column.fn("array_position", column, lit(value))
/**
- * Returns element of array at given index in value if column is array. Returns value for
- * the given key in value if column is map.
+ * Returns element of array at given index in value if column is array. Returns value for the
+ * given key in value if column is map.
*
* @group collection_funcs
* @since 2.4.0
@@ -5945,8 +6131,8 @@ object functions {
Column.fn("try_element_at", column, value)
/**
- * Returns element of array at given (0-based) index. If the index points
- * outside of the array boundaries, then this function returns NULL.
+ * Returns element of array at given (0-based) index. If the index points outside of the array
+ * boundaries, then this function returns NULL.
*
* @group array_funcs
* @since 3.4.0
@@ -5955,8 +6141,8 @@ object functions {
/**
* Sorts the input array in ascending order. The elements of the input array must be orderable.
- * NaN is greater than any non-NaN elements for double/float type.
- * Null elements will be placed at the end of the returned array.
+ * NaN is greater than any non-NaN elements for double/float type. Null elements will be placed
+ * at the end of the returned array.
*
* @group collection_funcs
* @since 2.4.0
@@ -6010,8 +6196,8 @@ object functions {
def array_distinct(e: Column): Column = Column.fn("array_distinct", e)
/**
- * Returns an array of the elements in the intersection of the given two arrays,
- * without duplicates.
+ * Returns an array of the elements in the intersection of the given two arrays, without
+ * duplicates.
*
* @group array_funcs
* @since 2.4.0
@@ -6038,8 +6224,8 @@ object functions {
Column.fn("array_union", col1, col2)
/**
- * Returns an array of the elements in the first array but not in the second array,
- * without duplicates. The order of elements in the result is not determined
+ * Returns an array of the elements in the first array but not in the second array, without
+ * duplicates. The order of elements in the result is not determined
*
* @group array_funcs
* @since 2.4.0
@@ -6047,7 +6233,6 @@ object functions {
def array_except(col1: Column, col2: Column): Column =
Column.fn("array_except", col1, col2)
-
private def createLambda(f: Column => Column) = {
val x = internal.UnresolvedNamedLambdaVariable("x")
val function = f(Column(x)).node
@@ -6070,14 +6255,16 @@ object functions {
}
/**
- * Returns an array of elements after applying a transformation to each element
- * in the input array.
+ * Returns an array of elements after applying a transformation to each element in the input
+ * array.
* {{{
* df.select(transform(col("i"), x => x + 1))
* }}}
*
- * @param column the input array column
- * @param f col => transformed_col, the lambda function to transform the input column
+ * @param column
+ * the input array column
+ * @param f
+ * col => transformed_col, the lambda function to transform the input column
*
* @group collection_funcs
* @since 3.0.0
@@ -6086,15 +6273,17 @@ object functions {
Column.fn("transform", column, createLambda(f))
/**
- * Returns an array of elements after applying a transformation to each element
- * in the input array.
+ * Returns an array of elements after applying a transformation to each element in the input
+ * array.
* {{{
* df.select(transform(col("i"), (x, i) => x + i))
* }}}
*
- * @param column the input array column
- * @param f (col, index) => transformed_col, the lambda function to transform the input
- * column given the index. Indices start at 0.
+ * @param column
+ * the input array column
+ * @param f
+ * (col, index) => transformed_col, the lambda function to transform the input column given
+ * the index. Indices start at 0.
*
* @group collection_funcs
* @since 3.0.0
@@ -6108,8 +6297,10 @@ object functions {
* df.select(exists(col("i"), _ % 2 === 0))
* }}}
*
- * @param column the input array column
- * @param f col => predicate, the Boolean predicate to check the input column
+ * @param column
+ * the input array column
+ * @param f
+ * col => predicate, the Boolean predicate to check the input column
*
* @group collection_funcs
* @since 3.0.0
@@ -6123,8 +6314,10 @@ object functions {
* df.select(forall(col("i"), x => x % 2 === 0))
* }}}
*
- * @param column the input array column
- * @param f col => predicate, the Boolean predicate to check the input column
+ * @param column
+ * the input array column
+ * @param f
+ * col => predicate, the Boolean predicate to check the input column
*
* @group collection_funcs
* @since 3.0.0
@@ -6138,8 +6331,10 @@ object functions {
* df.select(filter(col("s"), x => x % 2 === 0))
* }}}
*
- * @param column the input array column
- * @param f col => predicate, the Boolean predicate to filter the input column
+ * @param column
+ * the input array column
+ * @param f
+ * col => predicate, the Boolean predicate to filter the input column
*
* @group collection_funcs
* @since 3.0.0
@@ -6153,9 +6348,11 @@ object functions {
* df.select(filter(col("s"), (x, i) => i % 2 === 0))
* }}}
*
- * @param column the input array column
- * @param f (col, index) => predicate, the Boolean predicate to filter the input column
- * given the index. Indices start at 0.
+ * @param column
+ * the input array column
+ * @param f
+ * (col, index) => predicate, the Boolean predicate to filter the input column given the
+ * index. Indices start at 0.
*
* @group collection_funcs
* @since 3.0.0
@@ -6164,19 +6361,23 @@ object functions {
Column.fn("filter", column, createLambda(f))
/**
- * Applies a binary operator to an initial state and all elements in the array,
- * and reduces this to a single state. The final state is converted into the final result
- * by applying a finish function.
+ * Applies a binary operator to an initial state and all elements in the array, and reduces this
+ * to a single state. The final state is converted into the final result by applying a finish
+ * function.
* {{{
* df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10))
* }}}
*
- * @param expr the input array column
- * @param initialValue the initial value
- * @param merge (combined_value, input_value) => combined_value, the merge function to merge
- * an input value to the combined_value
- * @param finish combined_value => final_value, the lambda function to convert the combined value
- * of all inputs to final result
+ * @param expr
+ * the input array column
+ * @param initialValue
+ * the initial value
+ * @param merge
+ * (combined_value, input_value) => combined_value, the merge function to merge an input value
+ * to the combined_value
+ * @param finish
+ * combined_value => final_value, the lambda function to convert the combined value of all
+ * inputs to final result
*
* @group collection_funcs
* @since 3.0.0
@@ -6189,16 +6390,19 @@ object functions {
Column.fn("aggregate", expr, initialValue, createLambda(merge), createLambda(finish))
/**
- * Applies a binary operator to an initial state and all elements in the array,
- * and reduces this to a single state.
+ * Applies a binary operator to an initial state and all elements in the array, and reduces this
+ * to a single state.
* {{{
* df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x))
* }}}
*
- * @param expr the input array column
- * @param initialValue the initial value
- * @param merge (combined_value, input_value) => combined_value, the merge function to merge
- * an input value to the combined_value
+ * @param expr
+ * the input array column
+ * @param initialValue
+ * the initial value
+ * @param merge
+ * (combined_value, input_value) => combined_value, the merge function to merge an input value
+ * to the combined_value
* @group collection_funcs
* @since 3.0.0
*/
@@ -6206,19 +6410,23 @@ object functions {
aggregate(expr, initialValue, merge, c => c)
/**
- * Applies a binary operator to an initial state and all elements in the array,
- * and reduces this to a single state. The final state is converted into the final result
- * by applying a finish function.
+ * Applies a binary operator to an initial state and all elements in the array, and reduces this
+ * to a single state. The final state is converted into the final result by applying a finish
+ * function.
* {{{
* df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10))
* }}}
*
- * @param expr the input array column
- * @param initialValue the initial value
- * @param merge (combined_value, input_value) => combined_value, the merge function to merge
- * an input value to the combined_value
- * @param finish combined_value => final_value, the lambda function to convert the combined value
- * of all inputs to final result
+ * @param expr
+ * the input array column
+ * @param initialValue
+ * the initial value
+ * @param merge
+ * (combined_value, input_value) => combined_value, the merge function to merge an input value
+ * to the combined_value
+ * @param finish
+ * combined_value => final_value, the lambda function to convert the combined value of all
+ * inputs to final result
*
* @group collection_funcs
* @since 3.5.0
@@ -6231,16 +6439,19 @@ object functions {
Column.fn("reduce", expr, initialValue, createLambda(merge), createLambda(finish))
/**
- * Applies a binary operator to an initial state and all elements in the array,
- * and reduces this to a single state.
+ * Applies a binary operator to an initial state and all elements in the array, and reduces this
+ * to a single state.
* {{{
* df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x))
* }}}
*
- * @param expr the input array column
- * @param initialValue the initial value
- * @param merge (combined_value, input_value) => combined_value, the merge function to merge
- * an input value to the combined_value
+ * @param expr
+ * the input array column
+ * @param initialValue
+ * the initial value
+ * @param merge
+ * (combined_value, input_value) => combined_value, the merge function to merge an input value
+ * to the combined_value
* @group collection_funcs
* @since 3.5.0
*/
@@ -6248,16 +6459,19 @@ object functions {
reduce(expr, initialValue, merge, c => c)
/**
- * Merge two given arrays, element-wise, into a single array using a function.
- * If one array is shorter, nulls are appended at the end to match the length of the longer
- * array, before applying the function.
+ * Merge two given arrays, element-wise, into a single array using a function. If one array is
+ * shorter, nulls are appended at the end to match the length of the longer array, before
+ * applying the function.
* {{{
* df.select(zip_with(df1("val1"), df1("val2"), (x, y) => x + y))
* }}}
*
- * @param left the left input array column
- * @param right the right input array column
- * @param f (lCol, rCol) => col, the lambda function to merge two input columns into one column
+ * @param left
+ * the left input array column
+ * @param right
+ * the right input array column
+ * @param f
+ * (lCol, rCol) => col, the lambda function to merge two input columns into one column
*
* @group collection_funcs
* @since 3.0.0
@@ -6266,14 +6480,16 @@ object functions {
Column.fn("zip_with", left, right, createLambda(f))
/**
- * Applies a function to every key-value pair in a map and returns
- * a map with the results of those applications as the new keys for the pairs.
+ * Applies a function to every key-value pair in a map and returns a map with the results of
+ * those applications as the new keys for the pairs.
* {{{
* df.select(transform_keys(col("i"), (k, v) => k + v))
* }}}
*
- * @param expr the input map column
- * @param f (key, value) => new_key, the lambda function to transform the key of input map column
+ * @param expr
+ * the input map column
+ * @param f
+ * (key, value) => new_key, the lambda function to transform the key of input map column
*
* @group collection_funcs
* @since 3.0.0
@@ -6282,15 +6498,16 @@ object functions {
Column.fn("transform_keys", expr, createLambda(f))
/**
- * Applies a function to every key-value pair in a map and returns
- * a map with the results of those applications as the new values for the pairs.
+ * Applies a function to every key-value pair in a map and returns a map with the results of
+ * those applications as the new values for the pairs.
* {{{
* df.select(transform_values(col("i"), (k, v) => k + v))
* }}}
*
- * @param expr the input map column
- * @param f (key, value) => new_value, the lambda function to transform the value of input map
- * column
+ * @param expr
+ * the input map column
+ * @param f
+ * (key, value) => new_value, the lambda function to transform the value of input map column
*
* @group collection_funcs
* @since 3.0.0
@@ -6304,8 +6521,10 @@ object functions {
* df.select(map_filter(col("m"), (k, v) => k * 10 === v))
* }}}
*
- * @param expr the input map column
- * @param f (key, value) => predicate, the Boolean predicate to filter the input map column
+ * @param expr
+ * the input map column
+ * @param f
+ * (key, value) => predicate, the Boolean predicate to filter the input map column
*
* @group collection_funcs
* @since 3.0.0
@@ -6319,9 +6538,12 @@ object functions {
* df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2))
* }}}
*
- * @param left the left input map column
- * @param right the right input map column
- * @param f (key, value1, value2) => new_value, the lambda function to merge the map values
+ * @param left
+ * the left input map column
+ * @param right
+ * the right input map column
+ * @param f
+ * (key, value1, value2) => new_value, the lambda function to merge the map values
*
* @group collection_funcs
* @since 3.0.0
@@ -6330,9 +6552,9 @@ object functions {
Column.fn("map_zip_with", left, right, createLambda(f))
/**
- * Creates a new row for each element in the given array or map column.
- * Uses the default column name `col` for elements in the array and
- * `key` and `value` for elements in the map unless specified otherwise.
+ * Creates a new row for each element in the given array or map column. Uses the default column
+ * name `col` for elements in the array and `key` and `value` for elements in the map unless
+ * specified otherwise.
*
* @group generator_funcs
* @since 1.3.0
@@ -6340,10 +6562,9 @@ object functions {
def explode(e: Column): Column = Column.fn("explode", e)
/**
- * Creates a new row for each element in the given array or map column.
- * Uses the default column name `col` for elements in the array and
- * `key` and `value` for elements in the map unless specified otherwise.
- * Unlike explode, if the array/map is null or empty then null is produced.
+ * Creates a new row for each element in the given array or map column. Uses the default column
+ * name `col` for elements in the array and `key` and `value` for elements in the map unless
+ * specified otherwise. Unlike explode, if the array/map is null or empty then null is produced.
*
* @group generator_funcs
* @since 2.2.0
@@ -6351,9 +6572,9 @@ object functions {
def explode_outer(e: Column): Column = Column.fn("explode_outer", e)
/**
- * Creates a new row for each element with position in the given array or map column.
- * Uses the default column name `pos` for position, and `col` for elements in the array
- * and `key` and `value` for elements in the map unless specified otherwise.
+ * Creates a new row for each element with position in the given array or map column. Uses the
+ * default column name `pos` for position, and `col` for elements in the array and `key` and
+ * `value` for elements in the map unless specified otherwise.
*
* @group generator_funcs
* @since 2.1.0
@@ -6361,17 +6582,17 @@ object functions {
def posexplode(e: Column): Column = Column.fn("posexplode", e)
/**
- * Creates a new row for each element with position in the given array or map column.
- * Uses the default column name `pos` for position, and `col` for elements in the array
- * and `key` and `value` for elements in the map unless specified otherwise.
- * Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced.
+ * Creates a new row for each element with position in the given array or map column. Uses the
+ * default column name `pos` for position, and `col` for elements in the array and `key` and
+ * `value` for elements in the map unless specified otherwise. Unlike posexplode, if the
+ * array/map is null or empty then the row (null, null) is produced.
*
* @group generator_funcs
* @since 2.2.0
*/
def posexplode_outer(e: Column): Column = Column.fn("posexplode_outer", e)
- /**
+ /**
* Creates a new row for each element in the given array of structs.
*
* @group generator_funcs
@@ -6380,8 +6601,8 @@ object functions {
def inline(e: Column): Column = Column.fn("inline", e)
/**
- * Creates a new row for each element in the given array of structs.
- * Unlike inline, if the array is null or empty then null is produced for each nested column.
+ * Creates a new row for each element in the given array of structs. Unlike inline, if the array
+ * is null or empty then null is produced for each nested column.
*
* @group generator_funcs
* @since 3.4.0
@@ -6415,14 +6636,15 @@ object functions {
* (Scala-specific) Parses a column containing a JSON string into a `StructType` with the
* specified schema. Returns `null`, in the case of an unparseable string.
*
- * @param e a string column containing JSON data.
- * @param schema the schema to use when parsing the json string
- * @param options options to control how the json is parsed. Accepts the same options as the
- * json data source.
- * See
- *
- * Data Source Option in the version you use.
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema to use when parsing the json string
+ * @param options
+ * options to control how the json is parsed. Accepts the same options as the json data
+ * source. See Data
+ * Source Option in the version you use.
*
* @group json_funcs
* @since 2.1.0
@@ -6434,17 +6656,18 @@ object functions {
// scalastyle:off line.size.limit
/**
* (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType`
- * as keys type, `StructType` or `ArrayType` with the specified schema.
- * Returns `null`, in the case of an unparseable string.
- *
- * @param e a string column containing JSON data.
- * @param schema the schema to use when parsing the json string
- * @param options options to control how the json is parsed. accepts the same options and the
- * json data source.
- * See
- *
- * Data Source Option in the version you use.
+ * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the
+ * case of an unparseable string.
+ *
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema to use when parsing the json string
+ * @param options
+ * options to control how the json is parsed. accepts the same options and the json data
+ * source. See Data
+ * Source Option in the version you use.
*
* @group json_funcs
* @since 2.2.0
@@ -6459,14 +6682,15 @@ object functions {
* (Java-specific) Parses a column containing a JSON string into a `StructType` with the
* specified schema. Returns `null`, in the case of an unparseable string.
*
- * @param e a string column containing JSON data.
- * @param schema the schema to use when parsing the json string
- * @param options options to control how the json is parsed. accepts the same options and the
- * json data source.
- * See
- *
- * Data Source Option in the version you use.
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema to use when parsing the json string
+ * @param options
+ * options to control how the json is parsed. accepts the same options and the json data
+ * source. See Data
+ * Source Option in the version you use.
*
* @group json_funcs
* @since 2.1.0
@@ -6478,17 +6702,18 @@ object functions {
// scalastyle:off line.size.limit
/**
* (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType`
- * as keys type, `StructType` or `ArrayType` with the specified schema.
- * Returns `null`, in the case of an unparseable string.
- *
- * @param e a string column containing JSON data.
- * @param schema the schema to use when parsing the json string
- * @param options options to control how the json is parsed. accepts the same options and the
- * json data source.
- * See
- *
- * Data Source Option in the version you use.
+ * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the
+ * case of an unparseable string.
+ *
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema to use when parsing the json string
+ * @param options
+ * options to control how the json is parsed. accepts the same options and the json data
+ * source. See Data
+ * Source Option in the version you use.
*
* @group json_funcs
* @since 2.2.0
@@ -6502,8 +6727,10 @@ object functions {
* Parses a column containing a JSON string into a `StructType` with the specified schema.
* Returns `null`, in the case of an unparseable string.
*
- * @param e a string column containing JSON data.
- * @param schema the schema to use when parsing the json string
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema to use when parsing the json string
*
* @group json_funcs
* @since 2.1.0
@@ -6513,11 +6740,13 @@ object functions {
/**
* Parses a column containing a JSON string into a `MapType` with `StringType` as keys type,
- * `StructType` or `ArrayType` with the specified schema.
- * Returns `null`, in the case of an unparseable string.
+ * `StructType` or `ArrayType` with the specified schema. Returns `null`, in the case of an
+ * unparseable string.
*
- * @param e a string column containing JSON data.
- * @param schema the schema to use when parsing the json string
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema to use when parsing the json string
*
* @group json_funcs
* @since 2.2.0
@@ -6528,17 +6757,18 @@ object functions {
// scalastyle:off line.size.limit
/**
* (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType`
- * as keys type, `StructType` or `ArrayType` with the specified schema.
- * Returns `null`, in the case of an unparseable string.
- *
- * @param e a string column containing JSON data.
- * @param schema the schema as a DDL-formatted string.
- * @param options options to control how the json is parsed. accepts the same options and the
- * json data source.
- * See
- *
- * Data Source Option in the version you use.
+ * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the
+ * case of an unparseable string.
+ *
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema as a DDL-formatted string.
+ * @param options
+ * options to control how the json is parsed. accepts the same options and the json data
+ * source. See Data
+ * Source Option in the version you use.
*
* @group json_funcs
* @since 2.1.0
@@ -6551,17 +6781,18 @@ object functions {
// scalastyle:off line.size.limit
/**
* (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType`
- * as keys type, `StructType` or `ArrayType` with the specified schema.
- * Returns `null`, in the case of an unparseable string.
- *
- * @param e a string column containing JSON data.
- * @param schema the schema as a DDL-formatted string.
- * @param options options to control how the json is parsed. accepts the same options and the
- * json data source.
- * See
- *
- * Data Source Option in the version you use.
+ * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the
+ * case of an unparseable string.
+ *
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema as a DDL-formatted string.
+ * @param options
+ * options to control how the json is parsed. accepts the same options and the json data
+ * source. See Data
+ * Source Option in the version you use.
*
* @group json_funcs
* @since 2.3.0
@@ -6573,11 +6804,13 @@ object functions {
/**
* (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType`
- * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema.
- * Returns `null`, in the case of an unparseable string.
+ * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns
+ * `null`, in the case of an unparseable string.
*
- * @param e a string column containing JSON data.
- * @param schema the schema to use when parsing the json string
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema to use when parsing the json string
*
* @group json_funcs
* @since 2.4.0
@@ -6589,17 +6822,18 @@ object functions {
// scalastyle:off line.size.limit
/**
* (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType`
- * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema.
- * Returns `null`, in the case of an unparseable string.
- *
- * @param e a string column containing JSON data.
- * @param schema the schema to use when parsing the json string
- * @param options options to control how the json is parsed. accepts the same options and the
- * json data source.
- * See
- *
- * Data Source Option in the version you use.
+ * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns
+ * `null`, in the case of an unparseable string.
+ *
+ * @param e
+ * a string column containing JSON data.
+ * @param schema
+ * the schema to use when parsing the json string
+ * @param options
+ * options to control how the json is parsed. accepts the same options and the json data
+ * source. See Data
+ * Source Option in the version you use.
*
* @group json_funcs
* @since 2.4.0
@@ -6620,7 +6854,8 @@ object functions {
* Parses a JSON string and constructs a Variant value. Returns null if the input string is not
* a valid JSON value.
*
- * @param json a string column that contains JSON data.
+ * @param json
+ * a string column that contains JSON data.
*
* @group variant_funcs
* @since 4.0.0
@@ -6637,6 +6872,18 @@ object functions {
*/
def parse_json(json: Column): Column = Column.fn("parse_json", json)
+ /**
+ * Converts a column containing nested inputs (array/map/struct) into a variants where maps and
+ * structs are converted to variant objects which are unordered unlike SQL structs. Input maps
+ * can only have string keys.
+ *
+ * @param col
+ * a column with a nested schema or column name.
+ * @group variant_funcs
+ * @since 4.0.0
+ */
+ def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col)
+
/**
* Check if a variant value is a variant null. Returns true if and only if the input is a
* variant null and false otherwise (including in the case of SQL NULL).
@@ -6705,7 +6952,8 @@ object functions {
/**
* Parses a JSON string and infers its schema in DDL format.
*
- * @param json a JSON string.
+ * @param json
+ * a JSON string.
*
* @group json_funcs
* @since 2.4.0
@@ -6715,7 +6963,8 @@ object functions {
/**
* Parses a JSON string and infers its schema in DDL format.
*
- * @param json a foldable string column containing a JSON string.
+ * @param json
+ * a foldable string column containing a JSON string.
*
* @group json_funcs
* @since 2.4.0
@@ -6726,14 +6975,15 @@ object functions {
/**
* Parses a JSON string and infers its schema in DDL format using options.
*
- * @param json a foldable string column containing JSON data.
- * @param options options to control how the json is parsed. accepts the same options and the
- * json data source.
- * See
- *
- * Data Source Option in the version you use.
- * @return a column with string literal containing schema in DDL format.
+ * @param json
+ * a foldable string column containing JSON data.
+ * @param options
+ * options to control how the json is parsed. accepts the same options and the json data
+ * source. See Data
+ * Source Option in the version you use.
+ * @return
+ * a column with string literal containing schema in DDL format.
*
* @group json_funcs
* @since 3.0.0
@@ -6743,8 +6993,8 @@ object functions {
Column.fnWithOptions("schema_of_json", options.asScala.iterator, json)
/**
- * Returns the number of elements in the outermost JSON array. `NULL` is returned in case of
- * any other valid JSON string, `NULL` or an invalid JSON.
+ * Returns the number of elements in the outermost JSON array. `NULL` is returned in case of any
+ * other valid JSON string, `NULL` or an invalid JSON.
*
* @group json_funcs
* @since 3.5.0
@@ -6753,8 +7003,8 @@ object functions {
/**
* Returns all the keys of the outermost JSON object as an array. If a valid JSON object is
- * given, all the keys of the outermost object will be returned as an array. If it is any
- * other valid JSON string, an invalid JSON string or an empty string, the function returns null.
+ * given, all the keys of the outermost object will be returned as an array. If it is any other
+ * valid JSON string, an invalid JSON string or an empty string, the function returns null.
*
* @group json_funcs
* @since 3.5.0
@@ -6763,19 +7013,18 @@ object functions {
// scalastyle:off line.size.limit
/**
- * (Scala-specific) Converts a column containing a `StructType`, `ArrayType` or
- * a `MapType` into a JSON string with the specified schema.
- * Throws an exception, in the case of an unsupported type.
+ * (Scala-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into
+ * a JSON string with the specified schema. Throws an exception, in the case of an unsupported
+ * type.
*
- * @param e a column containing a struct, an array or a map.
- * @param options options to control how the struct column is converted into a json string.
- * accepts the same options and the json data source.
- * See
- *
- * Data Source Option in the version you use.
- * Additionally the function supports the `pretty` option which enables
- * pretty JSON generation.
+ * @param e
+ * a column containing a struct, an array or a map.
+ * @param options
+ * options to control how the struct column is converted into a json string. accepts the same
+ * options and the json data source. See Data
+ * Source Option in the version you use. Additionally the function supports the `pretty`
+ * option which enables pretty JSON generation.
*
* @group json_funcs
* @since 2.1.0
@@ -6786,19 +7035,18 @@ object functions {
// scalastyle:off line.size.limit
/**
- * (Java-specific) Converts a column containing a `StructType`, `ArrayType` or
- * a `MapType` into a JSON string with the specified schema.
- * Throws an exception, in the case of an unsupported type.
+ * (Java-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into
+ * a JSON string with the specified schema. Throws an exception, in the case of an unsupported
+ * type.
*
- * @param e a column containing a struct, an array or a map.
- * @param options options to control how the struct column is converted into a json string.
- * accepts the same options and the json data source.
- * See
- *
- * Data Source Option in the version you use.
- * Additionally the function supports the `pretty` option which enables
- * pretty JSON generation.
+ * @param e
+ * a column containing a struct, an array or a map.
+ * @param options
+ * options to control how the struct column is converted into a json string. accepts the same
+ * options and the json data source. See Data
+ * Source Option in the version you use. Additionally the function supports the `pretty`
+ * option which enables pretty JSON generation.
*
* @group json_funcs
* @since 2.1.0
@@ -6808,11 +7056,11 @@ object functions {
to_json(e, options.asScala.toMap)
/**
- * Converts a column containing a `StructType`, `ArrayType` or
- * a `MapType` into a JSON string with the specified schema.
- * Throws an exception, in the case of an unsupported type.
+ * Converts a column containing a `StructType`, `ArrayType` or a `MapType` into a JSON string
+ * with the specified schema. Throws an exception, in the case of an unsupported type.
*
- * @param e a column containing a struct, an array or a map.
+ * @param e
+ * a column containing a struct, an array or a map.
*
* @group json_funcs
* @since 2.1.0
@@ -6822,10 +7070,11 @@ object functions {
/**
* Masks the given string value. The function replaces characters with 'X' or 'x', and numbers
- * with 'n'.
- * This can be useful for creating copies of tables with sensitive information removed.
+ * with 'n'. This can be useful for creating copies of tables with sensitive information
+ * removed.
*
- * @param input string value to mask. Supported types: STRING, VARCHAR, CHAR
+ * @param input
+ * string value to mask. Supported types: STRING, VARCHAR, CHAR
*
* @group string_funcs
* @since 3.5.0
@@ -6834,8 +7083,8 @@ object functions {
/**
* Masks the given string value. The function replaces upper-case characters with specific
- * character, lower-case characters with 'x', and numbers with 'n'.
- * This can be useful for creating copies of tables with sensitive information removed.
+ * character, lower-case characters with 'x', and numbers with 'n'. This can be useful for
+ * creating copies of tables with sensitive information removed.
*
* @param input
* string value to mask. Supported types: STRING, VARCHAR, CHAR
@@ -6850,8 +7099,8 @@ object functions {
/**
* Masks the given string value. The function replaces upper-case and lower-case characters with
- * the characters specified respectively, and numbers with 'n'.
- * This can be useful for creating copies of tables with sensitive information removed.
+ * the characters specified respectively, and numbers with 'n'. This can be useful for creating
+ * copies of tables with sensitive information removed.
*
* @param input
* string value to mask. Supported types: STRING, VARCHAR, CHAR
@@ -6868,8 +7117,8 @@ object functions {
/**
* Masks the given string value. The function replaces upper-case, lower-case characters and
- * numbers with the characters specified respectively.
- * This can be useful for creating copies of tables with sensitive information removed.
+ * numbers with the characters specified respectively. This can be useful for creating copies of
+ * tables with sensitive information removed.
*
* @param input
* string value to mask. Supported types: STRING, VARCHAR, CHAR
@@ -6916,8 +7165,8 @@ object functions {
* Returns length of array or map.
*
* This function returns -1 for null input only if spark.sql.ansi.enabled is false and
- * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input.
- * With the default settings, the function returns null for null input.
+ * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the
+ * default settings, the function returns null for null input.
*
* @group collection_funcs
* @since 1.5.0
@@ -6928,8 +7177,8 @@ object functions {
* Returns length of array or map. This is an alias of `size` function.
*
* This function returns -1 for null input only if spark.sql.ansi.enabled is false and
- * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input.
- * With the default settings, the function returns null for null input.
+ * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the
+ * default settings, the function returns null for null input.
*
* @group collection_funcs
* @since 3.5.0
@@ -6937,9 +7186,9 @@ object functions {
def cardinality(e: Column): Column = Column.fn("cardinality", e)
/**
- * Sorts the input array for the given column in ascending order,
- * according to the natural ordering of the array elements.
- * Null elements will be placed at the beginning of the returned array.
+ * Sorts the input array for the given column in ascending order, according to the natural
+ * ordering of the array elements. Null elements will be placed at the beginning of the returned
+ * array.
*
* @group array_funcs
* @since 1.5.0
@@ -6947,11 +7196,10 @@ object functions {
def sort_array(e: Column): Column = sort_array(e, asc = true)
/**
- * Sorts the input array for the given column in ascending or descending order,
- * according to the natural ordering of the array elements. NaN is greater than any non-NaN
- * elements for double/float type. Null elements will be placed at the beginning of the returned
- * array in ascending order or
- * at the end of the returned array in descending order.
+ * Sorts the input array for the given column in ascending or descending order, according to the
+ * natural ordering of the array elements. NaN is greater than any non-NaN elements for
+ * double/float type. Null elements will be placed at the beginning of the returned array in
+ * ascending order or at the end of the returned array in descending order.
*
* @group array_funcs
* @since 1.5.0
@@ -6987,8 +7235,9 @@ object functions {
/**
* Aggregate function: returns a list of objects with duplicates.
*
- * @note The function is non-deterministic because the order of collected results depends
- * on the order of the rows which may be non-deterministic after a shuffle.
+ * @note
+ * The function is non-deterministic because the order of collected results depends on the
+ * order of the rows which may be non-deterministic after a shuffle.
* @group agg_funcs
* @since 3.5.0
*/
@@ -6997,7 +7246,8 @@ object functions {
/**
* Returns a random permutation of the given array.
*
- * @note The function is non-deterministic.
+ * @note
+ * The function is non-deterministic.
*
* @group array_funcs
* @since 2.4.0
@@ -7012,8 +7262,8 @@ object functions {
def reverse(e: Column): Column = Column.fn("reverse", e)
/**
- * Creates a single array from an array of arrays. If a structure of nested arrays is deeper than
- * two levels, only one level of nesting is removed.
+ * Creates a single array from an array of arrays. If a structure of nested arrays is deeper
+ * than two levels, only one level of nesting is removed.
* @group array_funcs
* @since 2.4.0
*/
@@ -7029,8 +7279,8 @@ object functions {
Column.fn("sequence", start, stop, step)
/**
- * Generate a sequence of integers from start to stop,
- * incrementing by 1 if start is less than or equal to stop, otherwise -1.
+ * Generate a sequence of integers from start to stop, incrementing by 1 if start is less than
+ * or equal to stop, otherwise -1.
*
* @group array_funcs
* @since 2.4.0
@@ -7038,8 +7288,8 @@ object functions {
def sequence(start: Column, stop: Column): Column = Column.fn("sequence", start, stop)
/**
- * Creates an array containing the left argument repeated the number of times given by the
- * right argument.
+ * Creates an array containing the left argument repeated the number of times given by the right
+ * argument.
*
* @group array_funcs
* @since 2.4.0
@@ -7047,8 +7297,8 @@ object functions {
def array_repeat(left: Column, right: Column): Column = Column.fn("array_repeat", left, right)
/**
- * Creates an array containing the left argument repeated the number of times given by the
- * right argument.
+ * Creates an array containing the left argument repeated the number of times given by the right
+ * argument.
*
* @group array_funcs
* @since 2.4.0
@@ -7113,14 +7363,15 @@ object functions {
* Parses a column containing a CSV string into a `StructType` with the specified schema.
* Returns `null`, in the case of an unparseable string.
*
- * @param e a string column containing CSV data.
- * @param schema the schema to use when parsing the CSV string
- * @param options options to control how the CSV is parsed. accepts the same options and the
- * CSV data source.
- * See
- *
- * Data Source Option in the version you use.
+ * @param e
+ * a string column containing CSV data.
+ * @param schema
+ * the schema to use when parsing the CSV string
+ * @param options
+ * options to control how the CSV is parsed. accepts the same options and the CSV data source.
+ * See Data
+ * Source Option in the version you use.
*
* @group csv_funcs
* @since 3.0.0
@@ -7131,17 +7382,18 @@ object functions {
// scalastyle:off line.size.limit
/**
- * (Java-specific) Parses a column containing a CSV string into a `StructType`
- * with the specified schema. Returns `null`, in the case of an unparseable string.
+ * (Java-specific) Parses a column containing a CSV string into a `StructType` with the
+ * specified schema. Returns `null`, in the case of an unparseable string.
*
- * @param e a string column containing CSV data.
- * @param schema the schema to use when parsing the CSV string
- * @param options options to control how the CSV is parsed. accepts the same options and the
- * CSV data source.
- * See
- *
- * Data Source Option in the version you use.
+ * @param e
+ * a string column containing CSV data.
+ * @param schema
+ * the schema to use when parsing the CSV string
+ * @param options
+ * options to control how the CSV is parsed. accepts the same options and the CSV data source.
+ * See Data
+ * Source Option in the version you use.
*
* @group csv_funcs
* @since 3.0.0
@@ -7156,7 +7408,8 @@ object functions {
/**
* Parses a CSV string and infers its schema in DDL format.
*
- * @param csv a CSV string.
+ * @param csv
+ * a CSV string.
*
* @group csv_funcs
* @since 3.0.0
@@ -7166,7 +7419,8 @@ object functions {
/**
* Parses a CSV string and infers its schema in DDL format.
*
- * @param csv a foldable string column containing a CSV string.
+ * @param csv
+ * a foldable string column containing a CSV string.
*
* @group csv_funcs
* @since 3.0.0
@@ -7177,14 +7431,15 @@ object functions {
/**
* Parses a CSV string and infers its schema in DDL format using options.
*
- * @param csv a foldable string column containing a CSV string.
- * @param options options to control how the CSV is parsed. accepts the same options and the
- * CSV data source.
- * See
- *
- * Data Source Option in the version you use.
- * @return a column with string literal containing schema in DDL format.
+ * @param csv
+ * a foldable string column containing a CSV string.
+ * @param options
+ * options to control how the CSV is parsed. accepts the same options and the CSV data source.
+ * See Data
+ * Source Option in the version you use.
+ * @return
+ * a column with string literal containing schema in DDL format.
*
* @group csv_funcs
* @since 3.0.0
@@ -7195,16 +7450,16 @@ object functions {
// scalastyle:off line.size.limit
/**
- * (Java-specific) Converts a column containing a `StructType` into a CSV string with
- * the specified schema. Throws an exception, in the case of an unsupported type.
+ * (Java-specific) Converts a column containing a `StructType` into a CSV string with the
+ * specified schema. Throws an exception, in the case of an unsupported type.
*
- * @param e a column containing a struct.
- * @param options options to control how the struct column is converted into a CSV string.
- * It accepts the same options and the CSV data source.
- * See
- *
- * Data Source Option in the version you use.
+ * @param e
+ * a column containing a struct.
+ * @param options
+ * options to control how the struct column is converted into a CSV string. It accepts the
+ * same options and the CSV data source. See Data
+ * Source Option in the version you use.
*
* @group csv_funcs
* @since 3.0.0
@@ -7217,7 +7472,8 @@ object functions {
* Converts a column containing a `StructType` into a CSV string with the specified schema.
* Throws an exception, in the case of an unsupported type.
*
- * @param e a column containing a struct.
+ * @param e
+ * a column containing a struct.
*
* @group csv_funcs
* @since 3.0.0
@@ -7226,17 +7482,18 @@ object functions {
// scalastyle:off line.size.limit
/**
- * Parses a column containing a XML string into the data type corresponding to the specified schema.
- * Returns `null`, in the case of an unparseable string.
- *
- * @param e a string column containing XML data.
- * @param schema the schema to use when parsing the XML string
- * @param options options to control how the XML is parsed. accepts the same options and the
- * XML data source.
- * See
- *
- * Data Source Option in the version you use.
+ * Parses a column containing a XML string into the data type corresponding to the specified
+ * schema. Returns `null`, in the case of an unparseable string.
+ *
+ * @param e
+ * a string column containing XML data.
+ * @param schema
+ * the schema to use when parsing the XML string
+ * @param options
+ * options to control how the XML is parsed. accepts the same options and the XML data source.
+ * See Data
+ * Source Option in the version you use.
* @group xml_funcs
* @since 4.0.0
*/
@@ -7246,18 +7503,18 @@ object functions {
// scalastyle:off line.size.limit
/**
- * (Java-specific) Parses a column containing a XML string into a `StructType`
- * with the specified schema.
- * Returns `null`, in the case of an unparseable string.
+ * (Java-specific) Parses a column containing a XML string into a `StructType` with the
+ * specified schema. Returns `null`, in the case of an unparseable string.
*
- * @param e a string column containing XML data.
- * @param schema the schema as a DDL-formatted string.
- * @param options options to control how the XML is parsed. accepts the same options and the
- * xml data source.
- * See
- *
- * Data Source Option in the version you use.
+ * @param e
+ * a string column containing XML data.
+ * @param schema
+ * the schema as a DDL-formatted string.
+ * @param options
+ * options to control how the XML is parsed. accepts the same options and the xml data source.
+ * See Data
+ * Source Option in the version you use.
* @group xml_funcs
* @since 4.0.0
*/
@@ -7268,11 +7525,13 @@ object functions {
// scalastyle:off line.size.limit
/**
- * (Java-specific) Parses a column containing a XML string into a `StructType`
- * with the specified schema. Returns `null`, in the case of an unparseable string.
+ * (Java-specific) Parses a column containing a XML string into a `StructType` with the
+ * specified schema. Returns `null`, in the case of an unparseable string.
*
- * @param e a string column containing XML data.
- * @param schema the schema to use when parsing the XML string
+ * @param e
+ * a string column containing XML data.
+ * @param schema
+ * the schema to use when parsing the XML string
* @group xml_funcs
* @since 4.0.0
*/
@@ -7283,17 +7542,18 @@ object functions {
// scalastyle:off line.size.limit
/**
- * (Java-specific) Parses a column containing a XML string into a `StructType`
- * with the specified schema. Returns `null`, in the case of an unparseable string.
- *
- * @param e a string column containing XML data.
- * @param schema the schema to use when parsing the XML string
- * @param options options to control how the XML is parsed. accepts the same options and the
- * XML data source.
- * See
- *
- * Data Source Option in the version you use.
+ * (Java-specific) Parses a column containing a XML string into a `StructType` with the
+ * specified schema. Returns `null`, in the case of an unparseable string.
+ *
+ * @param e
+ * a string column containing XML data.
+ * @param schema
+ * the schema to use when parsing the XML string
+ * @param options
+ * options to control how the XML is parsed. accepts the same options and the XML data source.
+ * See Data
+ * Source Option in the version you use.
* @group xml_funcs
* @since 4.0.0
*/
@@ -7302,13 +7562,14 @@ object functions {
from_xml(e, schema, options.asScala.iterator)
/**
- * Parses a column containing a XML string into the data type
- * corresponding to the specified schema.
- * Returns `null`, in the case of an unparseable string.
+ * Parses a column containing a XML string into the data type corresponding to the specified
+ * schema. Returns `null`, in the case of an unparseable string.
+ *
+ * @param e
+ * a string column containing XML data.
+ * @param schema
+ * the schema to use when parsing the XML string
*
- * @param e a string column containing XML data.
- * @param schema the schema to use when parsing the XML string
-
* @group xml_funcs
* @since 4.0.0
*/
@@ -7322,7 +7583,8 @@ object functions {
/**
* Parses a XML string and infers its schema in DDL format.
*
- * @param xml a XML string.
+ * @param xml
+ * a XML string.
* @group xml_funcs
* @since 4.0.0
*/
@@ -7331,7 +7593,8 @@ object functions {
/**
* Parses a XML string and infers its schema in DDL format.
*
- * @param xml a foldable string column containing a XML string.
+ * @param xml
+ * a foldable string column containing a XML string.
* @group xml_funcs
* @since 4.0.0
*/
@@ -7342,14 +7605,15 @@ object functions {
/**
* Parses a XML string and infers its schema in DDL format using options.
*
- * @param xml a foldable string column containing XML data.
- * @param options options to control how the xml is parsed. accepts the same options and the
- * XML data source.
- * See
- *
- * Data Source Option in the version you use.
- * @return a column with string literal containing schema in DDL format.
+ * @param xml
+ * a foldable string column containing XML data.
+ * @param options
+ * options to control how the xml is parsed. accepts the same options and the XML data source.
+ * See Data
+ * Source Option in the version you use.
+ * @return
+ * a column with string literal containing schema in DDL format.
* @group xml_funcs
* @since 4.0.0
*/
@@ -7360,16 +7624,16 @@ object functions {
// scalastyle:off line.size.limit
/**
- * (Java-specific) Converts a column containing a `StructType` into a XML string with
- * the specified schema. Throws an exception, in the case of an unsupported type.
+ * (Java-specific) Converts a column containing a `StructType` into a XML string with the
+ * specified schema. Throws an exception, in the case of an unsupported type.
*
- * @param e a column containing a struct.
- * @param options options to control how the struct column is converted into a XML string.
- * It accepts the same options as the XML data source.
- * See
- *
- * Data Source Option in the version you use.
+ * @param e
+ * a column containing a struct.
+ * @param options
+ * options to control how the struct column is converted into a XML string. It accepts the
+ * same options as the XML data source. See Data
+ * Source Option in the version you use.
* @group xml_funcs
* @since 4.0.0
*/
@@ -7381,7 +7645,8 @@ object functions {
* Converts a column containing a `StructType` into a XML string with the specified schema.
* Throws an exception, in the case of an unsupported type.
*
- * @param e a column containing a struct.
+ * @param e
+ * a column containing a struct.
* @group xml_funcs
* @since 4.0.0
*/
@@ -7430,8 +7695,8 @@ object functions {
Column.fn("xpath_boolean", xml, path)
/**
- * Returns a double value, the value zero if no match is found,
- * or NaN if a match is found but the value is non-numeric.
+ * Returns a double value, the value zero if no match is found, or NaN if a match is found but
+ * the value is non-numeric.
*
* @group xml_funcs
* @since 3.5.0
@@ -7440,8 +7705,8 @@ object functions {
Column.fn("xpath_double", xml, path)
/**
- * Returns a double value, the value zero if no match is found,
- * or NaN if a match is found but the value is non-numeric.
+ * Returns a double value, the value zero if no match is found, or NaN if a match is found but
+ * the value is non-numeric.
*
* @group xml_funcs
* @since 3.5.0
@@ -7450,8 +7715,8 @@ object functions {
Column.fn("xpath_number", xml, path)
/**
- * Returns a float value, the value zero if no match is found,
- * or NaN if a match is found but the value is non-numeric.
+ * Returns a float value, the value zero if no match is found, or NaN if a match is found but
+ * the value is non-numeric.
*
* @group xml_funcs
* @since 3.5.0
@@ -7460,8 +7725,8 @@ object functions {
Column.fn("xpath_float", xml, path)
/**
- * Returns an integer value, or the value zero if no match is found,
- * or a match is found but the value is non-numeric.
+ * Returns an integer value, or the value zero if no match is found, or a match is found but the
+ * value is non-numeric.
*
* @group xml_funcs
* @since 3.5.0
@@ -7470,8 +7735,8 @@ object functions {
Column.fn("xpath_int", xml, path)
/**
- * Returns a long integer value, or the value zero if no match is found,
- * or a match is found but the value is non-numeric.
+ * Returns a long integer value, or the value zero if no match is found, or a match is found but
+ * the value is non-numeric.
*
* @group xml_funcs
* @since 3.5.0
@@ -7480,8 +7745,8 @@ object functions {
Column.fn("xpath_long", xml, path)
/**
- * Returns a short integer value, or the value zero if no match is found,
- * or a match is found but the value is non-numeric.
+ * Returns a short integer value, or the value zero if no match is found, or a match is found
+ * but the value is non-numeric.
*
* @group xml_funcs
* @since 3.5.0
@@ -7507,13 +7772,16 @@ object functions {
def hours(e: Column): Column = partitioning.hours(e)
/**
- * Converts the timestamp without time zone `sourceTs`
- * from the `sourceTz` time zone to `targetTz`.
+ * Converts the timestamp without time zone `sourceTs` from the `sourceTz` time zone to
+ * `targetTz`.
*
- * @param sourceTz the time zone for the input timestamp. If it is missed,
- * the current session time zone is used as the source time zone.
- * @param targetTz the time zone to which the input timestamp should be converted.
- * @param sourceTs a timestamp without time zone.
+ * @param sourceTz
+ * the time zone for the input timestamp. If it is missed, the current session time zone is
+ * used as the source time zone.
+ * @param targetTz
+ * the time zone to which the input timestamp should be converted.
+ * @param sourceTs
+ * a timestamp without time zone.
* @group datetime_funcs
* @since 3.5.0
*/
@@ -7521,11 +7789,12 @@ object functions {
Column.fn("convert_timezone", sourceTz, targetTz, sourceTs)
/**
- * Converts the timestamp without time zone `sourceTs`
- * from the current time zone to `targetTz`.
+ * Converts the timestamp without time zone `sourceTs` from the current time zone to `targetTz`.
*
- * @param targetTz the time zone to which the input timestamp should be converted.
- * @param sourceTs a timestamp without time zone.
+ * @param targetTz
+ * the time zone to which the input timestamp should be converted.
+ * @param sourceTs
+ * a timestamp without time zone.
* @group datetime_funcs
* @since 3.5.0
*/
@@ -7818,8 +8087,8 @@ object functions {
def isnotnull(col: Column): Column = Column.fn("isnotnull", col)
/**
- * Returns same result as the EQUAL(=) operator for non-null operands,
- * but returns true if both are null, false if one of the them is null.
+ * Returns same result as the EQUAL(=) operator for non-null operands, but returns true if both
+ * are null, false if one of the them is null.
*
* @group predicate_funcs
* @since 3.5.0
@@ -7908,15 +8177,15 @@ object functions {
|}""".stripMargin)
}
- */
+ */
//////////////////////////////////////////////////////////////////////////////////////////////
// Scala UDF functions
//////////////////////////////////////////////////////////////////////////////////////////////
/**
- * Obtains a `UserDefinedFunction` that wraps the given `Aggregator`
- * so that it may be used with untyped Data Frames.
+ * Obtains a `UserDefinedFunction` that wraps the given `Aggregator` so that it may be used with
+ * untyped Data Frames.
* {{{
* val agg = // Aggregator[IN, BUF, OUT]
*
@@ -7928,24 +8197,30 @@ object functions {
* spark.udf.register("myAggName", udaf(agg))
* }}}
*
- * @tparam IN the aggregator input type
- * @tparam BUF the aggregating buffer type
- * @tparam OUT the finalized output type
+ * @tparam IN
+ * the aggregator input type
+ * @tparam BUF
+ * the aggregating buffer type
+ * @tparam OUT
+ * the finalized output type
*
- * @param agg the typed Aggregator
+ * @param agg
+ * the typed Aggregator
*
- * @return a UserDefinedFunction that can be used as an aggregating expression.
+ * @return
+ * a UserDefinedFunction that can be used as an aggregating expression.
*
* @group udf_funcs
- * @note The input encoder is inferred from the input type IN.
+ * @note
+ * The input encoder is inferred from the input type IN.
*/
def udaf[IN: TypeTag, BUF, OUT](agg: Aggregator[IN, BUF, OUT]): UserDefinedFunction = {
udaf(agg, ScalaReflection.encoderFor[IN])
}
/**
- * Obtains a `UserDefinedFunction` that wraps the given `Aggregator`
- * so that it may be used with untyped Data Frames.
+ * Obtains a `UserDefinedFunction` that wraps the given `Aggregator` so that it may be used with
+ * untyped Data Frames.
* {{{
* Aggregator agg = // custom Aggregator
* Encoder enc = // input encoder
@@ -7958,18 +8233,24 @@ object functions {
* spark.udf.register("myAggName", udaf(agg, enc))
* }}}
*
- * @tparam IN the aggregator input type
- * @tparam BUF the aggregating buffer type
- * @tparam OUT the finalized output type
+ * @tparam IN
+ * the aggregator input type
+ * @tparam BUF
+ * the aggregating buffer type
+ * @tparam OUT
+ * the finalized output type
*
- * @param agg the typed Aggregator
- * @param inputEncoder a specific input encoder to use
+ * @param agg
+ * the typed Aggregator
+ * @param inputEncoder
+ * a specific input encoder to use
*
- * @return a UserDefinedFunction that can be used as an aggregating expression
+ * @return
+ * a UserDefinedFunction that can be used as an aggregating expression
*
* @group udf_funcs
- * @note This overloading takes an explicit input encoder, to support UDAF
- * declarations in Java.
+ * @note
+ * This overloading takes an explicit input encoder, to support UDAF declarations in Java.
*/
def udaf[IN, BUF, OUT](
agg: Aggregator[IN, BUF, OUT],
@@ -7978,10 +8259,10 @@ object functions {
}
/**
- * Defines a Scala closure of 0 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 0 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
@@ -7991,10 +8272,10 @@ object functions {
}
/**
- * Defines a Scala closure of 1 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 1 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
@@ -8004,120 +8285,242 @@ object functions {
}
/**
- * Defines a Scala closure of 2 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 2 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
*/
- def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = {
- SparkUserDefinedFunction(f, implicitly[TypeTag[RT]], implicitly[TypeTag[A1]], implicitly[TypeTag[A2]])
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](
+ f: Function2[A1, A2, RT]): UserDefinedFunction = {
+ SparkUserDefinedFunction(
+ f,
+ implicitly[TypeTag[RT]],
+ implicitly[TypeTag[A1]],
+ implicitly[TypeTag[A2]])
}
/**
- * Defines a Scala closure of 3 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 3 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
*/
- def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
- SparkUserDefinedFunction(f, implicitly[TypeTag[RT]], implicitly[TypeTag[A1]], implicitly[TypeTag[A2]], implicitly[TypeTag[A3]])
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](
+ f: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
+ SparkUserDefinedFunction(
+ f,
+ implicitly[TypeTag[RT]],
+ implicitly[TypeTag[A1]],
+ implicitly[TypeTag[A2]],
+ implicitly[TypeTag[A3]])
}
/**
- * Defines a Scala closure of 4 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 4 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
*/
- def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
- SparkUserDefinedFunction(f, implicitly[TypeTag[RT]], implicitly[TypeTag[A1]], implicitly[TypeTag[A2]], implicitly[TypeTag[A3]], implicitly[TypeTag[A4]])
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](
+ f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
+ SparkUserDefinedFunction(
+ f,
+ implicitly[TypeTag[RT]],
+ implicitly[TypeTag[A1]],
+ implicitly[TypeTag[A2]],
+ implicitly[TypeTag[A3]],
+ implicitly[TypeTag[A4]])
}
/**
- * Defines a Scala closure of 5 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 5 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
*/
- def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
- SparkUserDefinedFunction(f, implicitly[TypeTag[RT]], implicitly[TypeTag[A1]], implicitly[TypeTag[A2]], implicitly[TypeTag[A3]], implicitly[TypeTag[A4]], implicitly[TypeTag[A5]])
+ def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](
+ f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
+ SparkUserDefinedFunction(
+ f,
+ implicitly[TypeTag[RT]],
+ implicitly[TypeTag[A1]],
+ implicitly[TypeTag[A2]],
+ implicitly[TypeTag[A3]],
+ implicitly[TypeTag[A4]],
+ implicitly[TypeTag[A5]])
}
/**
- * Defines a Scala closure of 6 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 6 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
*/
- def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
- SparkUserDefinedFunction(f, implicitly[TypeTag[RT]], implicitly[TypeTag[A1]], implicitly[TypeTag[A2]], implicitly[TypeTag[A3]], implicitly[TypeTag[A4]], implicitly[TypeTag[A5]], implicitly[TypeTag[A6]])
+ def udf[
+ RT: TypeTag,
+ A1: TypeTag,
+ A2: TypeTag,
+ A3: TypeTag,
+ A4: TypeTag,
+ A5: TypeTag,
+ A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
+ SparkUserDefinedFunction(
+ f,
+ implicitly[TypeTag[RT]],
+ implicitly[TypeTag[A1]],
+ implicitly[TypeTag[A2]],
+ implicitly[TypeTag[A3]],
+ implicitly[TypeTag[A4]],
+ implicitly[TypeTag[A5]],
+ implicitly[TypeTag[A6]])
}
/**
- * Defines a Scala closure of 7 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 7 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
*/
- def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
- SparkUserDefinedFunction(f, implicitly[TypeTag[RT]], implicitly[TypeTag[A1]], implicitly[TypeTag[A2]], implicitly[TypeTag[A3]], implicitly[TypeTag[A4]], implicitly[TypeTag[A5]], implicitly[TypeTag[A6]], implicitly[TypeTag[A7]])
+ def udf[
+ RT: TypeTag,
+ A1: TypeTag,
+ A2: TypeTag,
+ A3: TypeTag,
+ A4: TypeTag,
+ A5: TypeTag,
+ A6: TypeTag,
+ A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
+ SparkUserDefinedFunction(
+ f,
+ implicitly[TypeTag[RT]],
+ implicitly[TypeTag[A1]],
+ implicitly[TypeTag[A2]],
+ implicitly[TypeTag[A3]],
+ implicitly[TypeTag[A4]],
+ implicitly[TypeTag[A5]],
+ implicitly[TypeTag[A6]],
+ implicitly[TypeTag[A7]])
}
/**
- * Defines a Scala closure of 8 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 8 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
*/
- def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
- SparkUserDefinedFunction(f, implicitly[TypeTag[RT]], implicitly[TypeTag[A1]], implicitly[TypeTag[A2]], implicitly[TypeTag[A3]], implicitly[TypeTag[A4]], implicitly[TypeTag[A5]], implicitly[TypeTag[A6]], implicitly[TypeTag[A7]], implicitly[TypeTag[A8]])
+ def udf[
+ RT: TypeTag,
+ A1: TypeTag,
+ A2: TypeTag,
+ A3: TypeTag,
+ A4: TypeTag,
+ A5: TypeTag,
+ A6: TypeTag,
+ A7: TypeTag,
+ A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
+ SparkUserDefinedFunction(
+ f,
+ implicitly[TypeTag[RT]],
+ implicitly[TypeTag[A1]],
+ implicitly[TypeTag[A2]],
+ implicitly[TypeTag[A3]],
+ implicitly[TypeTag[A4]],
+ implicitly[TypeTag[A5]],
+ implicitly[TypeTag[A6]],
+ implicitly[TypeTag[A7]],
+ implicitly[TypeTag[A8]])
}
/**
- * Defines a Scala closure of 9 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 9 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
*/
- def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
- SparkUserDefinedFunction(f, implicitly[TypeTag[RT]], implicitly[TypeTag[A1]], implicitly[TypeTag[A2]], implicitly[TypeTag[A3]], implicitly[TypeTag[A4]], implicitly[TypeTag[A5]], implicitly[TypeTag[A6]], implicitly[TypeTag[A7]], implicitly[TypeTag[A8]], implicitly[TypeTag[A9]])
+ def udf[
+ RT: TypeTag,
+ A1: TypeTag,
+ A2: TypeTag,
+ A3: TypeTag,
+ A4: TypeTag,
+ A5: TypeTag,
+ A6: TypeTag,
+ A7: TypeTag,
+ A8: TypeTag,
+ A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
+ SparkUserDefinedFunction(
+ f,
+ implicitly[TypeTag[RT]],
+ implicitly[TypeTag[A1]],
+ implicitly[TypeTag[A2]],
+ implicitly[TypeTag[A3]],
+ implicitly[TypeTag[A4]],
+ implicitly[TypeTag[A5]],
+ implicitly[TypeTag[A6]],
+ implicitly[TypeTag[A7]],
+ implicitly[TypeTag[A8]],
+ implicitly[TypeTag[A9]])
}
/**
- * Defines a Scala closure of 10 arguments as user-defined function (UDF).
- * The data types are automatically inferred based on the Scala closure's
- * signature. By default the returned UDF is deterministic. To change it to
- * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Scala closure of 10 arguments as user-defined function (UDF). The data types are
+ * automatically inferred based on the Scala closure's signature. By default the returned UDF is
+ * deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 1.3.0
*/
- def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
- SparkUserDefinedFunction(f, implicitly[TypeTag[RT]], implicitly[TypeTag[A1]], implicitly[TypeTag[A2]], implicitly[TypeTag[A3]], implicitly[TypeTag[A4]], implicitly[TypeTag[A5]], implicitly[TypeTag[A6]], implicitly[TypeTag[A7]], implicitly[TypeTag[A8]], implicitly[TypeTag[A9]], implicitly[TypeTag[A10]])
+ def udf[
+ RT: TypeTag,
+ A1: TypeTag,
+ A2: TypeTag,
+ A3: TypeTag,
+ A4: TypeTag,
+ A5: TypeTag,
+ A6: TypeTag,
+ A7: TypeTag,
+ A8: TypeTag,
+ A9: TypeTag,
+ A10: TypeTag](
+ f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
+ SparkUserDefinedFunction(
+ f,
+ implicitly[TypeTag[RT]],
+ implicitly[TypeTag[A1]],
+ implicitly[TypeTag[A2]],
+ implicitly[TypeTag[A3]],
+ implicitly[TypeTag[A4]],
+ implicitly[TypeTag[A5]],
+ implicitly[TypeTag[A6]],
+ implicitly[TypeTag[A7]],
+ implicitly[TypeTag[A8]],
+ implicitly[TypeTag[A9]],
+ implicitly[TypeTag[A10]])
}
//////////////////////////////////////////////////////////////////////////////////////////////
@@ -8125,10 +8528,10 @@ object functions {
//////////////////////////////////////////////////////////////////////////////////////////////
/**
- * Defines a Java UDF0 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF0 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8138,10 +8541,10 @@ object functions {
}
/**
- * Defines a Java UDF1 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF1 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8151,10 +8554,10 @@ object functions {
}
/**
- * Defines a Java UDF2 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF2 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8164,10 +8567,10 @@ object functions {
}
/**
- * Defines a Java UDF3 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF3 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8177,10 +8580,10 @@ object functions {
}
/**
- * Defines a Java UDF4 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF4 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8190,10 +8593,10 @@ object functions {
}
/**
- * Defines a Java UDF5 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF5 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8203,10 +8606,10 @@ object functions {
}
/**
- * Defines a Java UDF6 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF6 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8216,10 +8619,10 @@ object functions {
}
/**
- * Defines a Java UDF7 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF7 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8229,10 +8632,10 @@ object functions {
}
/**
- * Defines a Java UDF8 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF8 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8242,10 +8645,10 @@ object functions {
}
/**
- * Defines a Java UDF9 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF9 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
@@ -8255,15 +8658,17 @@ object functions {
}
/**
- * Defines a Java UDF10 instance as user-defined function (UDF).
- * The caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * Defines a Java UDF10 instance as user-defined function (UDF). The caller must specify the
+ * output data type, and there is no automatic input type coercion. By default the returned UDF
+ * is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* @group udf_funcs
* @since 2.3.0
*/
- def udf(f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
+ def udf(
+ f: UDF10[_, _, _, _, _, _, _, _, _, _, _],
+ returnType: DataType): UserDefinedFunction = {
SparkUserDefinedFunction(ToScalaUDF(f), returnType, 10)
}
@@ -8273,8 +8678,8 @@ object functions {
/**
* Defines a deterministic user-defined function (UDF) using a Scala closure. For this variant,
* the caller must specify the output data type, and there is no automatic input type coercion.
- * By default the returned UDF is deterministic. To change it to nondeterministic, call the
- * API `UserDefinedFunction.asNondeterministic()`.
+ * By default the returned UDF is deterministic. To change it to nondeterministic, call the API
+ * `UserDefinedFunction.asNondeterministic()`.
*
* Note that, although the Scala closure can have primitive-type function argument, it doesn't
* work well with null values. Because the Scala closure is passed in as Any type, there is no
@@ -8283,14 +8688,18 @@ object functions {
* default value of the Java type for the null argument, e.g. `udf((x: Int) => x, IntegerType)`,
* the result is 0 for null input.
*
- * @param f A closure in Scala
- * @param dataType The output data type of the UDF
+ * @param f
+ * A closure in Scala
+ * @param dataType
+ * The output data type of the UDF
*
* @group udf_funcs
* @since 2.0.0
*/
- @deprecated("Scala `udf` method with return type parameter is deprecated. " +
- "Please use Scala `udf` method without return type parameter.", "3.0.0")
+ @deprecated(
+ "Scala `udf` method with return type parameter is deprecated. " +
+ "Please use Scala `udf` method without return type parameter.",
+ "3.0.0")
def udf(f: AnyRef, dataType: DataType): UserDefinedFunction = {
if (!SqlApiConf.get.legacyAllowUntypedScalaUDFs) {
throw CompilationErrors.usingUntypedScalaUDFError()
@@ -8309,8 +8718,7 @@ object functions {
def callUDF(udfName: String, cols: Column*): Column = call_function(udfName, cols: _*)
/**
- * Call an user-defined function.
- * Example:
+ * Call an user-defined function. Example:
* {{{
* import org.apache.spark.sql._
*
@@ -8329,9 +8737,10 @@ object functions {
/**
* Call a SQL function.
*
- * @param funcName function name that follows the SQL identifier syntax
- * (can be quoted, can be qualified)
- * @param cols the expression parameters of function
+ * @param funcName
+ * function name that follows the SQL identifier syntax (can be quoted, can be qualified)
+ * @param cols
+ * the expression parameters of function
* @group normal_funcs
* @since 3.5.0
*/
@@ -8352,7 +8761,7 @@ object functions {
// API in the same way. Once we land this fix, should deprecate
// functions.hours, days, months, years and bucket.
object partitioning {
- // scalastyle:on
+ // scalastyle:on
/**
* (Scala-specific) A transform for timestamps and dates to partition data into years.
*
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConf.scala b/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConf.scala
index 5762f9f6f5668..555a567053080 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConf.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConf.scala
@@ -25,8 +25,8 @@ import org.apache.spark.util.SparkClassUtils
/**
* Configuration for all objects that are placed in the `sql/api` project. The normal way of
- * accessing this class is through `SqlApiConf.get`. If this code is being used with sql/core
- * then its values are bound to the currently set SQLConf. With Spark Connect, it will default to
+ * accessing this class is through `SqlApiConf.get`. If this code is being used with sql/core then
+ * its values are bound to the currently set SQLConf. With Spark Connect, it will default to
* hardcoded values.
*/
private[sql] trait SqlApiConf {
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConfHelper.scala b/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConfHelper.scala
index b7b8e14afb387..13ef13e5894e0 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConfHelper.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/internal/SqlApiConfHelper.scala
@@ -21,9 +21,9 @@ import java.util.concurrent.atomic.AtomicReference
/**
* SqlApiConfHelper is created to avoid a deadlock during a concurrent access to SQLConf and
* SqlApiConf, which is because SQLConf and SqlApiConf tries to load each other upon
- * initializations. SqlApiConfHelper is private to sql package and is not supposed to be
- * accessed by end users. Variables and methods within SqlApiConfHelper are defined to
- * be used by SQLConf and SqlApiConf only.
+ * initializations. SqlApiConfHelper is private to sql package and is not supposed to be accessed
+ * by end users. Variables and methods within SqlApiConfHelper are defined to be used by SQLConf
+ * and SqlApiConf only.
*/
private[sql] object SqlApiConfHelper {
// Shared keys.
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/internal/ToScalaUDF.scala b/sql/api/src/main/scala/org/apache/spark/sql/internal/ToScalaUDF.scala
index 25ea37fadfa91..4d476108d9ec5 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/internal/ToScalaUDF.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/internal/ToScalaUDF.scala
@@ -16,14 +16,56 @@
*/
package org.apache.spark.sql.internal
+import scala.jdk.CollectionConverters._
+
+import org.apache.spark.api.java.function.{CoGroupFunction, FilterFunction, FlatMapFunction, FlatMapGroupsFunction, FlatMapGroupsWithStateFunction, ForeachFunction, ForeachPartitionFunction, MapFunction, MapGroupsFunction, MapGroupsWithStateFunction, MapPartitionsFunction, ReduceFunction}
import org.apache.spark.sql.api.java._
+import org.apache.spark.sql.streaming.GroupState
/**
- * Helper class that provides conversions from org.apache.spark.sql.api.java.Function* to
- * scala.Function*.
+ * Helper class that provides conversions from org.apache.spark.sql.api.java.Function* and
+ * org.apache.spark.api.java.function.* to scala functions.
+ *
+ * Please note that this class is being used in Spark Connect Scala UDFs. We need to be careful
+ * with any modifications to this class, otherwise we will break backwards compatibility.
+ * Concretely this means you can only add methods to this class. You cannot rename the class, move
+ * it, change its `serialVersionUID`, remove methods, change method signatures, or change method
+ * semantics.
*/
@SerialVersionUID(2019907615267866045L)
private[sql] object ToScalaUDF extends Serializable {
+ def apply[T](f: FilterFunction[T]): T => Boolean = f.call
+
+ def apply[T](f: ReduceFunction[T]): (T, T) => T = f.call
+
+ def apply[V, W](f: MapFunction[V, W]): V => W = f.call
+
+ def apply[K, V, U](f: MapGroupsFunction[K, V, U]): (K, Iterator[V]) => U =
+ (key, values) => f.call(key, values.asJava)
+
+ def apply[K, V, S, U](
+ f: MapGroupsWithStateFunction[K, V, S, U]): (K, Iterator[V], GroupState[S]) => U =
+ (key, values, state) => f.call(key, values.asJava, state)
+
+ def apply[V, U](f: MapPartitionsFunction[V, U]): Iterator[V] => Iterator[U] =
+ values => f.call(values.asJava).asScala
+
+ def apply[K, V, U](f: FlatMapGroupsFunction[K, V, U]): (K, Iterator[V]) => Iterator[U] =
+ (key, values) => f.call(key, values.asJava).asScala
+
+ def apply[K, V, S, U](f: FlatMapGroupsWithStateFunction[K, V, S, U])
+ : (K, Iterator[V], GroupState[S]) => Iterator[U] =
+ (key, values, state) => f.call(key, values.asJava, state).asScala
+
+ def apply[K, V, U, R](
+ f: CoGroupFunction[K, V, U, R]): (K, Iterator[V], Iterator[U]) => Iterator[R] =
+ (key, left, right) => f.call(key, left.asJava, right.asJava).asScala
+
+ def apply[V](f: ForeachFunction[V]): V => Unit = f.call
+
+ def apply[V](f: ForeachPartitionFunction[V]): Iterator[V] => Unit =
+ values => f.call(values.asJava)
+
// scalastyle:off line.size.limit
/* register 0-22 were generated by this script
@@ -38,171 +80,757 @@ private[sql] object ToScalaUDF extends Serializable {
|/**
| * Create a scala.Function$i wrapper for a org.apache.spark.sql.api.java.UDF$i instance.
| */
- |def apply(f: UDF$i[$extTypeArgs]): AnyRef = {
+ |def apply(f: UDF$i[$extTypeArgs]): Function$i[$anyTypeArgs] = {
| $funcCall
|}""".stripMargin)
}
- */
+ */
/**
* Create a scala.Function0 wrapper for a org.apache.spark.sql.api.java.UDF0 instance.
*/
- def apply(f: UDF0[_]): AnyRef = {
- () => f.asInstanceOf[UDF0[Any]].call()
+ def apply(f: UDF0[_]): () => Any = { () =>
+ f.asInstanceOf[UDF0[Any]].call()
}
/**
* Create a scala.Function1 wrapper for a org.apache.spark.sql.api.java.UDF1 instance.
*/
- def apply(f: UDF1[_, _]): AnyRef = {
+ def apply(f: UDF1[_, _]): (Any) => Any = {
f.asInstanceOf[UDF1[Any, Any]].call(_: Any)
}
/**
* Create a scala.Function2 wrapper for a org.apache.spark.sql.api.java.UDF2 instance.
*/
- def apply(f: UDF2[_, _, _]): AnyRef = {
+ def apply(f: UDF2[_, _, _]): (Any, Any) => Any = {
f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any)
}
/**
* Create a scala.Function3 wrapper for a org.apache.spark.sql.api.java.UDF3 instance.
*/
- def apply(f: UDF3[_, _, _, _]): AnyRef = {
+ def apply(f: UDF3[_, _, _, _]): (Any, Any, Any) => Any = {
f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any)
}
/**
* Create a scala.Function4 wrapper for a org.apache.spark.sql.api.java.UDF4 instance.
*/
- def apply(f: UDF4[_, _, _, _, _]): AnyRef = {
+ def apply(f: UDF4[_, _, _, _, _]): (Any, Any, Any, Any) => Any = {
f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any)
}
/**
* Create a scala.Function5 wrapper for a org.apache.spark.sql.api.java.UDF5 instance.
*/
- def apply(f: UDF5[_, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF5[_, _, _, _, _, _]): (Any, Any, Any, Any, Any) => Any = {
+ f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]]
+ .call(_: Any, _: Any, _: Any, _: Any, _: Any)
}
/**
* Create a scala.Function6 wrapper for a org.apache.spark.sql.api.java.UDF6 instance.
*/
- def apply(f: UDF6[_, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF6[_, _, _, _, _, _, _]): (Any, Any, Any, Any, Any, Any) => Any = {
+ f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]]
+ .call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
}
/**
* Create a scala.Function7 wrapper for a org.apache.spark.sql.api.java.UDF7 instance.
*/
- def apply(f: UDF7[_, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF7[_, _, _, _, _, _, _, _]): (Any, Any, Any, Any, Any, Any, Any) => Any = {
+ f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
}
/**
* Create a scala.Function8 wrapper for a org.apache.spark.sql.api.java.UDF8 instance.
*/
- def apply(f: UDF8[_, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(
+ f: UDF8[_, _, _, _, _, _, _, _, _]): (Any, Any, Any, Any, Any, Any, Any, Any) => Any = {
+ f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
}
/**
* Create a scala.Function9 wrapper for a org.apache.spark.sql.api.java.UDF9 instance.
*/
- def apply(f: UDF9[_, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF9[_, _, _, _, _, _, _, _, _, _])
+ : (Any, Any, Any, Any, Any, Any, Any, Any, Any) => Any = {
+ f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
}
/**
* Create a scala.Function10 wrapper for a org.apache.spark.sql.api.java.UDF10 instance.
*/
- def apply(f: UDF10[_, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF10[_, _, _, _, _, _, _, _, _, _, _])
+ : Function10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any] = {
+ f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
}
/**
* Create a scala.Function11 wrapper for a org.apache.spark.sql.api.java.UDF11 instance.
*/
- def apply(f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _])
+ : Function11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any] = {
+ f.asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function12 wrapper for a org.apache.spark.sql.api.java.UDF12 instance.
*/
- def apply(f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _])
+ : Function12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any] = {
+ f.asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function13 wrapper for a org.apache.spark.sql.api.java.UDF13 instance.
*/
- def apply(f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _])
+ : Function13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any] = {
+ f.asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function14 wrapper for a org.apache.spark.sql.api.java.UDF14 instance.
*/
- def apply(f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _])
+ : Function14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any] = {
+ f.asInstanceOf[
+ UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function15 wrapper for a org.apache.spark.sql.api.java.UDF15 instance.
*/
- def apply(f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): Function15[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any] = {
+ f.asInstanceOf[
+ UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function16 wrapper for a org.apache.spark.sql.api.java.UDF16 instance.
*/
- def apply(f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): Function16[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any] = {
+ f.asInstanceOf[
+ UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function17 wrapper for a org.apache.spark.sql.api.java.UDF17 instance.
*/
- def apply(f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): Function17[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any] = {
+ f.asInstanceOf[UDF17[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function18 wrapper for a org.apache.spark.sql.api.java.UDF18 instance.
*/
- def apply(f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): Function18[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any] = {
+ f.asInstanceOf[UDF18[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function19 wrapper for a org.apache.spark.sql.api.java.UDF19 instance.
*/
- def apply(f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): Function19[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any] = {
+ f.asInstanceOf[UDF19[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function20 wrapper for a org.apache.spark.sql.api.java.UDF20 instance.
*/
- def apply(f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): Function20[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any] = {
+ f.asInstanceOf[UDF20[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function21 wrapper for a org.apache.spark.sql.api.java.UDF21 instance.
*/
- def apply(f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(
+ f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): Function21[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any] = {
+ f.asInstanceOf[UDF21[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
/**
* Create a scala.Function22 wrapper for a org.apache.spark.sql.api.java.UDF22 instance.
*/
- def apply(f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): AnyRef = {
- f.asInstanceOf[UDF22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
+ def apply(
+ f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _]): Function22[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any] = {
+ f.asInstanceOf[UDF22[
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any,
+ Any]]
+ .call(
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any,
+ _: Any)
}
// scalastyle:on line.size.limit
}
+
+/**
+ * Adaptors from one UDF shape to another. For example adapting a foreach function for use in
+ * foreachPartition.
+ *
+ * Please note that this class is being used in Spark Connect Scala UDFs. We need to be careful
+ * with any modifications to this class, otherwise we will break backwards compatibility.
+ * Concretely this means you can only add methods to this class. You cannot rename the class, move
+ * it, change its `serialVersionUID`, remove methods, change method signatures, or change method
+ * semantics.
+ */
+@SerialVersionUID(0L) // TODO
+object UDFAdaptors extends Serializable {
+ def flatMapToMapPartitions[V, U](f: V => IterableOnce[U]): Iterator[V] => Iterator[U] =
+ values => values.flatMap(f)
+
+ def flatMapToMapPartitions[V, U](f: FlatMapFunction[V, U]): Iterator[V] => Iterator[U] =
+ values => values.flatMap(v => f.call(v).asScala)
+
+ def mapToMapPartitions[V, U](f: V => U): Iterator[V] => Iterator[U] = values => values.map(f)
+
+ def mapToMapPartitions[V, U](f: MapFunction[V, U]): Iterator[V] => Iterator[U] =
+ values => values.map(f.call)
+
+ def foreachToForeachPartition[T](f: T => Unit): Iterator[T] => Unit =
+ values => values.foreach(f)
+
+ def foreachToForeachPartition[T](f: ForeachFunction[T]): Iterator[T] => Unit =
+ values => values.foreach(f.call)
+
+ def foreachPartitionToMapPartitions[V, U](f: Iterator[V] => Unit): Iterator[V] => Iterator[U] =
+ values => {
+ f(values)
+ Iterator.empty[U]
+ }
+
+ def iterableOnceToSeq[A, B](f: A => IterableOnce[B]): A => Seq[B] =
+ value => f(value).iterator.toSeq
+
+ def mapGroupsToFlatMapGroups[K, V, U](
+ f: (K, Iterator[V]) => U): (K, Iterator[V]) => Iterator[U] =
+ (key, values) => Iterator.single(f(key, values))
+
+ def mapGroupsWithStateToFlatMapWithState[K, V, S, U](
+ f: (K, Iterator[V], GroupState[S]) => U): (K, Iterator[V], GroupState[S]) => Iterator[U] =
+ (key: K, values: Iterator[V], state: GroupState[S]) => Iterator(f(key, values, state))
+
+ def coGroupWithMappedValues[K, V, U, R, IV, IU](
+ f: (K, Iterator[V], Iterator[U]) => IterableOnce[R],
+ leftValueMapFunc: Option[IV => V],
+ rightValueMapFunc: Option[IU => U]): (K, Iterator[IV], Iterator[IU]) => IterableOnce[R] = {
+ (leftValueMapFunc, rightValueMapFunc) match {
+ case (None, None) =>
+ f.asInstanceOf[(K, Iterator[IV], Iterator[IU]) => IterableOnce[R]]
+ case (Some(mapLeft), None) =>
+ (k, left, right) => f(k, left.map(mapLeft), right.asInstanceOf[Iterator[U]])
+ case (None, Some(mapRight)) =>
+ (k, left, right) => f(k, left.asInstanceOf[Iterator[V]], right.map(mapRight))
+ case (Some(mapLeft), Some(mapRight)) =>
+ (k, left, right) => f(k, left.map(mapLeft), right.map(mapRight))
+ }
+ }
+
+ def flatMapGroupsWithMappedValues[K, IV, V, R](
+ f: (K, Iterator[V]) => IterableOnce[R],
+ valueMapFunc: Option[IV => V]): (K, Iterator[IV]) => IterableOnce[R] = valueMapFunc match {
+ case Some(mapValue) => (k, values) => f(k, values.map(mapValue))
+ case None => f.asInstanceOf[(K, Iterator[IV]) => IterableOnce[R]]
+ }
+
+ def flatMapGroupsWithStateWithMappedValues[K, IV, V, S, U](
+ f: (K, Iterator[V], GroupState[S]) => Iterator[U],
+ valueMapFunc: Option[IV => V]): (K, Iterator[IV], GroupState[S]) => Iterator[U] = {
+ valueMapFunc match {
+ case Some(mapValue) => (k, values, state) => f(k, values.map(mapValue), state)
+ case None => f.asInstanceOf[(K, Iterator[IV], GroupState[S]) => Iterator[U]]
+ }
+ }
+}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/internal/columnNodes.scala b/sql/api/src/main/scala/org/apache/spark/sql/internal/columnNodes.scala
index 9d77b2e6f3e22..51b26a1fa2435 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/internal/columnNodes.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/internal/columnNodes.scala
@@ -32,10 +32,12 @@ import org.apache.spark.util.SparkClassUtils
* implementation specific form (e.g. Catalyst expressions, or Connect protobuf messages).
*
* This API is a mirror image of Connect's expression.proto. There are a couple of extensions to
- * make constructing nodes easier (e.g. [[CaseWhenOtherwise]]). We could not use the actual connect
- * protobuf messages because of classpath clashes (e.g. Guava & gRPC) and Maven shading issues.
+ * make constructing nodes easier (e.g. [[CaseWhenOtherwise]]). We could not use the actual
+ * connect protobuf messages because of classpath clashes (e.g. Guava & gRPC) and Maven shading
+ * issues.
*/
private[sql] trait ColumnNode extends ColumnNodeLike {
+
/**
* Origin where the node was created.
*/
@@ -93,13 +95,17 @@ private[internal] object ColumnNode {
/**
* A literal column.
*
- * @param value of the literal. This is the unconverted input value.
- * @param dataType of the literal. If none is provided the dataType is inferred.
+ * @param value
+ * of the literal. This is the unconverted input value.
+ * @param dataType
+ * of the literal. If none is provided the dataType is inferred.
*/
private[sql] case class Literal(
value: Any,
dataType: Option[DataType] = None,
- override val origin: Origin = CurrentOrigin.get) extends ColumnNode with DataTypeErrorsBase {
+ override val origin: Origin = CurrentOrigin.get)
+ extends ColumnNode
+ with DataTypeErrorsBase {
override private[internal] def normalize(): Literal = copy(origin = NO_ORIGIN)
override def sql: String = value match {
@@ -116,16 +122,19 @@ private[sql] case class Literal(
/**
* Reference to an attribute produced by one of the underlying DataFrames.
*
- * @param unparsedIdentifier name of the attribute.
- * @param planId id of the plan (Dataframe) that produces the attribute.
- * @param isMetadataColumn whether this is a metadata column.
+ * @param unparsedIdentifier
+ * name of the attribute.
+ * @param planId
+ * id of the plan (Dataframe) that produces the attribute.
+ * @param isMetadataColumn
+ * whether this is a metadata column.
*/
private[sql] case class UnresolvedAttribute(
unparsedIdentifier: String,
planId: Option[Long] = None,
isMetadataColumn: Boolean = false,
override val origin: Origin = CurrentOrigin.get)
- extends ColumnNode {
+ extends ColumnNode {
override private[internal] def normalize(): UnresolvedAttribute =
copy(planId = None, origin = NO_ORIGIN)
override def sql: String = unparsedIdentifier
@@ -134,14 +143,16 @@ private[sql] case class UnresolvedAttribute(
/**
* Reference to all columns in a namespace (global, a Dataframe, or a nested struct).
*
- * @param unparsedTarget name of the namespace. None if the global namespace is supposed to be used.
- * @param planId id of the plan (Dataframe) that produces the attribute.
+ * @param unparsedTarget
+ * name of the namespace. None if the global namespace is supposed to be used.
+ * @param planId
+ * id of the plan (Dataframe) that produces the attribute.
*/
private[sql] case class UnresolvedStar(
unparsedTarget: Option[String],
planId: Option[Long] = None,
override val origin: Origin = CurrentOrigin.get)
- extends ColumnNode {
+ extends ColumnNode {
override private[internal] def normalize(): UnresolvedStar =
copy(planId = None, origin = NO_ORIGIN)
override def sql: String = unparsedTarget.map(_ + ".*").getOrElse("*")
@@ -151,10 +162,12 @@ private[sql] case class UnresolvedStar(
* Call a function. This can either be a built-in function, a UDF, or a UDF registered in the
* Catalog.
*
- * @param functionName of the function to invoke.
- * @param arguments to pass into the function.
- * @param isDistinct (aggregate only) whether the input of the aggregate function should be
- * de-duplicated.
+ * @param functionName
+ * of the function to invoke.
+ * @param arguments
+ * to pass into the function.
+ * @param isDistinct
+ * (aggregate only) whether the input of the aggregate function should be de-duplicated.
*/
private[sql] case class UnresolvedFunction(
functionName: String,
@@ -163,7 +176,7 @@ private[sql] case class UnresolvedFunction(
isUserDefinedFunction: Boolean = false,
isInternal: Boolean = false,
override val origin: Origin = CurrentOrigin.get)
- extends ColumnNode {
+ extends ColumnNode {
override private[internal] def normalize(): UnresolvedFunction =
copy(arguments = ColumnNode.normalize(arguments), origin = NO_ORIGIN)
@@ -173,11 +186,13 @@ private[sql] case class UnresolvedFunction(
/**
* Evaluate a SQL expression.
*
- * @param expression text to execute.
+ * @param expression
+ * text to execute.
*/
private[sql] case class SqlExpression(
expression: String,
- override val origin: Origin = CurrentOrigin.get) extends ColumnNode {
+ override val origin: Origin = CurrentOrigin.get)
+ extends ColumnNode {
override private[internal] def normalize(): SqlExpression = copy(origin = NO_ORIGIN)
override def sql: String = expression
}
@@ -185,15 +200,19 @@ private[sql] case class SqlExpression(
/**
* Name a column, and (optionally) modify its metadata.
*
- * @param child to name
- * @param name to use
- * @param metadata (optional) metadata to add.
+ * @param child
+ * to name
+ * @param name
+ * to use
+ * @param metadata
+ * (optional) metadata to add.
*/
private[sql] case class Alias(
child: ColumnNode,
name: Seq[String],
metadata: Option[Metadata] = None,
- override val origin: Origin = CurrentOrigin.get) extends ColumnNode {
+ override val origin: Origin = CurrentOrigin.get)
+ extends ColumnNode {
override private[internal] def normalize(): Alias =
copy(child = child.normalize(), origin = NO_ORIGIN)
@@ -210,15 +229,19 @@ private[sql] case class Alias(
* Cast the value of a Column to a different [[DataType]]. The behavior of the cast can be
* influenced by the `evalMode`.
*
- * @param child that produces the input value.
- * @param dataType to cast to.
- * @param evalMode (try/ansi/legacy) to use for the cast.
+ * @param child
+ * that produces the input value.
+ * @param dataType
+ * to cast to.
+ * @param evalMode
+ * (try/ansi/legacy) to use for the cast.
*/
private[sql] case class Cast(
child: ColumnNode,
dataType: DataType,
evalMode: Option[Cast.EvalMode] = None,
- override val origin: Origin = CurrentOrigin.get) extends ColumnNode {
+ override val origin: Origin = CurrentOrigin.get)
+ extends ColumnNode {
override private[internal] def normalize(): Cast =
copy(child = child.normalize(), origin = NO_ORIGIN)
@@ -237,13 +260,16 @@ private[sql] object Cast {
/**
* Reference to all columns in the global namespace in that match a regex.
*
- * @param regex name of the namespace. None if the global namespace is supposed to be used.
- * @param planId id of the plan (Dataframe) that produces the attribute.
+ * @param regex
+ * name of the namespace. None if the global namespace is supposed to be used.
+ * @param planId
+ * id of the plan (Dataframe) that produces the attribute.
*/
private[sql] case class UnresolvedRegex(
regex: String,
planId: Option[Long] = None,
- override val origin: Origin = CurrentOrigin.get) extends ColumnNode {
+ override val origin: Origin = CurrentOrigin.get)
+ extends ColumnNode {
override private[internal] def normalize(): UnresolvedRegex =
copy(planId = None, origin = NO_ORIGIN)
override def sql: String = regex
@@ -252,16 +278,19 @@ private[sql] case class UnresolvedRegex(
/**
* Sort the input column.
*
- * @param child to sort.
- * @param sortDirection to sort in, either Ascending or Descending.
- * @param nullOrdering where to place nulls, either at the begin or the end.
+ * @param child
+ * to sort.
+ * @param sortDirection
+ * to sort in, either Ascending or Descending.
+ * @param nullOrdering
+ * where to place nulls, either at the begin or the end.
*/
private[sql] case class SortOrder(
child: ColumnNode,
sortDirection: SortOrder.SortDirection,
nullOrdering: SortOrder.NullOrdering,
override val origin: Origin = CurrentOrigin.get)
- extends ColumnNode {
+ extends ColumnNode {
override private[internal] def normalize(): SortOrder =
copy(child = child.normalize(), origin = NO_ORIGIN)
@@ -280,14 +309,16 @@ private[sql] object SortOrder {
/**
* Evaluate a function within a window.
*
- * @param windowFunction function to execute.
- * @param windowSpec of the window.
+ * @param windowFunction
+ * function to execute.
+ * @param windowSpec
+ * of the window.
*/
private[sql] case class Window(
windowFunction: ColumnNode,
windowSpec: WindowSpec,
override val origin: Origin = CurrentOrigin.get)
- extends ColumnNode {
+ extends ColumnNode {
override private[internal] def normalize(): Window = copy(
windowFunction = windowFunction.normalize(),
windowSpec = windowSpec.normalize(),
@@ -299,7 +330,8 @@ private[sql] case class Window(
private[sql] case class WindowSpec(
partitionColumns: Seq[ColumnNode],
sortColumns: Seq[SortOrder],
- frame: Option[WindowFrame] = None) extends ColumnNodeLike {
+ frame: Option[WindowFrame] = None)
+ extends ColumnNodeLike {
override private[internal] def normalize(): WindowSpec = copy(
partitionColumns = ColumnNode.normalize(partitionColumns),
sortColumns = ColumnNode.normalize(sortColumns),
@@ -317,7 +349,7 @@ private[sql] case class WindowFrame(
frameType: WindowFrame.FrameType,
lower: WindowFrame.FrameBoundary,
upper: WindowFrame.FrameBoundary)
- extends ColumnNodeLike {
+ extends ColumnNodeLike {
override private[internal] def normalize(): WindowFrame =
copy(lower = lower.normalize(), upper = upper.normalize())
override private[internal] def sql: String =
@@ -352,13 +384,16 @@ private[sql] object WindowFrame {
/**
* Lambda function to execute. This typically passed as an argument to a function.
*
- * @param function to execute.
- * @param arguments the bound lambda variables.
+ * @param function
+ * to execute.
+ * @param arguments
+ * the bound lambda variables.
*/
private[sql] case class LambdaFunction(
function: ColumnNode,
arguments: Seq[UnresolvedNamedLambdaVariable],
- override val origin: Origin) extends ColumnNode {
+ override val origin: Origin)
+ extends ColumnNode {
override private[internal] def normalize(): LambdaFunction = copy(
function = function.normalize(),
@@ -382,11 +417,13 @@ object LambdaFunction {
/**
* Variable used in a [[LambdaFunction]].
*
- * @param name of the variable.
+ * @param name
+ * of the variable.
*/
private[sql] case class UnresolvedNamedLambdaVariable(
name: String,
- override val origin: Origin = CurrentOrigin.get) extends ColumnNode {
+ override val origin: Origin = CurrentOrigin.get)
+ extends ColumnNode {
override private[internal] def normalize(): UnresolvedNamedLambdaVariable =
copy(origin = NO_ORIGIN)
@@ -413,21 +450,22 @@ object UnresolvedNamedLambdaVariable {
}
/**
- * Extract a value from a complex type. This can be a field from a struct, a value from a map,
- * or an element from an array.
+ * Extract a value from a complex type. This can be a field from a struct, a value from a map, or
+ * an element from an array.
*
- * @param child that produces a complex value.
- * @param extraction that is used to access the complex type. This needs to be a string type for
- * structs and maps, and it needs to be an integer for arrays.
+ * @param child
+ * that produces a complex value.
+ * @param extraction
+ * that is used to access the complex type. This needs to be a string type for structs and maps,
+ * and it needs to be an integer for arrays.
*/
private[sql] case class UnresolvedExtractValue(
child: ColumnNode,
extraction: ColumnNode,
- override val origin: Origin = CurrentOrigin.get) extends ColumnNode {
- override private[internal] def normalize(): UnresolvedExtractValue = copy(
- child = child.normalize(),
- extraction = extraction.normalize(),
- origin = NO_ORIGIN)
+ override val origin: Origin = CurrentOrigin.get)
+ extends ColumnNode {
+ override private[internal] def normalize(): UnresolvedExtractValue =
+ copy(child = child.normalize(), extraction = extraction.normalize(), origin = NO_ORIGIN)
override def sql: String = s"${child.sql}[${extraction.sql}]"
}
@@ -435,15 +473,19 @@ private[sql] case class UnresolvedExtractValue(
/**
* Update or drop the field of a struct.
*
- * @param structExpression that will be updated.
- * @param fieldName name of the field to update.
- * @param valueExpression new value of the field. If this is None the field will be dropped.
+ * @param structExpression
+ * that will be updated.
+ * @param fieldName
+ * name of the field to update.
+ * @param valueExpression
+ * new value of the field. If this is None the field will be dropped.
*/
private[sql] case class UpdateFields(
structExpression: ColumnNode,
fieldName: String,
valueExpression: Option[ColumnNode] = None,
- override val origin: Origin = CurrentOrigin.get) extends ColumnNode {
+ override val origin: Origin = CurrentOrigin.get)
+ extends ColumnNode {
override private[internal] def normalize(): UpdateFields = copy(
structExpression = structExpression.normalize(),
valueExpression = ColumnNode.normalize(valueExpression),
@@ -455,18 +497,20 @@ private[sql] case class UpdateFields(
}
/**
- * Evaluate one or more conditional branches. The value of the first branch for which the predicate
- * evalutes to true is returned. If none of the branches evaluate to true, the value of `otherwise`
- * is returned.
+ * Evaluate one or more conditional branches. The value of the first branch for which the
+ * predicate evalutes to true is returned. If none of the branches evaluate to true, the value of
+ * `otherwise` is returned.
*
- * @param branches to evaluate. Each entry if a pair of condition and value.
- * @param otherwise (optional) to evaluate when none of the branches evaluate to true.
+ * @param branches
+ * to evaluate. Each entry if a pair of condition and value.
+ * @param otherwise
+ * (optional) to evaluate when none of the branches evaluate to true.
*/
private[sql] case class CaseWhenOtherwise(
branches: Seq[(ColumnNode, ColumnNode)],
otherwise: Option[ColumnNode] = None,
override val origin: Origin = CurrentOrigin.get)
- extends ColumnNode {
+ extends ColumnNode {
assert(branches.nonEmpty)
override private[internal] def normalize(): CaseWhenOtherwise = copy(
branches = branches.map(kv => (kv._1.normalize(), kv._2.normalize())),
@@ -483,15 +527,17 @@ private[sql] case class CaseWhenOtherwise(
/**
* Invoke an inline user defined function.
*
- * @param function to invoke.
- * @param arguments to pass into the user defined function.
+ * @param function
+ * to invoke.
+ * @param arguments
+ * to pass into the user defined function.
*/
private[sql] case class InvokeInlineUserDefinedFunction(
function: UserDefinedFunctionLike,
arguments: Seq[ColumnNode],
isDistinct: Boolean = false,
override val origin: Origin = CurrentOrigin.get)
- extends ColumnNode {
+ extends ColumnNode {
override private[internal] def normalize(): InvokeInlineUserDefinedFunction =
copy(arguments = ColumnNode.normalize(arguments), origin = NO_ORIGIN)
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractArrayType.scala b/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractArrayType.scala
index 406449a337271..5c8c77985bb2c 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractArrayType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractArrayType.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.internal.types
import org.apache.spark.sql.types.{AbstractDataType, ArrayType, DataType}
-
/**
* Use AbstractArrayType(AbstractDataType) for defining expected types for expression parameters.
*/
@@ -30,7 +29,7 @@ case class AbstractArrayType(elementType: AbstractDataType) extends AbstractData
override private[sql] def acceptsType(other: DataType): Boolean = {
other.isInstanceOf[ArrayType] &&
- elementType.acceptsType(other.asInstanceOf[ArrayType].elementType)
+ elementType.acceptsType(other.asInstanceOf[ArrayType].elementType)
}
override private[spark] def simpleString: String = s"array<${elementType.simpleString}>"
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractMapType.scala b/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractMapType.scala
index 62f422f6f80a7..32f4341839f01 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractMapType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractMapType.scala
@@ -19,23 +19,20 @@ package org.apache.spark.sql.internal.types
import org.apache.spark.sql.types.{AbstractDataType, DataType, MapType}
-
/**
- * Use AbstractMapType(AbstractDataType, AbstractDataType)
- * for defining expected types for expression parameters.
+ * Use AbstractMapType(AbstractDataType, AbstractDataType) for defining expected types for
+ * expression parameters.
*/
-case class AbstractMapType(
- keyType: AbstractDataType,
- valueType: AbstractDataType
- ) extends AbstractDataType {
+case class AbstractMapType(keyType: AbstractDataType, valueType: AbstractDataType)
+ extends AbstractDataType {
override private[sql] def defaultConcreteType: DataType =
MapType(keyType.defaultConcreteType, valueType.defaultConcreteType, valueContainsNull = true)
override private[sql] def acceptsType(other: DataType): Boolean = {
other.isInstanceOf[MapType] &&
- keyType.acceptsType(other.asInstanceOf[MapType].keyType) &&
- valueType.acceptsType(other.asInstanceOf[MapType].valueType)
+ keyType.acceptsType(other.asInstanceOf[MapType].keyType) &&
+ valueType.acceptsType(other.asInstanceOf[MapType].valueType)
}
override private[spark] def simpleString: String =
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractStringType.scala b/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractStringType.scala
index 05d1701eff74d..dc4ee013fd189 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractStringType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/internal/types/AbstractStringType.scala
@@ -51,3 +51,12 @@ case object StringTypeBinaryLcase extends AbstractStringType {
case object StringTypeAnyCollation extends AbstractStringType {
override private[sql] def acceptsType(other: DataType): Boolean = other.isInstanceOf[StringType]
}
+
+/**
+ * Use StringTypeNonCSAICollation for expressions supporting all possible collation types except
+ * CS_AI collation types.
+ */
+case object StringTypeNonCSAICollation extends AbstractStringType {
+ override private[sql] def acceptsType(other: DataType): Boolean =
+ other.isInstanceOf[StringType] && other.asInstanceOf[StringType].isNonCSAI
+}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/ExpiredTimerInfo.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/ExpiredTimerInfo.scala
index 49dc393f8481b..a0958aceb3b3a 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/ExpiredTimerInfo.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/ExpiredTimerInfo.scala
@@ -22,12 +22,13 @@ import java.io.Serializable
import org.apache.spark.annotation.{Evolving, Experimental}
/**
- * Class used to provide access to expired timer's expiry time. These values
- * are only relevant if the ExpiredTimerInfo is valid.
+ * Class used to provide access to expired timer's expiry time. These values are only relevant if
+ * the ExpiredTimerInfo is valid.
*/
@Experimental
@Evolving
private[sql] trait ExpiredTimerInfo extends Serializable {
+
/**
* Check if provided ExpiredTimerInfo is valid.
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/GroupState.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/GroupState.scala
index f08a2fd3cc55c..146990917a3fc 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/GroupState.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/GroupState.scala
@@ -27,89 +27,89 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalGroupState
* `flatMapGroupsWithState` operations on `KeyValueGroupedDataset`.
*
* Detail description on `[map/flatMap]GroupsWithState` operation
- * --------------------------------------------------------------
- * Both, `mapGroupsWithState` and `flatMapGroupsWithState` in `KeyValueGroupedDataset`
- * will invoke the user-given function on each group (defined by the grouping function in
- * `Dataset.groupByKey()`) while maintaining a user-defined per-group state between invocations.
- * For a static batch Dataset, the function will be invoked once per group. For a streaming
- * Dataset, the function will be invoked for each group repeatedly in every trigger.
- * That is, in every batch of the `StreamingQuery`,
- * the function will be invoked once for each group that has data in the trigger. Furthermore,
- * if timeout is set, then the function will be invoked on timed-out groups (more detail below).
+ * -------------------------------------------------------------- Both, `mapGroupsWithState` and
+ * `flatMapGroupsWithState` in `KeyValueGroupedDataset` will invoke the user-given function on
+ * each group (defined by the grouping function in `Dataset.groupByKey()`) while maintaining a
+ * user-defined per-group state between invocations. For a static batch Dataset, the function will
+ * be invoked once per group. For a streaming Dataset, the function will be invoked for each group
+ * repeatedly in every trigger. That is, in every batch of the `StreamingQuery`, the function will
+ * be invoked once for each group that has data in the trigger. Furthermore, if timeout is set,
+ * then the function will be invoked on timed-out groups (more detail below).
*
* The function is invoked with the following parameters.
- * - The key of the group.
- * - An iterator containing all the values for this group.
- * - A user-defined state object set by previous invocations of the given function.
+ * - The key of the group.
+ * - An iterator containing all the values for this group.
+ * - A user-defined state object set by previous invocations of the given function.
*
* In case of a batch Dataset, there is only one invocation and the state object will be empty as
- * there is no prior state. Essentially, for batch Datasets, `[map/flatMap]GroupsWithState`
- * is equivalent to `[map/flatMap]Groups` and any updates to the state and/or timeouts have
- * no effect.
+ * there is no prior state. Essentially, for batch Datasets, `[map/flatMap]GroupsWithState` is
+ * equivalent to `[map/flatMap]Groups` and any updates to the state and/or timeouts have no
+ * effect.
*
* The major difference between `mapGroupsWithState` and `flatMapGroupsWithState` is that the
- * former allows the function to return one and only one record, whereas the latter
- * allows the function to return any number of records (including no records). Furthermore, the
+ * former allows the function to return one and only one record, whereas the latter allows the
+ * function to return any number of records (including no records). Furthermore, the
* `flatMapGroupsWithState` is associated with an operation output mode, which can be either
- * `Append` or `Update`. Semantically, this defines whether the output records of one trigger
- * is effectively replacing the previously output records (from previous triggers) or is appending
- * to the list of previously output records. Essentially, this defines how the Result Table (refer
- * to the semantics in the programming guide) is updated, and allows us to reason about the
- * semantics of later operations.
+ * `Append` or `Update`. Semantically, this defines whether the output records of one trigger is
+ * effectively replacing the previously output records (from previous triggers) or is appending to
+ * the list of previously output records. Essentially, this defines how the Result Table (refer to
+ * the semantics in the programming guide) is updated, and allows us to reason about the semantics
+ * of later operations.
*
- * Important points to note about the function (both mapGroupsWithState and flatMapGroupsWithState).
- * - In a trigger, the function will be called only the groups present in the batch. So do not
- * assume that the function will be called in every trigger for every group that has state.
- * - There is no guaranteed ordering of values in the iterator in the function, neither with
- * batch, nor with streaming Datasets.
- * - All the data will be shuffled before applying the function.
- * - If timeout is set, then the function will also be called with no values.
- * See more details on `GroupStateTimeout` below.
+ * Important points to note about the function (both mapGroupsWithState and
+ * flatMapGroupsWithState).
+ * - In a trigger, the function will be called only the groups present in the batch. So do not
+ * assume that the function will be called in every trigger for every group that has state.
+ * - There is no guaranteed ordering of values in the iterator in the function, neither with
+ * batch, nor with streaming Datasets.
+ * - All the data will be shuffled before applying the function.
+ * - If timeout is set, then the function will also be called with no values. See more details
+ * on `GroupStateTimeout` below.
*
* Important points to note about using `GroupState`.
- * - The value of the state cannot be null. So updating state with null will throw
- * `IllegalArgumentException`.
- * - Operations on `GroupState` are not thread-safe. This is to avoid memory barriers.
- * - If `remove()` is called, then `exists()` will return `false`,
- * `get()` will throw `NoSuchElementException` and `getOption()` will return `None`
- * - After that, if `update(newState)` is called, then `exists()` will again return `true`,
- * `get()` and `getOption()`will return the updated value.
+ * - The value of the state cannot be null. So updating state with null will throw
+ * `IllegalArgumentException`.
+ * - Operations on `GroupState` are not thread-safe. This is to avoid memory barriers.
+ * - If `remove()` is called, then `exists()` will return `false`, `get()` will throw
+ * `NoSuchElementException` and `getOption()` will return `None`
+ * - After that, if `update(newState)` is called, then `exists()` will again return `true`,
+ * `get()` and `getOption()`will return the updated value.
*
* Important points to note about using `GroupStateTimeout`.
- * - The timeout type is a global param across all the groups (set as `timeout` param in
- * `[map|flatMap]GroupsWithState`, but the exact timeout duration/timestamp is configurable per
- * group by calling `setTimeout...()` in `GroupState`.
- * - Timeouts can be either based on processing time (i.e.
- * `GroupStateTimeout.ProcessingTimeTimeout`) or event time (i.e.
- * `GroupStateTimeout.EventTimeTimeout`).
- * - With `ProcessingTimeTimeout`, the timeout duration can be set by calling
- * `GroupState.setTimeoutDuration`. The timeout will occur when the clock has advanced by the set
- * duration. Guarantees provided by this timeout with a duration of D ms are as follows:
- * - Timeout will never occur before the clock time has advanced by D ms
- * - Timeout will occur eventually when there is a trigger in the query
- * (i.e. after D ms). So there is no strict upper bound on when the timeout would occur.
- * For example, the trigger interval of the query will affect when the timeout actually occurs.
- * If there is no data in the stream (for any group) for a while, then there will not be
- * any trigger and timeout function call will not occur until there is data.
- * - Since the processing time timeout is based on the clock time, it is affected by the
- * variations in the system clock (i.e. time zone changes, clock skew, etc.).
- * - With `EventTimeTimeout`, the user also has to specify the event time watermark in
- * the query using `Dataset.withWatermark()`. With this setting, data that is older than the
- * watermark is filtered out. The timeout can be set for a group by setting a timeout timestamp
- * using`GroupState.setTimeoutTimestamp()`, and the timeout would occur when the watermark
- * advances beyond the set timestamp. You can control the timeout delay by two parameters -
- * (i) watermark delay and an additional duration beyond the timestamp in the event (which
- * is guaranteed to be newer than watermark due to the filtering). Guarantees provided by this
- * timeout are as follows:
- * - Timeout will never occur before the watermark has exceeded the set timeout.
- * - Similar to processing time timeouts, there is no strict upper bound on the delay when
- * the timeout actually occurs. The watermark can advance only when there is data in the
- * stream and the event time of the data has actually advanced.
- * - When the timeout occurs for a group, the function is called for that group with no values, and
- * `GroupState.hasTimedOut()` set to true.
- * - The timeout is reset every time the function is called on a group, that is,
- * when the group has new data, or the group has timed out. So the user has to set the timeout
- * duration every time the function is called, otherwise, there will not be any timeout set.
+ * - The timeout type is a global param across all the groups (set as `timeout` param in
+ * `[map|flatMap]GroupsWithState`, but the exact timeout duration/timestamp is configurable
+ * per group by calling `setTimeout...()` in `GroupState`.
+ * - Timeouts can be either based on processing time (i.e.
+ * `GroupStateTimeout.ProcessingTimeTimeout`) or event time (i.e.
+ * `GroupStateTimeout.EventTimeTimeout`).
+ * - With `ProcessingTimeTimeout`, the timeout duration can be set by calling
+ * `GroupState.setTimeoutDuration`. The timeout will occur when the clock has advanced by the
+ * set duration. Guarantees provided by this timeout with a duration of D ms are as follows:
+ * - Timeout will never occur before the clock time has advanced by D ms
+ * - Timeout will occur eventually when there is a trigger in the query (i.e. after D ms). So
+ * there is no strict upper bound on when the timeout would occur. For example, the trigger
+ * interval of the query will affect when the timeout actually occurs. If there is no data
+ * in the stream (for any group) for a while, then there will not be any trigger and timeout
+ * function call will not occur until there is data.
+ * - Since the processing time timeout is based on the clock time, it is affected by the
+ * variations in the system clock (i.e. time zone changes, clock skew, etc.).
+ * - With `EventTimeTimeout`, the user also has to specify the event time watermark in the query
+ * using `Dataset.withWatermark()`. With this setting, data that is older than the watermark
+ * is filtered out. The timeout can be set for a group by setting a timeout timestamp
+ * using`GroupState.setTimeoutTimestamp()`, and the timeout would occur when the watermark
+ * advances beyond the set timestamp. You can control the timeout delay by two parameters -
+ * (i) watermark delay and an additional duration beyond the timestamp in the event (which is
+ * guaranteed to be newer than watermark due to the filtering). Guarantees provided by this
+ * timeout are as follows:
+ * - Timeout will never occur before the watermark has exceeded the set timeout.
+ * - Similar to processing time timeouts, there is no strict upper bound on the delay when the
+ * timeout actually occurs. The watermark can advance only when there is data in the stream
+ * and the event time of the data has actually advanced.
+ * - When the timeout occurs for a group, the function is called for that group with no values,
+ * and `GroupState.hasTimedOut()` set to true.
+ * - The timeout is reset every time the function is called on a group, that is, when the group
+ * has new data, or the group has timed out. So the user has to set the timeout duration every
+ * time the function is called, otherwise, there will not be any timeout set.
*
* `[map/flatMap]GroupsWithState` can take a user defined initial state as an additional argument.
* This state will be applied when the first batch of the streaming query is processed. If there
@@ -181,7 +181,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalGroupState
* state.setTimeoutDuration("1 hour"); // Set the timeout
* }
* ...
-* // return something
+ * // return something
* }
* };
*
@@ -191,8 +191,9 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalGroupState
* mappingFunction, Encoders.INT, Encoders.STRING, GroupStateTimeout.ProcessingTimeTimeout);
* }}}
*
- * @tparam S User-defined type of the state to be stored for each group. Must be encodable into
- * Spark SQL types (see `Encoder` for more details).
+ * @tparam S
+ * User-defined type of the state to be stored for each group. Must be encodable into Spark SQL
+ * types (see `Encoder` for more details).
* @since 2.2.0
*/
@Experimental
@@ -217,44 +218,48 @@ trait GroupState[S] extends LogicalGroupState[S] {
/**
* Whether the function has been called because the key has timed out.
- * @note This can return true only when timeouts are enabled in `[map/flatMap]GroupsWithState`.
+ * @note
+ * This can return true only when timeouts are enabled in `[map/flatMap]GroupsWithState`.
*/
def hasTimedOut: Boolean
-
/**
* Set the timeout duration in ms for this key.
*
- * @note [[GroupStateTimeout Processing time timeout]] must be enabled in
- * `[map/flatMap]GroupsWithState` for calling this method.
- * @note This method has no effect when used in a batch query.
+ * @note
+ * [[GroupStateTimeout Processing time timeout]] must be enabled in
+ * `[map/flatMap]GroupsWithState` for calling this method.
+ * @note
+ * This method has no effect when used in a batch query.
*/
@throws[IllegalArgumentException]("if 'durationMs' is not positive")
@throws[UnsupportedOperationException](
"if processing time timeout has not been enabled in [map|flatMap]GroupsWithState")
def setTimeoutDuration(durationMs: Long): Unit
-
/**
* Set the timeout duration for this key as a string. For example, "1 hour", "2 days", etc.
*
- * @note [[GroupStateTimeout Processing time timeout]] must be enabled in
- * `[map/flatMap]GroupsWithState` for calling this method.
- * @note This method has no effect when used in a batch query.
+ * @note
+ * [[GroupStateTimeout Processing time timeout]] must be enabled in
+ * `[map/flatMap]GroupsWithState` for calling this method.
+ * @note
+ * This method has no effect when used in a batch query.
*/
@throws[IllegalArgumentException]("if 'duration' is not a valid duration")
@throws[UnsupportedOperationException](
"if processing time timeout has not been enabled in [map|flatMap]GroupsWithState")
def setTimeoutDuration(duration: String): Unit
-
/**
- * Set the timeout timestamp for this key as milliseconds in epoch time.
- * This timestamp cannot be older than the current watermark.
+ * Set the timeout timestamp for this key as milliseconds in epoch time. This timestamp cannot
+ * be older than the current watermark.
*
- * @note [[GroupStateTimeout Event time timeout]] must be enabled in
- * `[map/flatMap]GroupsWithState` for calling this method.
- * @note This method has no effect when used in a batch query.
+ * @note
+ * [[GroupStateTimeout Event time timeout]] must be enabled in `[map/flatMap]GroupsWithState`
+ * for calling this method.
+ * @note
+ * This method has no effect when used in a batch query.
*/
@throws[IllegalArgumentException](
"if 'timestampMs' is not positive or less than the current watermark in a streaming query")
@@ -262,16 +267,16 @@ trait GroupState[S] extends LogicalGroupState[S] {
"if event time timeout has not been enabled in [map|flatMap]GroupsWithState")
def setTimeoutTimestamp(timestampMs: Long): Unit
-
/**
* Set the timeout timestamp for this key as milliseconds in epoch time and an additional
- * duration as a string (e.g. "1 hour", "2 days", etc.).
- * The final timestamp (including the additional duration) cannot be older than the
- * current watermark.
+ * duration as a string (e.g. "1 hour", "2 days", etc.). The final timestamp (including the
+ * additional duration) cannot be older than the current watermark.
*
- * @note [[GroupStateTimeout Event time timeout]] must be enabled in
- * `[map/flatMap]GroupsWithState` for calling this method.
- * @note This method has no side effect when used in a batch query.
+ * @note
+ * [[GroupStateTimeout Event time timeout]] must be enabled in `[map/flatMap]GroupsWithState`
+ * for calling this method.
+ * @note
+ * This method has no side effect when used in a batch query.
*/
@throws[IllegalArgumentException](
"if 'additionalDuration' is invalid or the final timeout timestamp is less than " +
@@ -280,56 +285,57 @@ trait GroupState[S] extends LogicalGroupState[S] {
"if event time timeout has not been enabled in [map|flatMap]GroupsWithState")
def setTimeoutTimestamp(timestampMs: Long, additionalDuration: String): Unit
-
/**
- * Set the timeout timestamp for this key as a java.sql.Date.
- * This timestamp cannot be older than the current watermark.
+ * Set the timeout timestamp for this key as a java.sql.Date. This timestamp cannot be older
+ * than the current watermark.
*
- * @note [[GroupStateTimeout Event time timeout]] must be enabled in
- * `[map/flatMap]GroupsWithState` for calling this method.
- * @note This method has no side effect when used in a batch query.
+ * @note
+ * [[GroupStateTimeout Event time timeout]] must be enabled in `[map/flatMap]GroupsWithState`
+ * for calling this method.
+ * @note
+ * This method has no side effect when used in a batch query.
*/
@throws[UnsupportedOperationException](
"if event time timeout has not been enabled in [map|flatMap]GroupsWithState")
def setTimeoutTimestamp(timestamp: java.sql.Date): Unit
-
/**
- * Set the timeout timestamp for this key as a java.sql.Date and an additional
- * duration as a string (e.g. "1 hour", "2 days", etc.).
- * The final timestamp (including the additional duration) cannot be older than the
- * current watermark.
+ * Set the timeout timestamp for this key as a java.sql.Date and an additional duration as a
+ * string (e.g. "1 hour", "2 days", etc.). The final timestamp (including the additional
+ * duration) cannot be older than the current watermark.
*
- * @note [[GroupStateTimeout Event time timeout]] must be enabled in
- * `[map/flatMap]GroupsWithState` for calling this method.
- * @note This method has no side effect when used in a batch query.
+ * @note
+ * [[GroupStateTimeout Event time timeout]] must be enabled in `[map/flatMap]GroupsWithState`
+ * for calling this method.
+ * @note
+ * This method has no side effect when used in a batch query.
*/
@throws[IllegalArgumentException]("if 'additionalDuration' is invalid")
@throws[UnsupportedOperationException](
"if event time timeout has not been enabled in [map|flatMap]GroupsWithState")
def setTimeoutTimestamp(timestamp: java.sql.Date, additionalDuration: String): Unit
-
/**
* Get the current event time watermark as milliseconds in epoch time.
*
- * @note In a streaming query, this can be called only when watermark is set before calling
- * `[map/flatMap]GroupsWithState`. In a batch query, this method always returns -1.
- * @note The watermark gets propagated in the end of each query. As a result, this method will
- * return 0 (1970-01-01T00:00:00) for the first micro-batch. If you use this value
- * as a part of the timestamp set in the `setTimeoutTimestamp`, it may lead to the
- * state expiring immediately in the next micro-batch, once the watermark gets the
- * real value from your data.
+ * @note
+ * In a streaming query, this can be called only when watermark is set before calling
+ * `[map/flatMap]GroupsWithState`. In a batch query, this method always returns -1.
+ * @note
+ * The watermark gets propagated in the end of each query. As a result, this method will
+ * return 0 (1970-01-01T00:00:00) for the first micro-batch. If you use this value as a part
+ * of the timestamp set in the `setTimeoutTimestamp`, it may lead to the state expiring
+ * immediately in the next micro-batch, once the watermark gets the real value from your data.
*/
@throws[UnsupportedOperationException](
"if watermark has not been set before in [map|flatMap]GroupsWithState")
def getCurrentWatermarkMs(): Long
-
/**
* Get the current processing time as milliseconds in epoch time.
- * @note In a streaming query, this will return a constant value throughout the duration of a
- * trigger, even if the trigger is re-executed.
+ * @note
+ * In a streaming query, this will return a constant value throughout the duration of a
+ * trigger, even if the trigger is re-executed.
*/
def getCurrentProcessingTimeMs(): Long
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/ListState.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/ListState.scala
index 0e2d6cc3778c6..568578d1f4ff6 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/ListState.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/ListState.scala
@@ -21,8 +21,7 @@ import org.apache.spark.annotation.{Evolving, Experimental}
@Experimental
@Evolving
/**
- * Interface used for arbitrary stateful operations with the v2 API to capture
- * list value state.
+ * Interface used for arbitrary stateful operations with the v2 API to capture list value state.
*/
private[sql] trait ListState[S] extends Serializable {
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/MapState.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/MapState.scala
index 030c3ee989c6f..7b01888bbac49 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/MapState.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/MapState.scala
@@ -21,10 +21,10 @@ import org.apache.spark.annotation.{Evolving, Experimental}
@Experimental
@Evolving
/**
- * Interface used for arbitrary stateful operations with the v2 API to capture
- * map value state.
+ * Interface used for arbitrary stateful operations with the v2 API to capture map value state.
*/
trait MapState[K, V] extends Serializable {
+
/** Whether state exists or not. */
def exists(): Boolean
@@ -35,7 +35,7 @@ trait MapState[K, V] extends Serializable {
def containsKey(key: K): Boolean
/** Update value for given user key */
- def updateValue(key: K, value: V) : Unit
+ def updateValue(key: K, value: V): Unit
/** Get the map associated with grouping key */
def iterator(): Iterator[(K, V)]
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/QueryInfo.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/QueryInfo.scala
index 7754a514fdd6b..f239bcff49fea 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/QueryInfo.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/QueryInfo.scala
@@ -22,8 +22,8 @@ import java.util.UUID
import org.apache.spark.annotation.{Evolving, Experimental}
/**
- * Represents the query info provided to the stateful processor used in the
- * arbitrary state API v2 to easily identify task retries on the same partition.
+ * Represents the query info provided to the stateful processor used in the arbitrary state API v2
+ * to easily identify task retries on the same partition.
*/
@Experimental
@Evolving
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/StatefulProcessor.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/StatefulProcessor.scala
index 54e6a9a4ab678..d2c6010454c55 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/StatefulProcessor.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/StatefulProcessor.scala
@@ -31,31 +31,35 @@ import org.apache.spark.sql.errors.ExecutionErrors
private[sql] abstract class StatefulProcessor[K, I, O] extends Serializable {
/**
- * Handle to the stateful processor that provides access to the state store and other
- * stateful processing related APIs.
+ * Handle to the stateful processor that provides access to the state store and other stateful
+ * processing related APIs.
*/
private var statefulProcessorHandle: StatefulProcessorHandle = null
/**
- * Function that will be invoked as the first method that allows for users to
- * initialize all their state variables and perform other init actions before handling data.
- * @param outputMode - output mode for the stateful processor
- * @param timeMode - time mode for the stateful processor.
+ * Function that will be invoked as the first method that allows for users to initialize all
+ * their state variables and perform other init actions before handling data.
+ * @param outputMode
+ * \- output mode for the stateful processor
+ * @param timeMode
+ * \- time mode for the stateful processor.
*/
- def init(
- outputMode: OutputMode,
- timeMode: TimeMode): Unit
+ def init(outputMode: OutputMode, timeMode: TimeMode): Unit
/**
* Function that will allow users to interact with input data rows along with the grouping key
* and current timer values and optionally provide output rows.
- * @param key - grouping key
- * @param inputRows - iterator of input rows associated with grouping key
- * @param timerValues - instance of TimerValues that provides access to current processing/event
- * time if available
- * @param expiredTimerInfo - instance of ExpiredTimerInfo that provides access to expired timer
- * if applicable
- * @return - Zero or more output rows
+ * @param key
+ * \- grouping key
+ * @param inputRows
+ * \- iterator of input rows associated with grouping key
+ * @param timerValues
+ * \- instance of TimerValues that provides access to current processing/event time if
+ * available
+ * @param expiredTimerInfo
+ * \- instance of ExpiredTimerInfo that provides access to expired timer if applicable
+ * @return
+ * \- Zero or more output rows
*/
def handleInputRows(
key: K,
@@ -64,16 +68,17 @@ private[sql] abstract class StatefulProcessor[K, I, O] extends Serializable {
expiredTimerInfo: ExpiredTimerInfo): Iterator[O]
/**
- * Function called as the last method that allows for users to perform
- * any cleanup or teardown operations.
+ * Function called as the last method that allows for users to perform any cleanup or teardown
+ * operations.
*/
- def close (): Unit = {}
+ def close(): Unit = {}
/**
* Function to set the stateful processor handle that will be used to interact with the state
* store and other stateful processor related operations.
*
- * @param handle - instance of StatefulProcessorHandle
+ * @param handle
+ * \- instance of StatefulProcessorHandle
*/
final def setHandle(handle: StatefulProcessorHandle): Unit = {
statefulProcessorHandle = handle
@@ -82,7 +87,8 @@ private[sql] abstract class StatefulProcessor[K, I, O] extends Serializable {
/**
* Function to get the stateful processor handle that will be used to interact with the state
*
- * @return handle - instance of StatefulProcessorHandle
+ * @return
+ * handle - instance of StatefulProcessorHandle
*/
final def getHandle: StatefulProcessorHandle = {
if (statefulProcessorHandle == null) {
@@ -93,23 +99,25 @@ private[sql] abstract class StatefulProcessor[K, I, O] extends Serializable {
}
/**
- * Stateful processor with support for specifying initial state.
- * Accepts a user-defined type as initial state to be initialized in the first batch.
- * This can be used for starting a new streaming query with existing state from a
- * previous streaming query.
+ * Stateful processor with support for specifying initial state. Accepts a user-defined type as
+ * initial state to be initialized in the first batch. This can be used for starting a new
+ * streaming query with existing state from a previous streaming query.
*/
@Experimental
@Evolving
private[sql] abstract class StatefulProcessorWithInitialState[K, I, O, S]
- extends StatefulProcessor[K, I, O] {
+ extends StatefulProcessor[K, I, O] {
/**
* Function that will be invoked only in the first batch for users to process initial states.
*
- * @param key - grouping key
- * @param initialState - A row in the initial state to be processed
- * @param timerValues - instance of TimerValues that provides access to current processing/event
- * time if available
+ * @param key
+ * \- grouping key
+ * @param initialState
+ * \- A row in the initial state to be processed
+ * @param timerValues
+ * \- instance of TimerValues that provides access to current processing/event time if
+ * available
*/
def handleInitialState(key: K, initialState: S, timerValues: TimerValues): Unit
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/StatefulProcessorHandle.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/StatefulProcessorHandle.scala
index 4dc2ca875ef0e..d1eca0f3967d9 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/StatefulProcessorHandle.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/StatefulProcessorHandle.scala
@@ -22,39 +22,46 @@ import org.apache.spark.annotation.{Evolving, Experimental}
import org.apache.spark.sql.Encoder
/**
- * Represents the operation handle provided to the stateful processor used in the
- * arbitrary state API v2.
+ * Represents the operation handle provided to the stateful processor used in the arbitrary state
+ * API v2.
*/
@Experimental
@Evolving
private[sql] trait StatefulProcessorHandle extends Serializable {
/**
- * Function to create new or return existing single value state variable of given type.
- * The user must ensure to call this function only within the `init()` method of the
- * StatefulProcessor.
+ * Function to create new or return existing single value state variable of given type. The user
+ * must ensure to call this function only within the `init()` method of the StatefulProcessor.
*
- * @param stateName - name of the state variable
- * @param valEncoder - SQL encoder for state variable
- * @tparam T - type of state variable
- * @return - instance of ValueState of type T that can be used to store state persistently
+ * @param stateName
+ * \- name of the state variable
+ * @param valEncoder
+ * \- SQL encoder for state variable
+ * @tparam T
+ * \- type of state variable
+ * @return
+ * \- instance of ValueState of type T that can be used to store state persistently
*/
def getValueState[T](stateName: String, valEncoder: Encoder[T]): ValueState[T]
/**
- * Function to create new or return existing single value state variable of given type
- * with ttl. State values will not be returned past ttlDuration, and will be eventually removed
- * from the state store. Any state update resets the ttl to current processing time plus
- * ttlDuration.
+ * Function to create new or return existing single value state variable of given type with ttl.
+ * State values will not be returned past ttlDuration, and will be eventually removed from the
+ * state store. Any state update resets the ttl to current processing time plus ttlDuration.
*
* The user must ensure to call this function only within the `init()` method of the
* StatefulProcessor.
*
- * @param stateName - name of the state variable
- * @param valEncoder - SQL encoder for state variable
- * @param ttlConfig - the ttl configuration (time to live duration etc.)
- * @tparam T - type of state variable
- * @return - instance of ValueState of type T that can be used to store state persistently
+ * @param stateName
+ * \- name of the state variable
+ * @param valEncoder
+ * \- SQL encoder for state variable
+ * @param ttlConfig
+ * \- the ttl configuration (time to live duration etc.)
+ * @tparam T
+ * \- type of state variable
+ * @return
+ * \- instance of ValueState of type T that can be used to store state persistently
*/
def getValueState[T](
stateName: String,
@@ -62,30 +69,39 @@ private[sql] trait StatefulProcessorHandle extends Serializable {
ttlConfig: TTLConfig): ValueState[T]
/**
- * Creates new or returns existing list state associated with stateName.
- * The ListState persists values of type T.
+ * Creates new or returns existing list state associated with stateName. The ListState persists
+ * values of type T.
*
- * @param stateName - name of the state variable
- * @param valEncoder - SQL encoder for state variable
- * @tparam T - type of state variable
- * @return - instance of ListState of type T that can be used to store state persistently
+ * @param stateName
+ * \- name of the state variable
+ * @param valEncoder
+ * \- SQL encoder for state variable
+ * @tparam T
+ * \- type of state variable
+ * @return
+ * \- instance of ListState of type T that can be used to store state persistently
*/
def getListState[T](stateName: String, valEncoder: Encoder[T]): ListState[T]
/**
- * Function to create new or return existing list state variable of given type
- * with ttl. State values will not be returned past ttlDuration, and will be eventually removed
- * from the state store. Any values in listState which have expired after ttlDuration will not
- * be returned on get() and will be eventually removed from the state.
+ * Function to create new or return existing list state variable of given type with ttl. State
+ * values will not be returned past ttlDuration, and will be eventually removed from the state
+ * store. Any values in listState which have expired after ttlDuration will not be returned on
+ * get() and will be eventually removed from the state.
*
* The user must ensure to call this function only within the `init()` method of the
* StatefulProcessor.
*
- * @param stateName - name of the state variable
- * @param valEncoder - SQL encoder for state variable
- * @param ttlConfig - the ttl configuration (time to live duration etc.)
- * @tparam T - type of state variable
- * @return - instance of ListState of type T that can be used to store state persistently
+ * @param stateName
+ * \- name of the state variable
+ * @param valEncoder
+ * \- SQL encoder for state variable
+ * @param ttlConfig
+ * \- the ttl configuration (time to live duration etc.)
+ * @tparam T
+ * \- type of state variable
+ * @return
+ * \- instance of ListState of type T that can be used to store state persistently
*/
def getListState[T](
stateName: String,
@@ -93,15 +109,21 @@ private[sql] trait StatefulProcessorHandle extends Serializable {
ttlConfig: TTLConfig): ListState[T]
/**
- * Creates new or returns existing map state associated with stateName.
- * The MapState persists Key-Value pairs of type [K, V].
+ * Creates new or returns existing map state associated with stateName. The MapState persists
+ * Key-Value pairs of type [K, V].
*
- * @param stateName - name of the state variable
- * @param userKeyEnc - spark sql encoder for the map key
- * @param valEncoder - spark sql encoder for the map value
- * @tparam K - type of key for map state variable
- * @tparam V - type of value for map state variable
- * @return - instance of MapState of type [K,V] that can be used to store state persistently
+ * @param stateName
+ * \- name of the state variable
+ * @param userKeyEnc
+ * \- spark sql encoder for the map key
+ * @param valEncoder
+ * \- spark sql encoder for the map value
+ * @tparam K
+ * \- type of key for map state variable
+ * @tparam V
+ * \- type of value for map state variable
+ * @return
+ * \- instance of MapState of type [K,V] that can be used to store state persistently
*/
def getMapState[K, V](
stateName: String,
@@ -109,57 +131,68 @@ private[sql] trait StatefulProcessorHandle extends Serializable {
valEncoder: Encoder[V]): MapState[K, V]
/**
- * Function to create new or return existing map state variable of given type
- * with ttl. State values will not be returned past ttlDuration, and will be eventually removed
- * from the state store. Any values in mapState which have expired after ttlDuration will not
- * returned on get() and will be eventually removed from the state.
+ * Function to create new or return existing map state variable of given type with ttl. State
+ * values will not be returned past ttlDuration, and will be eventually removed from the state
+ * store. Any values in mapState which have expired after ttlDuration will not returned on get()
+ * and will be eventually removed from the state.
*
* The user must ensure to call this function only within the `init()` method of the
* StatefulProcessor.
*
- * @param stateName - name of the state variable
- * @param userKeyEnc - spark sql encoder for the map key
- * @param valEncoder - SQL encoder for state variable
- * @param ttlConfig - the ttl configuration (time to live duration etc.)
- * @tparam K - type of key for map state variable
- * @tparam V - type of value for map state variable
- * @return - instance of MapState of type [K,V] that can be used to store state persistently
+ * @param stateName
+ * \- name of the state variable
+ * @param userKeyEnc
+ * \- spark sql encoder for the map key
+ * @param valEncoder
+ * \- SQL encoder for state variable
+ * @param ttlConfig
+ * \- the ttl configuration (time to live duration etc.)
+ * @tparam K
+ * \- type of key for map state variable
+ * @tparam V
+ * \- type of value for map state variable
+ * @return
+ * \- instance of MapState of type [K,V] that can be used to store state persistently
*/
def getMapState[K, V](
- stateName: String,
- userKeyEnc: Encoder[K],
- valEncoder: Encoder[V],
- ttlConfig: TTLConfig): MapState[K, V]
+ stateName: String,
+ userKeyEnc: Encoder[K],
+ valEncoder: Encoder[V],
+ ttlConfig: TTLConfig): MapState[K, V]
/** Function to return queryInfo for currently running task */
def getQueryInfo(): QueryInfo
/**
- * Function to register a processing/event time based timer for given implicit grouping key
- * and provided timestamp
- * @param expiryTimestampMs - timer expiry timestamp in milliseconds
+ * Function to register a processing/event time based timer for given implicit grouping key and
+ * provided timestamp
+ * @param expiryTimestampMs
+ * \- timer expiry timestamp in milliseconds
*/
def registerTimer(expiryTimestampMs: Long): Unit
/**
- * Function to delete a processing/event time based timer for given implicit grouping key
- * and provided timestamp
- * @param expiryTimestampMs - timer expiry timestamp in milliseconds
+ * Function to delete a processing/event time based timer for given implicit grouping key and
+ * provided timestamp
+ * @param expiryTimestampMs
+ * \- timer expiry timestamp in milliseconds
*/
def deleteTimer(expiryTimestampMs: Long): Unit
/**
- * Function to list all the timers registered for given implicit grouping key
- * Note: calling listTimers() within the `handleInputRows` method of the StatefulProcessor
- * will return all the unprocessed registered timers, including the one being fired within the
- * invocation of `handleInputRows`.
- * @return - list of all the registered timers for given implicit grouping key
+ * Function to list all the timers registered for given implicit grouping key Note: calling
+ * listTimers() within the `handleInputRows` method of the StatefulProcessor will return all the
+ * unprocessed registered timers, including the one being fired within the invocation of
+ * `handleInputRows`.
+ * @return
+ * \- list of all the registered timers for given implicit grouping key
*/
def listTimers(): Iterator[Long]
/**
* Function to delete and purge state variable if defined previously
- * @param stateName - name of the state variable
+ * @param stateName
+ * \- name of the state variable
*/
def deleteIfExists(stateName: String): Unit
}
diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
similarity index 63%
rename from connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
rename to sql/api/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
index fcb4bdcb327bc..a6684969ff1ec 100644
--- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
@@ -20,19 +20,24 @@ package org.apache.spark.sql.streaming
import java.util.UUID
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
+import com.fasterxml.jackson.databind.node.TreeTraversingParser
import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule}
-import org.json4s.{JObject, JString, JValue}
+import org.json4s.{JObject, JString}
+import org.json4s.JsonAST.JValue
import org.json4s.JsonDSL.{jobject2assoc, pair2Assoc}
import org.json4s.jackson.JsonMethods.{compact, render}
import org.apache.spark.annotation.Evolving
+import org.apache.spark.scheduler.SparkListenerEvent
/**
- * Interface for listening to events related to [[StreamingQuery StreamingQueries]].
+ * Interface for listening to events related to
+ * [[org.apache.spark.sql.api.StreamingQuery StreamingQueries]].
+ *
* @note
* The methods are not thread-safe as they may be called from different threads.
*
- * @since 3.5.0
+ * @since 2.0.0
*/
@Evolving
abstract class StreamingQueryListener extends Serializable {
@@ -42,12 +47,11 @@ abstract class StreamingQueryListener extends Serializable {
/**
* Called when a query is started.
* @note
- * This is called synchronously with
- * [[org.apache.spark.sql.streaming.DataStreamWriter `DataStreamWriter.start()`]], that is,
- * `onQueryStart` will be called on all listeners before `DataStreamWriter.start()` returns
- * the corresponding [[StreamingQuery]]. Please don't block this method as it will block your
- * query.
- * @since 3.5.0
+ * This is called synchronously with `DataStreamWriter.start()`, that is, `onQueryStart` will
+ * be called on all listeners before `DataStreamWriter.start()` returns the corresponding
+ * [[org.apache.spark.sql.api.StreamingQuery]]. Please don't block this method as it will
+ * block your query.
+ * @since 2.0.0
*/
def onQueryStarted(event: QueryStartedEvent): Unit
@@ -55,11 +59,12 @@ abstract class StreamingQueryListener extends Serializable {
* Called when there is some status update (ingestion rate updated, etc.)
*
* @note
- * This method is asynchronous. The status in [[StreamingQuery]] will always be latest no
- * matter when this method is called. Therefore, the status of [[StreamingQuery]] may be
- * changed before/when you process the event. E.g., you may find [[StreamingQuery]] is
- * terminated when you are processing `QueryProgressEvent`.
- * @since 3.5.0
+ * This method is asynchronous. The status in [[org.apache.spark.sql.api.StreamingQuery]] will
+ * always be latest no matter when this method is called. Therefore, the status of
+ * [[org.apache.spark.sql.api.StreamingQuery]] may be changed before/when you process the
+ * event. E.g., you may find [[org.apache.spark.sql.api.StreamingQuery]] is terminated when
+ * you are processing `QueryProgressEvent`.
+ * @since 2.0.0
*/
def onQueryProgress(event: QueryProgressEvent): Unit
@@ -71,24 +76,68 @@ abstract class StreamingQueryListener extends Serializable {
/**
* Called when a query is stopped, with or without error.
- * @since 3.5.0
+ * @since 2.0.0
*/
def onQueryTerminated(event: QueryTerminatedEvent): Unit
}
+/**
+ * Py4J allows a pure interface so this proxy is required.
+ */
+private[spark] trait PythonStreamingQueryListener {
+ import StreamingQueryListener._
+
+ def onQueryStarted(event: QueryStartedEvent): Unit
+
+ def onQueryProgress(event: QueryProgressEvent): Unit
+
+ def onQueryIdle(event: QueryIdleEvent): Unit
+
+ def onQueryTerminated(event: QueryTerminatedEvent): Unit
+}
+
+private[spark] class PythonStreamingQueryListenerWrapper(listener: PythonStreamingQueryListener)
+ extends StreamingQueryListener {
+ import StreamingQueryListener._
+
+ def onQueryStarted(event: QueryStartedEvent): Unit = listener.onQueryStarted(event)
+
+ def onQueryProgress(event: QueryProgressEvent): Unit = listener.onQueryProgress(event)
+
+ override def onQueryIdle(event: QueryIdleEvent): Unit = listener.onQueryIdle(event)
+
+ def onQueryTerminated(event: QueryTerminatedEvent): Unit = listener.onQueryTerminated(event)
+}
+
/**
* Companion object of [[StreamingQueryListener]] that defines the listener events.
- * @since 3.5.0
+ * @since 2.0.0
*/
@Evolving
object StreamingQueryListener extends Serializable {
+ private val mapper = {
+ val ret = new ObjectMapper() with ClassTagExtensions
+ ret.registerModule(DefaultScalaModule)
+ ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+ ret
+ }
+
+ private case class EventParser(json: String) {
+ private val tree = mapper.readTree(json)
+ def getString(name: String): String = tree.get(name).asText()
+ def getUUID(name: String): UUID = UUID.fromString(getString(name))
+ def getProgress(name: String): StreamingQueryProgress = {
+ val parser = new TreeTraversingParser(tree.get(name), mapper)
+ parser.readValueAs(classOf[StreamingQueryProgress])
+ }
+ }
/**
* Base type of [[StreamingQueryListener]] events
- * @since 3.5.0
+ * @since 2.0.0
*/
@Evolving
- trait Event
+ trait Event extends SparkListenerEvent
/**
* Event representing the start of a query
@@ -100,7 +149,7 @@ object StreamingQueryListener extends Serializable {
* User-specified name of the query, null if not specified.
* @param timestamp
* The timestamp to start a query.
- * @since 3.5.0
+ * @since 2.1.0
*/
@Evolving
class QueryStartedEvent private[sql] (
@@ -122,25 +171,22 @@ object StreamingQueryListener extends Serializable {
}
private[spark] object QueryStartedEvent {
- private val mapper = {
- val ret = new ObjectMapper() with ClassTagExtensions
- ret.registerModule(DefaultScalaModule)
- ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
- ret
- }
- private[spark] def jsonString(event: QueryStartedEvent): String =
- mapper.writeValueAsString(event)
-
- private[spark] def fromJson(json: String): QueryStartedEvent =
- mapper.readValue[QueryStartedEvent](json)
+ private[spark] def fromJson(json: String): QueryStartedEvent = {
+ val parser = EventParser(json)
+ new QueryStartedEvent(
+ parser.getUUID("id"),
+ parser.getUUID("runId"),
+ parser.getString("name"),
+ parser.getString("name"))
+ }
}
/**
* Event representing any progress updates in a query.
* @param progress
* The query progress updates.
- * @since 3.5.0
+ * @since 2.1.0
*/
@Evolving
class QueryProgressEvent private[sql] (val progress: StreamingQueryProgress)
@@ -153,18 +199,11 @@ object StreamingQueryListener extends Serializable {
}
private[spark] object QueryProgressEvent {
- private val mapper = {
- val ret = new ObjectMapper() with ClassTagExtensions
- ret.registerModule(DefaultScalaModule)
- ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
- ret
- }
-
- private[spark] def jsonString(event: QueryProgressEvent): String =
- mapper.writeValueAsString(event)
- private[spark] def fromJson(json: String): QueryProgressEvent =
- mapper.readValue[QueryProgressEvent](json)
+ private[spark] def fromJson(json: String): QueryProgressEvent = {
+ val parser = EventParser(json)
+ new QueryProgressEvent(parser.getProgress("progress"))
+ }
}
/**
@@ -193,18 +232,14 @@ object StreamingQueryListener extends Serializable {
}
private[spark] object QueryIdleEvent {
- private val mapper = {
- val ret = new ObjectMapper() with ClassTagExtensions
- ret.registerModule(DefaultScalaModule)
- ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
- ret
- }
- private[spark] def jsonString(event: QueryTerminatedEvent): String =
- mapper.writeValueAsString(event)
-
- private[spark] def fromJson(json: String): QueryTerminatedEvent =
- mapper.readValue[QueryTerminatedEvent](json)
+ private[spark] def fromJson(json: String): QueryIdleEvent = {
+ val parser = EventParser(json)
+ new QueryIdleEvent(
+ parser.getUUID("id"),
+ parser.getUUID("runId"),
+ parser.getString("timestamp"))
+ }
}
/**
@@ -221,7 +256,7 @@ object StreamingQueryListener extends Serializable {
* The error class from the exception if the query was terminated with an exception which is a
* part of error class framework. If the query was terminated without an exception, or the
* exception is not a part of error class framework, it will be `None`.
- * @since 3.5.0
+ * @since 2.1.0
*/
@Evolving
class QueryTerminatedEvent private[sql] (
@@ -247,17 +282,13 @@ object StreamingQueryListener extends Serializable {
}
private[spark] object QueryTerminatedEvent {
- private val mapper = {
- val ret = new ObjectMapper() with ClassTagExtensions
- ret.registerModule(DefaultScalaModule)
- ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
- ret
+ private[spark] def fromJson(json: String): QueryTerminatedEvent = {
+ val parser = EventParser(json)
+ new QueryTerminatedEvent(
+ parser.getUUID("id"),
+ parser.getUUID("runId"),
+ Option(parser.getString("exception")),
+ Option(parser.getString("errorClassOnException")))
}
-
- private[spark] def jsonString(event: QueryTerminatedEvent): String =
- mapper.writeValueAsString(event)
-
- private[spark] def fromJson(json: String): QueryTerminatedEvent =
- mapper.readValue[QueryTerminatedEvent](json)
}
}
diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala
similarity index 95%
rename from connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala
rename to sql/api/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala
index cdda25876b250..c37cdd00c8866 100644
--- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.scala
@@ -36,7 +36,7 @@ import org.apache.spark.annotation.Evolving
* True when the trigger is actively firing, false when waiting for the next trigger time.
* Doesn't apply to ContinuousExecution where it is always false.
*
- * @since 3.5.0
+ * @since 2.1.0
*/
@Evolving
class StreamingQueryStatus protected[sql] (
@@ -44,7 +44,6 @@ class StreamingQueryStatus protected[sql] (
val isDataAvailable: Boolean,
val isTriggerActive: Boolean)
extends Serializable {
- // This is a copy of the same class in sql/core/.../streaming/StreamingQueryStatus.scala
/** The compact JSON representation of this status. */
def json: String = compact(render(jsonValue))
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/TTLConfig.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/TTLConfig.scala
index 576e09d5d7fe2..ce786aa943d89 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/TTLConfig.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/TTLConfig.scala
@@ -20,11 +20,11 @@ package org.apache.spark.sql.streaming
import java.time.Duration
/**
- * TTL Configuration for state variable. State values will not be returned past ttlDuration,
- * and will be eventually removed from the state store. Any state update resets the ttl to
- * current processing time plus ttlDuration.
+ * TTL Configuration for state variable. State values will not be returned past ttlDuration, and
+ * will be eventually removed from the state store. Any state update resets the ttl to current
+ * processing time plus ttlDuration.
*
- * @param ttlDuration time to live duration for state
- * stored in the state variable.
+ * @param ttlDuration
+ * time to live duration for state stored in the state variable.
*/
case class TTLConfig(ttlDuration: Duration)
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/TimerValues.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/TimerValues.scala
index f0aef58228188..04c5f59428f7f 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/TimerValues.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/TimerValues.scala
@@ -22,25 +22,29 @@ import java.io.Serializable
import org.apache.spark.annotation.{Evolving, Experimental}
/**
- * Class used to provide access to timer values for processing and event time populated
- * before method invocations using the arbitrary state API v2.
+ * Class used to provide access to timer values for processing and event time populated before
+ * method invocations using the arbitrary state API v2.
*/
@Experimental
@Evolving
private[sql] trait TimerValues extends Serializable {
+
/**
* Get the current processing time as milliseconds in epoch time.
- * @note This will return a constant value throughout the duration of a streaming query trigger,
- * even if the trigger is re-executed.
+ * @note
+ * This will return a constant value throughout the duration of a streaming query trigger,
+ * even if the trigger is re-executed.
*/
def getCurrentProcessingTimeInMs(): Long
/**
* Get the current event time watermark as milliseconds in epoch time.
*
- * @note This can be called only when watermark is set before calling `transformWithState`.
- * @note The watermark gets propagated at the end of each query. As a result, this method will
- * return 0 (1970-01-01T00:00:00) for the first micro-batch.
+ * @note
+ * This can be called only when watermark is set before calling `transformWithState`.
+ * @note
+ * The watermark gets propagated at the end of each query. As a result, this method will
+ * return 0 (1970-01-01T00:00:00) for the first micro-batch.
*/
def getCurrentWatermarkInMs(): Long
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/streaming/ValueState.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/ValueState.scala
index 8a2661e1a55b1..edb5f65365ab8 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/streaming/ValueState.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/ValueState.scala
@@ -24,8 +24,7 @@ import org.apache.spark.annotation.{Evolving, Experimental}
@Experimental
@Evolving
/**
- * Interface used for arbitrary stateful operations with the v2 API to capture
- * single value state.
+ * Interface used for arbitrary stateful operations with the v2 API to capture single value state.
*/
private[sql] trait ValueState[S] extends Serializable {
@@ -34,7 +33,8 @@ private[sql] trait ValueState[S] extends Serializable {
/**
* Get the state value if it exists
- * @throws java.util.NoSuchElementException if the state does not exist
+ * @throws java.util.NoSuchElementException
+ * if the state does not exist
*/
@throws[NoSuchElementException]
def get(): S
@@ -45,7 +45,8 @@ private[sql] trait ValueState[S] extends Serializable {
/**
* Update the value of the state.
*
- * @param newState the new value
+ * @param newState
+ * the new value
*/
def update(newState: S): Unit
diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/streaming/progress.scala b/sql/api/src/main/scala/org/apache/spark/sql/streaming/progress.scala
similarity index 99%
rename from connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/streaming/progress.scala
rename to sql/api/src/main/scala/org/apache/spark/sql/streaming/progress.scala
index ebd13bc248f97..b7573cb280444 100644
--- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/streaming/progress.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/streaming/progress.scala
@@ -139,7 +139,7 @@ class StateOperatorProgress private[spark] (
* Information about operators in the query that store state.
* @param sources
* detailed statistics on data being read from each of the streaming sources.
- * @since 3.5.0
+ * @since 2.1.0
*/
@Evolving
class StreamingQueryProgress private[spark] (
@@ -195,7 +195,7 @@ class StreamingQueryProgress private[spark] (
}
private[spark] object StreamingQueryProgress {
- private val mapper = {
+ private[this] val mapper = {
val ret = new ObjectMapper() with ClassTagExtensions
ret.registerModule(DefaultScalaModule)
ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
@@ -227,7 +227,7 @@ private[spark] object StreamingQueryProgress {
* The rate at which data is arriving from this source.
* @param processedRowsPerSecond
* The rate at which data from this source is being processed by Spark.
- * @since 3.5.0
+ * @since 2.1.0
*/
@Evolving
class SourceProgress protected[spark] (
@@ -276,7 +276,7 @@ class SourceProgress protected[spark] (
* @param numOutputRows
* Number of rows written to the sink or -1 for Continuous Mode (temporarily) or Sink V1 (until
* decommissioned).
- * @since 3.5.0
+ * @since 2.1.0
*/
@Evolving
class SinkProgress protected[spark] (
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
index 67f634f8379cd..9590fb23e16b1 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.errors.DataTypeErrors
* A non-concrete data type, reserved for internal uses.
*/
private[sql] abstract class AbstractDataType {
+
/**
* The default concrete type to use if we want to cast a null literal into this type.
*/
@@ -47,7 +48,6 @@ private[sql] abstract class AbstractDataType {
private[sql] def simpleString: String
}
-
/**
* A collection of types that can be used to specify type constraints. The sequence also specifies
* precedence: an earlier type takes precedence over a latter type.
@@ -59,7 +59,7 @@ private[sql] abstract class AbstractDataType {
* This means that we prefer StringType over BinaryType if it is possible to cast to StringType.
*/
private[sql] class TypeCollection(private val types: Seq[AbstractDataType])
- extends AbstractDataType {
+ extends AbstractDataType {
require(types.nonEmpty, s"TypeCollection ($types) cannot be empty")
@@ -73,22 +73,20 @@ private[sql] class TypeCollection(private val types: Seq[AbstractDataType])
}
}
-
private[sql] object TypeCollection {
/**
* Types that include numeric types and ANSI interval types.
*/
- val NumericAndAnsiInterval = TypeCollection(
- NumericType,
- DayTimeIntervalType,
- YearMonthIntervalType)
+ val NumericAndAnsiInterval =
+ TypeCollection(NumericType, DayTimeIntervalType, YearMonthIntervalType)
/**
- * Types that include numeric and ANSI interval types, and additionally the legacy interval type.
- * They are only used in unary_minus, unary_positive, add and subtract operations.
+ * Types that include numeric and ANSI interval types, and additionally the legacy interval
+ * type. They are only used in unary_minus, unary_positive, add and subtract operations.
*/
- val NumericAndInterval = new TypeCollection(NumericAndAnsiInterval.types :+ CalendarIntervalType)
+ val NumericAndInterval = new TypeCollection(
+ NumericAndAnsiInterval.types :+ CalendarIntervalType)
def apply(types: AbstractDataType*): TypeCollection = new TypeCollection(types)
@@ -98,7 +96,6 @@ private[sql] object TypeCollection {
}
}
-
/**
* An `AbstractDataType` that matches any concrete data types.
*/
@@ -114,15 +111,14 @@ protected[sql] object AnyDataType extends AbstractDataType with Serializable {
override private[sql] def acceptsType(other: DataType): Boolean = true
}
-
/**
- * An internal type used to represent everything that is not null, UDTs, arrays, structs, and maps.
+ * An internal type used to represent everything that is not null, UDTs, arrays, structs, and
+ * maps.
*/
protected[sql] abstract class AtomicType extends DataType
object AtomicType
-
/**
* Numeric data types.
*
@@ -131,7 +127,6 @@ object AtomicType
@Stable
abstract class NumericType extends AtomicType
-
private[spark] object NumericType extends AbstractDataType {
override private[spark] def defaultConcreteType: DataType = DoubleType
@@ -141,22 +136,19 @@ private[spark] object NumericType extends AbstractDataType {
other.isInstanceOf[NumericType]
}
-
private[sql] object IntegralType extends AbstractDataType {
override private[sql] def defaultConcreteType: DataType = IntegerType
override private[sql] def simpleString: String = "integral"
- override private[sql] def acceptsType(other: DataType): Boolean = other.isInstanceOf[IntegralType]
+ override private[sql] def acceptsType(other: DataType): Boolean =
+ other.isInstanceOf[IntegralType]
}
-
private[sql] abstract class IntegralType extends NumericType
-
private[sql] object FractionalType
-
private[sql] abstract class FractionalType extends NumericType
private[sql] object AnyTimestampType extends AbstractDataType with Serializable {
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/ArrayType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
index e5af472d90e25..fc32248b4baf3 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/ArrayType.scala
@@ -29,12 +29,14 @@ import org.apache.spark.sql.catalyst.util.StringConcat
*/
@Stable
object ArrayType extends AbstractDataType {
+
/**
* Construct a [[ArrayType]] object with the given element type. The `containsNull` is true.
*/
def apply(elementType: DataType): ArrayType = ArrayType(elementType, containsNull = true)
- override private[sql] def defaultConcreteType: DataType = ArrayType(NullType, containsNull = true)
+ override private[sql] def defaultConcreteType: DataType =
+ ArrayType(NullType, containsNull = true)
override private[sql] def acceptsType(other: DataType): Boolean = {
other.isInstanceOf[ArrayType]
@@ -44,18 +46,19 @@ object ArrayType extends AbstractDataType {
}
/**
- * The data type for collections of multiple values.
- * Internally these are represented as columns that contain a ``scala.collection.Seq``.
+ * The data type for collections of multiple values. Internally these are represented as columns
+ * that contain a ``scala.collection.Seq``.
*
* Please use `DataTypes.createArrayType()` to create a specific instance.
*
- * An [[ArrayType]] object comprises two fields, `elementType: [[DataType]]` and
- * `containsNull: Boolean`.
- * The field of `elementType` is used to specify the type of array elements.
- * The field of `containsNull` is used to specify if the array can have `null` values.
+ * An [[ArrayType]] object comprises two fields, `elementType: [[DataType]]` and `containsNull:
+ * Boolean`. The field of `elementType` is used to specify the type of array elements. The field
+ * of `containsNull` is used to specify if the array can have `null` values.
*
- * @param elementType The data type of values.
- * @param containsNull Indicates if the array can have `null` values
+ * @param elementType
+ * The data type of values.
+ * @param containsNull
+ * Indicates if the array can have `null` values
*
* @since 1.3.0
*/
@@ -82,8 +85,8 @@ case class ArrayType(elementType: DataType, containsNull: Boolean) extends DataT
("containsNull" -> containsNull)
/**
- * The default size of a value of the ArrayType is the default size of the element type.
- * We assume that there is only 1 element on average in an array. See SPARK-18853.
+ * The default size of a value of the ArrayType is the default size of the element type. We
+ * assume that there is only 1 element on average in an array. See SPARK-18853.
*/
override def defaultSize: Int = 1 * elementType.defaultSize
@@ -97,8 +100,8 @@ case class ArrayType(elementType: DataType, containsNull: Boolean) extends DataT
ArrayType(elementType.asNullable, containsNull = true)
/**
- * Returns the same data type but set all nullability fields are true
- * (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
+ * Returns the same data type but set all nullability fields are true (`StructField.nullable`,
+ * `ArrayType.containsNull`, and `MapType.valueContainsNull`).
*
* @since 4.0.0
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/BinaryType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
index c280f66f943aa..20bfd9bf5684f 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
@@ -20,11 +20,12 @@ package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
/**
- * The data type representing `Array[Byte]` values.
- * Please use the singleton `DataTypes.BinaryType`.
+ * The data type representing `Array[Byte]` values. Please use the singleton
+ * `DataTypes.BinaryType`.
*/
@Stable
-class BinaryType private() extends AtomicType {
+class BinaryType private () extends AtomicType {
+
/**
* The default size of a value of the BinaryType is 100 bytes.
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/BooleanType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
index 836c41a996ac4..090c56eaf9af7 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
@@ -25,7 +25,8 @@ import org.apache.spark.annotation.Stable
* @since 1.3.0
*/
@Stable
-class BooleanType private() extends AtomicType {
+class BooleanType private () extends AtomicType {
+
/**
* The default size of a value of the BooleanType is 1 byte.
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/ByteType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/ByteType.scala
index 546ac02f2639a..4a27a00dacb8a 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/ByteType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/ByteType.scala
@@ -25,7 +25,8 @@ import org.apache.spark.annotation.Stable
* @since 1.3.0
*/
@Stable
-class ByteType private() extends IntegralType {
+class ByteType private () extends IntegralType {
+
/**
* The default size of a value of the ByteType is 1 byte.
*/
@@ -36,7 +37,6 @@ class ByteType private() extends IntegralType {
private[spark] override def asNullable: ByteType = this
}
-
/**
* @since 1.3.0
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
index d506a1521e183..f6b6256db0417 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/CalendarIntervalType.scala
@@ -21,19 +21,19 @@ import org.apache.spark.annotation.Stable
/**
* The data type representing calendar intervals. The calendar interval is stored internally in
- * three components:
- * an integer value representing the number of `months` in this interval,
- * an integer value representing the number of `days` in this interval,
- * a long value representing the number of `microseconds` in this interval.
+ * three components: an integer value representing the number of `months` in this interval, an
+ * integer value representing the number of `days` in this interval, a long value representing the
+ * number of `microseconds` in this interval.
*
* Please use the singleton `DataTypes.CalendarIntervalType` to refer the type.
*
- * @note Calendar intervals are not comparable.
+ * @note
+ * Calendar intervals are not comparable.
*
* @since 1.5.0
*/
@Stable
-class CalendarIntervalType private() extends DataType {
+class CalendarIntervalType private () extends DataType {
override def defaultSize: Int = 16
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala
index 277d5c9458d6f..008c9cd07076c 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -49,6 +49,7 @@ import org.apache.spark.util.SparkClassUtils
@JsonSerialize(using = classOf[DataTypeJsonSerializer])
@JsonDeserialize(using = classOf[DataTypeJsonDeserializer])
abstract class DataType extends AbstractDataType {
+
/**
* The default size of a value of this data type, used internally for size estimation.
*/
@@ -57,7 +58,9 @@ abstract class DataType extends AbstractDataType {
/** Name of the type used in JSON serialization. */
def typeName: String = {
this.getClass.getSimpleName
- .stripSuffix("$").stripSuffix("Type").stripSuffix("UDT")
+ .stripSuffix("$")
+ .stripSuffix("Type")
+ .stripSuffix("UDT")
.toLowerCase(Locale.ROOT)
}
@@ -92,8 +95,8 @@ abstract class DataType extends AbstractDataType {
}
/**
- * Returns the same data type but set all nullability fields are true
- * (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
+ * Returns the same data type but set all nullability fields are true (`StructField.nullable`,
+ * `ArrayType.containsNull`, and `MapType.valueContainsNull`).
*/
private[spark] def asNullable: DataType
@@ -107,7 +110,6 @@ abstract class DataType extends AbstractDataType {
override private[sql] def acceptsType(other: DataType): Boolean = sameType(other)
}
-
/**
* @since 1.3.0
*/
@@ -128,14 +130,18 @@ object DataType {
}
/**
- * Parses data type from a string with schema. It calls `parser` for `schema`.
- * If it fails, calls `fallbackParser`. If the fallback function fails too, combines error message
- * from `parser` and `fallbackParser`.
+ * Parses data type from a string with schema. It calls `parser` for `schema`. If it fails,
+ * calls `fallbackParser`. If the fallback function fails too, combines error message from
+ * `parser` and `fallbackParser`.
*
- * @param schema The schema string to parse by `parser` or `fallbackParser`.
- * @param parser The function that should be invoke firstly.
- * @param fallbackParser The function that is called when `parser` fails.
- * @return The data type parsed from the `schema` schema.
+ * @param schema
+ * The schema string to parse by `parser` or `fallbackParser`.
+ * @param parser
+ * The function that should be invoke firstly.
+ * @param fallbackParser
+ * The function that is called when `parser` fails.
+ * @return
+ * The data type parsed from the `schema` schema.
*/
def parseTypeWithFallback(
schema: String,
@@ -161,8 +167,20 @@ object DataType {
def fromJson(json: String): DataType = parseDataType(parse(json))
private val otherTypes = {
- Seq(NullType, DateType, TimestampType, BinaryType, IntegerType, BooleanType, LongType,
- DoubleType, FloatType, ShortType, ByteType, StringType, CalendarIntervalType,
+ Seq(
+ NullType,
+ DateType,
+ TimestampType,
+ BinaryType,
+ IntegerType,
+ BooleanType,
+ LongType,
+ DoubleType,
+ FloatType,
+ ShortType,
+ ByteType,
+ StringType,
+ CalendarIntervalType,
DayTimeIntervalType(DAY),
DayTimeIntervalType(DAY, HOUR),
DayTimeIntervalType(DAY, MINUTE),
@@ -178,7 +196,8 @@ object DataType {
YearMonthIntervalType(YEAR, MONTH),
TimestampNTZType,
VariantType)
- .map(t => t.typeName -> t).toMap
+ .map(t => t.typeName -> t)
+ .toMap
}
/** Given the string representation of a type, return its DataType */
@@ -191,11 +210,12 @@ object DataType {
// For backwards compatibility, previously the type name of NullType is "null"
case "null" => NullType
case "timestamp_ltz" => TimestampType
- case other => otherTypes.getOrElse(
- other,
- throw new SparkIllegalArgumentException(
- errorClass = "INVALID_JSON_DATA_TYPE",
- messageParameters = Map("invalidType" -> name)))
+ case other =>
+ otherTypes.getOrElse(
+ other,
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_JSON_DATA_TYPE",
+ messageParameters = Map("invalidType" -> name)))
}
}
@@ -220,56 +240,55 @@ object DataType {
}
case JSortedObject(
- ("containsNull", JBool(n)),
- ("elementType", t: JValue),
- ("type", JString("array"))) =>
+ ("containsNull", JBool(n)),
+ ("elementType", t: JValue),
+ ("type", JString("array"))) =>
assertValidTypeForCollations(fieldPath, "array", collationsMap)
val elementType = parseDataType(t, appendFieldToPath(fieldPath, "element"), collationsMap)
ArrayType(elementType, n)
case JSortedObject(
- ("keyType", k: JValue),
- ("type", JString("map")),
- ("valueContainsNull", JBool(n)),
- ("valueType", v: JValue)) =>
+ ("keyType", k: JValue),
+ ("type", JString("map")),
+ ("valueContainsNull", JBool(n)),
+ ("valueType", v: JValue)) =>
assertValidTypeForCollations(fieldPath, "map", collationsMap)
val keyType = parseDataType(k, appendFieldToPath(fieldPath, "key"), collationsMap)
val valueType = parseDataType(v, appendFieldToPath(fieldPath, "value"), collationsMap)
MapType(keyType, valueType, n)
- case JSortedObject(
- ("fields", JArray(fields)),
- ("type", JString("struct"))) =>
+ case JSortedObject(("fields", JArray(fields)), ("type", JString("struct"))) =>
assertValidTypeForCollations(fieldPath, "struct", collationsMap)
StructType(fields.map(parseStructField))
// Scala/Java UDT
case JSortedObject(
- ("class", JString(udtClass)),
- ("pyClass", _),
- ("sqlType", _),
- ("type", JString("udt"))) =>
+ ("class", JString(udtClass)),
+ ("pyClass", _),
+ ("sqlType", _),
+ ("type", JString("udt"))) =>
SparkClassUtils.classForName[UserDefinedType[_]](udtClass).getConstructor().newInstance()
// Python UDT
case JSortedObject(
- ("pyClass", JString(pyClass)),
- ("serializedClass", JString(serialized)),
- ("sqlType", v: JValue),
- ("type", JString("udt"))) =>
- new PythonUserDefinedType(parseDataType(v), pyClass, serialized)
-
- case other => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_JSON_DATA_TYPE",
- messageParameters = Map("invalidType" -> compact(render(other))))
+ ("pyClass", JString(pyClass)),
+ ("serializedClass", JString(serialized)),
+ ("sqlType", v: JValue),
+ ("type", JString("udt"))) =>
+ new PythonUserDefinedType(parseDataType(v), pyClass, serialized)
+
+ case other =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_JSON_DATA_TYPE",
+ messageParameters = Map("invalidType" -> compact(render(other))))
}
private def parseStructField(json: JValue): StructField = json match {
case JSortedObject(
- ("metadata", JObject(metadataFields)),
- ("name", JString(name)),
- ("nullable", JBool(nullable)),
- ("type", dataType: JValue)) =>
+ ("metadata", JObject(metadataFields)),
+ ("name", JString(name)),
+ ("nullable", JBool(nullable)),
+ ("type", dataType: JValue)) =>
val collationsMap = getCollationsMap(metadataFields)
val metadataWithoutCollations =
JObject(metadataFields.filterNot(_._1 == COLLATIONS_METADATA_KEY))
@@ -280,18 +299,17 @@ object DataType {
Metadata.fromJObject(metadataWithoutCollations))
// Support reading schema when 'metadata' is missing.
case JSortedObject(
- ("name", JString(name)),
- ("nullable", JBool(nullable)),
- ("type", dataType: JValue)) =>
+ ("name", JString(name)),
+ ("nullable", JBool(nullable)),
+ ("type", dataType: JValue)) =>
StructField(name, parseDataType(dataType), nullable)
// Support reading schema when 'nullable' is missing.
- case JSortedObject(
- ("name", JString(name)),
- ("type", dataType: JValue)) =>
+ case JSortedObject(("name", JString(name)), ("type", dataType: JValue)) =>
StructField(name, parseDataType(dataType))
- case other => throw new SparkIllegalArgumentException(
- errorClass = "INVALID_JSON_DATA_TYPE",
- messageParameters = Map("invalidType" -> compact(render(other))))
+ case other =>
+ throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_JSON_DATA_TYPE",
+ messageParameters = Map("invalidType" -> compact(render(other))))
}
private def assertValidTypeForCollations(
@@ -319,13 +337,12 @@ object DataType {
val collationsJsonOpt = metadataFields.find(_._1 == COLLATIONS_METADATA_KEY).map(_._2)
collationsJsonOpt match {
case Some(JObject(fields)) =>
- fields.collect {
- case (fieldPath, JString(collation)) =>
- collation.split("\\.", 2) match {
- case Array(provider: String, collationName: String) =>
- CollationFactory.assertValidProvider(provider)
- fieldPath -> collationName
- }
+ fields.collect { case (fieldPath, JString(collation)) =>
+ collation.split("\\.", 2) match {
+ case Array(provider: String, collationName: String) =>
+ CollationFactory.assertValidProvider(provider)
+ fieldPath -> collationName
+ }
}.toMap
case _ => Map.empty
@@ -356,15 +373,15 @@ object DataType {
* Compares two types, ignoring compatible nullability of ArrayType, MapType, StructType.
*
* Compatible nullability is defined as follows:
- * - If `from` and `to` are ArrayTypes, `from` has a compatible nullability with `to`
- * if and only if `to.containsNull` is true, or both of `from.containsNull` and
- * `to.containsNull` are false.
- * - If `from` and `to` are MapTypes, `from` has a compatible nullability with `to`
- * if and only if `to.valueContainsNull` is true, or both of `from.valueContainsNull` and
- * `to.valueContainsNull` are false.
- * - If `from` and `to` are StructTypes, `from` has a compatible nullability with `to`
- * if and only if for all every pair of fields, `to.nullable` is true, or both
- * of `fromField.nullable` and `toField.nullable` are false.
+ * - If `from` and `to` are ArrayTypes, `from` has a compatible nullability with `to` if and
+ * only if `to.containsNull` is true, or both of `from.containsNull` and `to.containsNull`
+ * are false.
+ * - If `from` and `to` are MapTypes, `from` has a compatible nullability with `to` if and
+ * only if `to.valueContainsNull` is true, or both of `from.valueContainsNull` and
+ * `to.valueContainsNull` are false.
+ * - If `from` and `to` are StructTypes, `from` has a compatible nullability with `to` if and
+ * only if for all every pair of fields, `to.nullable` is true, or both of
+ * `fromField.nullable` and `toField.nullable` are false.
*/
private[sql] def equalsIgnoreCompatibleNullability(from: DataType, to: DataType): Boolean = {
equalsIgnoreCompatibleNullability(from, to, ignoreName = false)
@@ -375,15 +392,15 @@ object DataType {
* also the field name. It compares based on the position.
*
* Compatible nullability is defined as follows:
- * - If `from` and `to` are ArrayTypes, `from` has a compatible nullability with `to`
- * if and only if `to.containsNull` is true, or both of `from.containsNull` and
- * `to.containsNull` are false.
- * - If `from` and `to` are MapTypes, `from` has a compatible nullability with `to`
- * if and only if `to.valueContainsNull` is true, or both of `from.valueContainsNull` and
- * `to.valueContainsNull` are false.
- * - If `from` and `to` are StructTypes, `from` has a compatible nullability with `to`
- * if and only if for all every pair of fields, `to.nullable` is true, or both
- * of `fromField.nullable` and `toField.nullable` are false.
+ * - If `from` and `to` are ArrayTypes, `from` has a compatible nullability with `to` if and
+ * only if `to.containsNull` is true, or both of `from.containsNull` and `to.containsNull`
+ * are false.
+ * - If `from` and `to` are MapTypes, `from` has a compatible nullability with `to` if and
+ * only if `to.valueContainsNull` is true, or both of `from.valueContainsNull` and
+ * `to.valueContainsNull` are false.
+ * - If `from` and `to` are StructTypes, `from` has a compatible nullability with `to` if and
+ * only if for all every pair of fields, `to.nullable` is true, or both of
+ * `fromField.nullable` and `toField.nullable` are false.
*/
private[sql] def equalsIgnoreNameAndCompatibleNullability(
from: DataType,
@@ -401,16 +418,16 @@ object DataType {
case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) =>
(tn || !fn) &&
- equalsIgnoreCompatibleNullability(fromKey, toKey, ignoreName) &&
- equalsIgnoreCompatibleNullability(fromValue, toValue, ignoreName)
+ equalsIgnoreCompatibleNullability(fromKey, toKey, ignoreName) &&
+ equalsIgnoreCompatibleNullability(fromValue, toValue, ignoreName)
case (StructType(fromFields), StructType(toFields)) =>
fromFields.length == toFields.length &&
- fromFields.zip(toFields).forall { case (fromField, toField) =>
- (ignoreName || fromField.name == toField.name) &&
- (toField.nullable || !fromField.nullable) &&
- equalsIgnoreCompatibleNullability(fromField.dataType, toField.dataType, ignoreName)
- }
+ fromFields.zip(toFields).forall { case (fromField, toField) =>
+ (ignoreName || fromField.name == toField.name) &&
+ (toField.nullable || !fromField.nullable) &&
+ equalsIgnoreCompatibleNullability(fromField.dataType, toField.dataType, ignoreName)
+ }
case (fromDataType, toDataType) => fromDataType == toDataType
}
@@ -420,42 +437,42 @@ object DataType {
* Check if `from` is equal to `to` type except for collations, which are checked to be
* compatible so that data of type `from` can be interpreted as of type `to`.
*/
- private[sql] def equalsIgnoreCompatibleCollation(
- from: DataType,
- to: DataType): Boolean = {
+ private[sql] def equalsIgnoreCompatibleCollation(from: DataType, to: DataType): Boolean = {
(from, to) match {
// String types with possibly different collations are compatible.
case (_: StringType, _: StringType) => true
case (ArrayType(fromElement, fromContainsNull), ArrayType(toElement, toContainsNull)) =>
(fromContainsNull == toContainsNull) &&
- equalsIgnoreCompatibleCollation(fromElement, toElement)
+ equalsIgnoreCompatibleCollation(fromElement, toElement)
- case (MapType(fromKey, fromValue, fromContainsNull),
- MapType(toKey, toValue, toContainsNull)) =>
+ case (
+ MapType(fromKey, fromValue, fromContainsNull),
+ MapType(toKey, toValue, toContainsNull)) =>
fromContainsNull == toContainsNull &&
- // Map keys cannot change collation.
- fromKey == toKey &&
- equalsIgnoreCompatibleCollation(fromValue, toValue)
+ // Map keys cannot change collation.
+ fromKey == toKey &&
+ equalsIgnoreCompatibleCollation(fromValue, toValue)
case (StructType(fromFields), StructType(toFields)) =>
fromFields.length == toFields.length &&
- fromFields.zip(toFields).forall { case (fromField, toField) =>
- fromField.name == toField.name &&
- fromField.nullable == toField.nullable &&
- fromField.metadata == toField.metadata &&
- equalsIgnoreCompatibleCollation(fromField.dataType, toField.dataType)
- }
+ fromFields.zip(toFields).forall { case (fromField, toField) =>
+ fromField.name == toField.name &&
+ fromField.nullable == toField.nullable &&
+ fromField.metadata == toField.metadata &&
+ equalsIgnoreCompatibleCollation(fromField.dataType, toField.dataType)
+ }
case (fromDataType, toDataType) => fromDataType == toDataType
}
}
/**
- * Returns true if the two data types share the same "shape", i.e. the types
- * are the same, but the field names don't need to be the same.
+ * Returns true if the two data types share the same "shape", i.e. the types are the same, but
+ * the field names don't need to be the same.
*
- * @param ignoreNullability whether to ignore nullability when comparing the types
+ * @param ignoreNullability
+ * whether to ignore nullability when comparing the types
*/
def equalsStructurally(
from: DataType,
@@ -464,20 +481,21 @@ object DataType {
(from, to) match {
case (left: ArrayType, right: ArrayType) =>
equalsStructurally(left.elementType, right.elementType, ignoreNullability) &&
- (ignoreNullability || left.containsNull == right.containsNull)
+ (ignoreNullability || left.containsNull == right.containsNull)
case (left: MapType, right: MapType) =>
equalsStructurally(left.keyType, right.keyType, ignoreNullability) &&
- equalsStructurally(left.valueType, right.valueType, ignoreNullability) &&
- (ignoreNullability || left.valueContainsNull == right.valueContainsNull)
+ equalsStructurally(left.valueType, right.valueType, ignoreNullability) &&
+ (ignoreNullability || left.valueContainsNull == right.valueContainsNull)
case (StructType(fromFields), StructType(toFields)) =>
fromFields.length == toFields.length &&
- fromFields.zip(toFields)
- .forall { case (l, r) =>
- equalsStructurally(l.dataType, r.dataType, ignoreNullability) &&
- (ignoreNullability || l.nullable == r.nullable)
- }
+ fromFields
+ .zip(toFields)
+ .forall { case (l, r) =>
+ equalsStructurally(l.dataType, r.dataType, ignoreNullability) &&
+ (ignoreNullability || l.nullable == r.nullable)
+ }
case (fromDataType, toDataType) => fromDataType == toDataType
}
@@ -496,14 +514,15 @@ object DataType {
case (left: MapType, right: MapType) =>
equalsStructurallyByName(left.keyType, right.keyType, resolver) &&
- equalsStructurallyByName(left.valueType, right.valueType, resolver)
+ equalsStructurallyByName(left.valueType, right.valueType, resolver)
case (StructType(fromFields), StructType(toFields)) =>
fromFields.length == toFields.length &&
- fromFields.zip(toFields)
- .forall { case (l, r) =>
- resolver(l.name, r.name) && equalsStructurallyByName(l.dataType, r.dataType, resolver)
- }
+ fromFields
+ .zip(toFields)
+ .forall { case (l, r) =>
+ resolver(l.name, r.name) && equalsStructurallyByName(l.dataType, r.dataType, resolver)
+ }
case _ => true
}
@@ -518,12 +537,12 @@ object DataType {
equalsIgnoreNullability(leftElementType, rightElementType)
case (MapType(leftKeyType, leftValueType, _), MapType(rightKeyType, rightValueType, _)) =>
equalsIgnoreNullability(leftKeyType, rightKeyType) &&
- equalsIgnoreNullability(leftValueType, rightValueType)
+ equalsIgnoreNullability(leftValueType, rightValueType)
case (StructType(leftFields), StructType(rightFields)) =>
leftFields.length == rightFields.length &&
- leftFields.zip(rightFields).forall { case (l, r) =>
- l.name == r.name && equalsIgnoreNullability(l.dataType, r.dataType)
- }
+ leftFields.zip(rightFields).forall { case (l, r) =>
+ l.name == r.name && equalsIgnoreNullability(l.dataType, r.dataType)
+ }
case (l, r) => l == r
}
}
@@ -539,14 +558,14 @@ object DataType {
case (MapType(fromKey, fromValue, _), MapType(toKey, toValue, _)) =>
equalsIgnoreCaseAndNullability(fromKey, toKey) &&
- equalsIgnoreCaseAndNullability(fromValue, toValue)
+ equalsIgnoreCaseAndNullability(fromValue, toValue)
case (StructType(fromFields), StructType(toFields)) =>
fromFields.length == toFields.length &&
- fromFields.zip(toFields).forall { case (l, r) =>
- l.name.equalsIgnoreCase(r.name) &&
- equalsIgnoreCaseAndNullability(l.dataType, r.dataType)
- }
+ fromFields.zip(toFields).forall { case (l, r) =>
+ l.name.equalsIgnoreCase(r.name) &&
+ equalsIgnoreCaseAndNullability(l.dataType, r.dataType)
+ }
case (fromDataType, toDataType) => fromDataType == toDataType
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/DateType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/DateType.scala
index d37ebbcdad727..402c4c0d95298 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/DateType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/DateType.scala
@@ -20,14 +20,15 @@ package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
/**
- * The date type represents a valid date in the proleptic Gregorian calendar.
- * Valid range is [0001-01-01, 9999-12-31].
+ * The date type represents a valid date in the proleptic Gregorian calendar. Valid range is
+ * [0001-01-01, 9999-12-31].
*
* Please use the singleton `DataTypes.DateType` to refer the type.
* @since 1.3.0
*/
@Stable
-class DateType private() extends DatetimeType {
+class DateType private () extends DatetimeType {
+
/**
* The default size of a value of the DateType is 4 bytes.
*/
@@ -37,10 +38,10 @@ class DateType private() extends DatetimeType {
}
/**
- * The companion case object and the DateType class is separated so the companion object
- * also subclasses the class. Otherwise, the companion object would be of type "DateType$"
- * in byte code. The DateType class is defined with a private constructor so its companion
- * object is the only possible instantiation.
+ * The companion case object and the DateType class is separated so the companion object also
+ * subclasses the class. Otherwise, the companion object would be of type "DateType$" in byte
+ * code. The DateType class is defined with a private constructor so its companion object is the
+ * only possible instantiation.
*
* @since 1.3.0
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
index a1d014fa51f1c..90d6d7c29a6ba 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
@@ -22,8 +22,8 @@ import org.apache.spark.sql.errors.DataTypeErrors
import org.apache.spark.sql.types.DayTimeIntervalType.fieldToString
/**
- * The type represents day-time intervals of the SQL standard. A day-time interval is made up
- * of a contiguous subset of the following fields:
+ * The type represents day-time intervals of the SQL standard. A day-time interval is made up of a
+ * contiguous subset of the following fields:
* - SECOND, seconds within minutes and possibly fractions of a second [0..59.999999],
* - MINUTE, minutes within hours [0..59],
* - HOUR, hours within days [0..23],
@@ -31,18 +31,21 @@ import org.apache.spark.sql.types.DayTimeIntervalType.fieldToString
*
* `DayTimeIntervalType` represents positive as well as negative day-time intervals.
*
- * @param startField The leftmost field which the type comprises of. Valid values:
- * 0 (DAY), 1 (HOUR), 2 (MINUTE), 3 (SECOND).
- * @param endField The rightmost field which the type comprises of. Valid values:
- * 0 (DAY), 1 (HOUR), 2 (MINUTE), 3 (SECOND).
+ * @param startField
+ * The leftmost field which the type comprises of. Valid values: 0 (DAY), 1 (HOUR), 2 (MINUTE),
+ * 3 (SECOND).
+ * @param endField
+ * The rightmost field which the type comprises of. Valid values: 0 (DAY), 1 (HOUR), 2 (MINUTE),
+ * 3 (SECOND).
*
* @since 3.2.0
*/
@Unstable
case class DayTimeIntervalType(startField: Byte, endField: Byte) extends AnsiIntervalType {
+
/**
- * The day-time interval type has constant precision. A value of the type always occupies 8 bytes.
- * The DAY field is constrained by the upper bound 106751991 to fit to `Long`.
+ * The day-time interval type has constant precision. A value of the type always occupies 8
+ * bytes. The DAY field is constrained by the upper bound 106751991 to fit to `Long`.
*/
override def defaultSize: Int = 8
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 6de8570b1422f..bd94c386ab533 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -31,9 +31,9 @@ import org.apache.spark.unsafe.types.UTF8String
* A mutable implementation of BigDecimal that can hold a Long if values are small enough.
*
* The semantics of the fields are as follows:
- * - _precision and _scale represent the SQL precision and scale we are looking for
- * - If decimalVal is set, it represents the whole decimal value
- * - Otherwise, the decimal value is longVal / (10 ** _scale)
+ * - _precision and _scale represent the SQL precision and scale we are looking for
+ * - If decimalVal is set, it represents the whole decimal value
+ * - Otherwise, the decimal value is longVal / (10 ** _scale)
*
* Note, for values between -1.0 and 1.0, precision digits are only counted after dot.
*/
@@ -88,22 +88,22 @@ final class Decimal extends Ordered[Decimal] with Serializable {
}
/**
- * Set this Decimal to the given unscaled Long, with a given precision and scale,
- * and return it, or return null if it cannot be set due to overflow.
+ * Set this Decimal to the given unscaled Long, with a given precision and scale, and return it,
+ * or return null if it cannot be set due to overflow.
*/
def setOrNull(unscaled: Long, precision: Int, scale: Int): Decimal = {
DecimalType.checkNegativeScale(scale)
if (unscaled <= -POW_10(MAX_LONG_DIGITS) || unscaled >= POW_10(MAX_LONG_DIGITS)) {
// We can't represent this compactly as a long without risking overflow
if (precision < 19) {
- return null // Requested precision is too low to represent this value
+ return null // Requested precision is too low to represent this value
}
this.decimalVal = BigDecimal(unscaled, scale)
this.longVal = 0L
} else {
val p = POW_10(math.min(precision, MAX_LONG_DIGITS))
if (unscaled <= -p || unscaled >= p) {
- return null // Requested precision is too low to represent this value
+ return null // Requested precision is too low to represent this value
}
this.decimalVal = null
this.longVal = unscaled
@@ -126,7 +126,8 @@ final class Decimal extends Ordered[Decimal] with Serializable {
"roundedValue" -> decimalVal.toString,
"originalValue" -> decimal.toString,
"precision" -> precision.toString,
- "scale" -> scale.toString), Array.empty)
+ "scale" -> scale.toString),
+ Array.empty)
}
this.longVal = 0L
this._precision = precision
@@ -160,8 +161,8 @@ final class Decimal extends Ordered[Decimal] with Serializable {
}
/**
- * If the value is not in the range of long, convert it to BigDecimal and
- * the precision and scale are based on the converted value.
+ * If the value is not in the range of long, convert it to BigDecimal and the precision and
+ * scale are based on the converted value.
*
* This code avoids BigDecimal object allocation as possible to improve runtime efficiency
*/
@@ -262,37 +263,47 @@ final class Decimal extends Ordered[Decimal] with Serializable {
def toByte: Byte = toLong.toByte
/**
- * @return the Byte value that is equal to the rounded decimal.
- * @throws ArithmeticException if the decimal is too big to fit in Byte type.
+ * @return
+ * the Byte value that is equal to the rounded decimal.
+ * @throws ArithmeticException
+ * if the decimal is too big to fit in Byte type.
*/
private[sql] def roundToByte(): Byte =
- roundToNumeric[Byte](ByteType, Byte.MaxValue, Byte.MinValue) (_.toByte) (_.toByte)
+ roundToNumeric[Byte](ByteType, Byte.MaxValue, Byte.MinValue)(_.toByte)(_.toByte)
/**
- * @return the Short value that is equal to the rounded decimal.
- * @throws ArithmeticException if the decimal is too big to fit in Short type.
+ * @return
+ * the Short value that is equal to the rounded decimal.
+ * @throws ArithmeticException
+ * if the decimal is too big to fit in Short type.
*/
private[sql] def roundToShort(): Short =
- roundToNumeric[Short](ShortType, Short.MaxValue, Short.MinValue) (_.toShort) (_.toShort)
+ roundToNumeric[Short](ShortType, Short.MaxValue, Short.MinValue)(_.toShort)(_.toShort)
/**
- * @return the Int value that is equal to the rounded decimal.
- * @throws ArithmeticException if the decimal too big to fit in Int type.
+ * @return
+ * the Int value that is equal to the rounded decimal.
+ * @throws ArithmeticException
+ * if the decimal too big to fit in Int type.
*/
private[sql] def roundToInt(): Int =
- roundToNumeric[Int](IntegerType, Int.MaxValue, Int.MinValue) (_.toInt) (_.toInt)
+ roundToNumeric[Int](IntegerType, Int.MaxValue, Int.MinValue)(_.toInt)(_.toInt)
private def toSqlValue: String = this.toString + "BD"
- private def roundToNumeric[T <: AnyVal](integralType: IntegralType, maxValue: Int, minValue: Int)
- (f1: Long => T) (f2: Double => T): T = {
+ private def roundToNumeric[T <: AnyVal](
+ integralType: IntegralType,
+ maxValue: Int,
+ minValue: Int)(f1: Long => T)(f2: Double => T): T = {
if (decimalVal.eq(null)) {
val numericVal = f1(actualLongVal)
if (actualLongVal == numericVal) {
numericVal
} else {
throw DataTypeErrors.castingCauseOverflowError(
- toSqlValue, DecimalType(this.precision, this.scale), integralType)
+ toSqlValue,
+ DecimalType(this.precision, this.scale),
+ integralType)
}
} else {
val doubleVal = decimalVal.toDouble
@@ -300,14 +311,18 @@ final class Decimal extends Ordered[Decimal] with Serializable {
f2(doubleVal)
} else {
throw DataTypeErrors.castingCauseOverflowError(
- toSqlValue, DecimalType(this.precision, this.scale), integralType)
+ toSqlValue,
+ DecimalType(this.precision, this.scale),
+ integralType)
}
}
}
/**
- * @return the Long value that is equal to the rounded decimal.
- * @throws ArithmeticException if the decimal too big to fit in Long type.
+ * @return
+ * the Long value that is equal to the rounded decimal.
+ * @throws ArithmeticException
+ * if the decimal too big to fit in Long type.
*/
private[sql] def roundToLong(): Long = {
if (decimalVal.eq(null)) {
@@ -321,7 +336,9 @@ final class Decimal extends Ordered[Decimal] with Serializable {
} catch {
case _: ArithmeticException =>
throw DataTypeErrors.castingCauseOverflowError(
- toSqlValue, DecimalType(this.precision, this.scale), LongType)
+ toSqlValue,
+ DecimalType(this.precision, this.scale),
+ LongType)
}
}
}
@@ -329,7 +346,8 @@ final class Decimal extends Ordered[Decimal] with Serializable {
/**
* Update precision and scale while keeping our value the same, and return true if successful.
*
- * @return true if successful, false if overflow would occur
+ * @return
+ * true if successful, false if overflow would occur
*/
def changePrecision(precision: Int, scale: Int): Boolean = {
changePrecision(precision, scale, ROUND_HALF_UP)
@@ -338,8 +356,9 @@ final class Decimal extends Ordered[Decimal] with Serializable {
/**
* Create new `Decimal` with given precision and scale.
*
- * @return a non-null `Decimal` value if successful. Otherwise, if `nullOnOverflow` is true, null
- * is returned; if `nullOnOverflow` is false, an `ArithmeticException` is thrown.
+ * @return
+ * a non-null `Decimal` value if successful. Otherwise, if `nullOnOverflow` is true, null is
+ * returned; if `nullOnOverflow` is false, an `ArithmeticException` is thrown.
*/
private[sql] def toPrecision(
precision: Int,
@@ -354,8 +373,7 @@ final class Decimal extends Ordered[Decimal] with Serializable {
if (nullOnOverflow) {
null
} else {
- throw DataTypeErrors.cannotChangeDecimalPrecisionError(
- this, precision, scale, context)
+ throw DataTypeErrors.cannotChangeDecimalPrecisionError(this, precision, scale, context)
}
}
}
@@ -363,7 +381,8 @@ final class Decimal extends Ordered[Decimal] with Serializable {
/**
* Update precision and scale while keeping our value the same, and return true if successful.
*
- * @return true if successful, false if overflow would occur
+ * @return
+ * true if successful, false if overflow would occur
*/
private[sql] def changePrecision(
precision: Int,
@@ -482,7 +501,7 @@ final class Decimal extends Ordered[Decimal] with Serializable {
// ------------------------------------------------------------------------
// e1 + e2 max(s1, s2) + max(p1-s1, p2-s2) + 1 max(s1, s2)
// e1 - e2 max(s1, s2) + max(p1-s1, p2-s2) + 1 max(s1, s2)
- def + (that: Decimal): Decimal = {
+ def +(that: Decimal): Decimal = {
if (decimalVal.eq(null) && that.decimalVal.eq(null) && scale == that.scale) {
Decimal(longVal + that.longVal, Math.max(precision, that.precision) + 1, scale)
} else {
@@ -490,7 +509,7 @@ final class Decimal extends Ordered[Decimal] with Serializable {
}
}
- def - (that: Decimal): Decimal = {
+ def -(that: Decimal): Decimal = {
if (decimalVal.eq(null) && that.decimalVal.eq(null) && scale == that.scale) {
Decimal(longVal - that.longVal, Math.max(precision, that.precision) + 1, scale)
} else {
@@ -499,14 +518,19 @@ final class Decimal extends Ordered[Decimal] with Serializable {
}
// TypeCoercion will take care of the precision, scale of result
- def * (that: Decimal): Decimal =
+ def *(that: Decimal): Decimal =
Decimal(toJavaBigDecimal.multiply(that.toJavaBigDecimal, MATH_CONTEXT))
- def / (that: Decimal): Decimal =
- if (that.isZero) null else Decimal(toJavaBigDecimal.divide(that.toJavaBigDecimal,
- DecimalType.MAX_SCALE + 1, MATH_CONTEXT.getRoundingMode))
+ def /(that: Decimal): Decimal =
+ if (that.isZero) {
+ null
+ } else {
+ Decimal(
+ toJavaBigDecimal
+ .divide(that.toJavaBigDecimal, DecimalType.MAX_SCALE + 1, MATH_CONTEXT.getRoundingMode))
+ }
- def % (that: Decimal): Decimal =
+ def %(that: Decimal): Decimal =
if (that.isZero) null
else Decimal(toJavaBigDecimal.remainder(that.toJavaBigDecimal, MATH_CONTEXT))
@@ -526,12 +550,14 @@ final class Decimal extends Ordered[Decimal] with Serializable {
def abs: Decimal = if (this < Decimal.ZERO) this.unary_- else this
- def floor: Decimal = if (scale == 0) this else {
+ def floor: Decimal = if (scale == 0) this
+ else {
val newPrecision = DecimalType.bounded(precision - scale + 1, 0).precision
toPrecision(newPrecision, 0, ROUND_FLOOR, nullOnOverflow = false)
}
- def ceil: Decimal = if (scale == 0) this else {
+ def ceil: Decimal = if (scale == 0) this
+ else {
val newPrecision = DecimalType.bounded(precision - scale + 1, 0).precision
toPrecision(newPrecision, 0, ROUND_CEILING, nullOnOverflow = false)
}
@@ -612,7 +638,7 @@ object Decimal {
// We fast fail because constructing a very large JavaBigDecimal to Decimal is very slow.
// For example: Decimal("6.0790316E+25569151")
if (numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION &&
- !SqlApiConf.get.allowNegativeScaleOfDecimalEnabled) {
+ !SqlApiConf.get.allowNegativeScaleOfDecimalEnabled) {
null
} else {
Decimal(bigDecimal)
@@ -632,7 +658,7 @@ object Decimal {
// We fast fail because constructing a very large JavaBigDecimal to Decimal is very slow.
// For example: Decimal("6.0790316E+25569151")
if (numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION &&
- !SqlApiConf.get.allowNegativeScaleOfDecimalEnabled) {
+ !SqlApiConf.get.allowNegativeScaleOfDecimalEnabled) {
throw DataTypeErrors.outOfDecimalTypeRangeError(str)
} else {
Decimal(bigDecimal)
@@ -657,16 +683,18 @@ object Decimal {
// Max precision of a decimal value stored in `numBytes` bytes
def maxPrecisionForBytes(numBytes: Int): Int = {
- Math.round( // convert double to long
- Math.floor(Math.log10( // number of base-10 digits
- Math.pow(2, 8 * numBytes - 1) - 1))) // max value stored in numBytes
+ Math
+ .round( // convert double to long
+ Math.floor(Math.log10( // number of base-10 digits
+ Math.pow(2, 8 * numBytes - 1) - 1))
+ ) // max value stored in numBytes
.asInstanceOf[Int]
}
// Returns the minimum number of bytes needed to store a decimal with a given `precision`.
lazy val minBytesForPrecision = Array.tabulate[Int](39)(computeMinBytesForPrecision)
- private def computeMinBytesForPrecision(precision : Int) : Int = {
+ private def computeMinBytesForPrecision(precision: Int): Int = {
var numBytes = 1
while (math.pow(2.0, 8 * numBytes - 1) < math.pow(10.0, precision)) {
numBytes += 1
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
index 9de34d0b3bc16..bff483cefda91 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
@@ -26,9 +26,8 @@ import org.apache.spark.sql.errors.DataTypeErrors
import org.apache.spark.sql.internal.SqlApiConf
/**
- * The data type representing `java.math.BigDecimal` values.
- * A Decimal that must have fixed precision (the maximum number of digits) and scale (the number
- * of digits on right side of dot).
+ * The data type representing `java.math.BigDecimal` values. A Decimal that must have fixed
+ * precision (the maximum number of digits) and scale (the number of digits on right side of dot).
*
* The precision can be up to 38, scale can also be up to 38 (less or equal to precision).
*
@@ -49,7 +48,8 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
if (precision > DecimalType.MAX_PRECISION) {
throw DataTypeErrors.decimalPrecisionExceedsMaxPrecisionError(
- precision, DecimalType.MAX_PRECISION)
+ precision,
+ DecimalType.MAX_PRECISION)
}
// default constructor for Java
@@ -63,8 +63,8 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
override def sql: String = typeName.toUpperCase(Locale.ROOT)
/**
- * Returns whether this DecimalType is wider than `other`. If yes, it means `other`
- * can be casted into `this` safely without losing any precision or range.
+ * Returns whether this DecimalType is wider than `other`. If yes, it means `other` can be
+ * casted into `this` safely without losing any precision or range.
*/
private[sql] def isWiderThan(other: DataType): Boolean = isWiderThanInternal(other)
@@ -78,8 +78,8 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
}
/**
- * Returns whether this DecimalType is tighter than `other`. If yes, it means `this`
- * can be casted into `other` safely without losing any precision or range.
+ * Returns whether this DecimalType is tighter than `other`. If yes, it means `this` can be
+ * casted into `other` safely without losing any precision or range.
*/
private[sql] def isTighterThan(other: DataType): Boolean = other match {
case dt: DecimalType =>
@@ -94,8 +94,8 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
}
/**
- * The default size of a value of the DecimalType is 8 bytes when precision is at most 18,
- * and 16 bytes otherwise.
+ * The default size of a value of the DecimalType is 8 bytes when precision is at most 18, and
+ * 16 bytes otherwise.
*/
override def defaultSize: Int = if (precision <= Decimal.MAX_LONG_DIGITS) 8 else 16
@@ -104,7 +104,6 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
private[spark] override def asNullable: DecimalType = this
}
-
/**
* Extra factory methods and pattern matchers for Decimals.
*
@@ -167,10 +166,11 @@ object DecimalType extends AbstractDataType {
/**
* Scale adjustment implementation is based on Hive's one, which is itself inspired to
* SQLServer's one. In particular, when a result precision is greater than
- * {@link #MAX_PRECISION}, the corresponding scale is reduced to prevent the integral part of a
+ * {@link #MAX_PRECISION} , the corresponding scale is reduced to prevent the integral part of a
* result from being truncated.
*
- * This method is used only when `spark.sql.decimalOperations.allowPrecisionLoss` is set to true.
+ * This method is used only when `spark.sql.decimalOperations.allowPrecisionLoss` is set to
+ * true.
*/
private[sql] def adjustPrecisionScale(precision: Int, scale: Int): DecimalType = {
// Assumptions:
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/DoubleType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
index bc0ed725cf266..873f0c237c6c4 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
@@ -27,7 +27,8 @@ import org.apache.spark.annotation.Stable
* @since 1.3.0
*/
@Stable
-class DoubleType private() extends FractionalType {
+class DoubleType private () extends FractionalType {
+
/**
* The default size of a value of the DoubleType is 8 bytes.
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/FloatType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/FloatType.scala
index 8b54f830d48a3..df4b03cd42bd4 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/FloatType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/FloatType.scala
@@ -27,7 +27,8 @@ import org.apache.spark.annotation.Stable
* @since 1.3.0
*/
@Stable
-class FloatType private() extends FractionalType {
+class FloatType private () extends FractionalType {
+
/**
* The default size of a value of the FloatType is 4 bytes.
*/
@@ -36,7 +37,6 @@ class FloatType private() extends FractionalType {
private[spark] override def asNullable: FloatType = this
}
-
/**
* @since 1.3.0
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/IntegerType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
index b26a555c9b572..dc4727cb1215b 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
@@ -25,7 +25,8 @@ import org.apache.spark.annotation.Stable
* @since 1.3.0
*/
@Stable
-class IntegerType private() extends IntegralType {
+class IntegerType private () extends IntegralType {
+
/**
* The default size of a value of the IntegerType is 4 bytes.
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/LongType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/LongType.scala
index 87ebacfe9ce88..f65c4c70acd27 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/LongType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/LongType.scala
@@ -25,7 +25,8 @@ import org.apache.spark.annotation.Stable
* @since 1.3.0
*/
@Stable
-class LongType private() extends IntegralType {
+class LongType private () extends IntegralType {
+
/**
* The default size of a value of the LongType is 8 bytes.
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/MapType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/MapType.scala
index dba870466fc1c..1dfb9aaf9e29b 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/MapType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/MapType.scala
@@ -28,15 +28,16 @@ import org.apache.spark.sql.catalyst.util.StringConcat
*
* Please use `DataTypes.createMapType()` to create a specific instance.
*
- * @param keyType The data type of map keys.
- * @param valueType The data type of map values.
- * @param valueContainsNull Indicates if map values have `null` values.
+ * @param keyType
+ * The data type of map keys.
+ * @param valueType
+ * The data type of map values.
+ * @param valueContainsNull
+ * Indicates if map values have `null` values.
*/
@Stable
-case class MapType(
- keyType: DataType,
- valueType: DataType,
- valueContainsNull: Boolean) extends DataType {
+case class MapType(keyType: DataType, valueType: DataType, valueContainsNull: Boolean)
+ extends DataType {
/** No-arg constructor for kryo. */
def this() = this(null, null, false)
@@ -48,8 +49,9 @@ case class MapType(
if (maxDepth > 0) {
stringConcat.append(s"$prefix-- key: ${keyType.typeName}\n")
DataType.buildFormattedString(keyType, s"$prefix |", stringConcat, maxDepth)
- stringConcat.append(s"$prefix-- value: ${valueType.typeName} " +
- s"(valueContainsNull = $valueContainsNull)\n")
+ stringConcat.append(
+ s"$prefix-- value: ${valueType.typeName} " +
+ s"(valueContainsNull = $valueContainsNull)\n")
DataType.buildFormattedString(valueType, s"$prefix |", stringConcat, maxDepth)
}
}
@@ -61,9 +63,9 @@ case class MapType(
("valueContainsNull" -> valueContainsNull)
/**
- * The default size of a value of the MapType is
- * (the default size of the key type + the default size of the value type).
- * We assume that there is only 1 element on average in a map. See SPARK-18853.
+ * The default size of a value of the MapType is (the default size of the key type + the default
+ * size of the value type). We assume that there is only 1 element on average in a map. See
+ * SPARK-18853.
*/
override def defaultSize: Int = 1 * (keyType.defaultSize + valueType.defaultSize)
@@ -77,8 +79,8 @@ case class MapType(
MapType(keyType.asNullable, valueType.asNullable, valueContainsNull = true)
/**
- * Returns the same data type but set all nullability fields are true
- * (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
+ * Returns the same data type but set all nullability fields are true (`StructField.nullable`,
+ * `ArrayType.containsNull`, and `MapType.valueContainsNull`).
*
* @since 4.0.0
*/
@@ -104,8 +106,8 @@ object MapType extends AbstractDataType {
override private[sql] def simpleString: String = "map"
/**
- * Construct a [[MapType]] object with the given key type and value type.
- * The `valueContainsNull` is true.
+ * Construct a [[MapType]] object with the given key type and value type. The
+ * `valueContainsNull` is true.
*/
def apply(keyType: DataType, valueType: DataType): MapType =
MapType(keyType: DataType, valueType: DataType, valueContainsNull = true)
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/Metadata.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/Metadata.scala
index 70e03905d4b05..05f91b5ba2313 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/Metadata.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/Metadata.scala
@@ -26,7 +26,6 @@ import org.apache.spark.annotation.Stable
import org.apache.spark.sql.errors.DataTypeErrors
import org.apache.spark.util.ArrayImplicits._
-
/**
* Metadata is a wrapper over Map[String, Any] that limits the value type to simple ones: Boolean,
* Long, Double, String, Metadata, Array[Boolean], Array[Long], Array[Double], Array[String], and
@@ -35,13 +34,14 @@ import org.apache.spark.util.ArrayImplicits._
* The default constructor is private. User should use either [[MetadataBuilder]] or
* `Metadata.fromJson()` to create Metadata instances.
*
- * @param map an immutable map that stores the data
+ * @param map
+ * an immutable map that stores the data
*
* @since 1.3.0
*/
@Stable
sealed class Metadata private[types] (private[types] val map: Map[String, Any])
- extends Serializable {
+ extends Serializable {
/** No-arg constructor for kryo. */
protected def this() = this(null)
@@ -173,7 +173,8 @@ object Metadata {
builder.putStringArray(key, value.asInstanceOf[List[JString]].map(_.s).toArray)
case _: JObject =>
builder.putMetadataArray(
- key, value.asInstanceOf[List[JObject]].map(fromJObject).toArray)
+ key,
+ value.asInstanceOf[List[JObject]].map(fromJObject).toArray)
case other =>
throw DataTypeErrors.unsupportedArrayTypeError(other.getClass)
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/NullType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/NullType.scala
index d211fac70c641..4e7fd3a00a8af 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/NullType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/NullType.scala
@@ -25,7 +25,7 @@ import org.apache.spark.annotation.Stable
* @since 1.3.0
*/
@Stable
-class NullType private() extends DataType {
+class NullType private () extends DataType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "NullType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/ShortType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/ShortType.scala
index 66696793e6279..c3b6bc75facd3 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/ShortType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/ShortType.scala
@@ -25,7 +25,8 @@ import org.apache.spark.annotation.Stable
* @since 1.3.0
*/
@Stable
-class ShortType private() extends IntegralType {
+class ShortType private () extends IntegralType {
+
/**
* The default size of a value of the ShortType is 2 bytes.
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/StringType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/StringType.scala
index df77f091f41f4..c2dd6cec7ba74 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/StringType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/StringType.scala
@@ -26,15 +26,17 @@ import org.apache.spark.sql.catalyst.util.CollationFactory
* The data type representing `String` values. Please use the singleton `DataTypes.StringType`.
*
* @since 1.3.0
- * @param collationId The id of collation for this StringType.
+ * @param collationId
+ * The id of collation for this StringType.
*/
@Stable
-class StringType private(val collationId: Int) extends AtomicType with Serializable {
+class StringType private (val collationId: Int) extends AtomicType with Serializable {
+
/**
- * Support for Binary Equality implies that strings are considered equal only if
- * they are byte for byte equal. E.g. all accent or case-insensitive collations are considered
- * non-binary. If this field is true, byte level operations can be used against this datatype
- * (e.g. for equality and hashing).
+ * Support for Binary Equality implies that strings are considered equal only if they are byte
+ * for byte equal. E.g. all accent or case-insensitive collations are considered non-binary. If
+ * this field is true, byte level operations can be used against this datatype (e.g. for
+ * equality and hashing).
*/
private[sql] def supportsBinaryEquality: Boolean =
CollationFactory.fetchCollation(collationId).supportsBinaryEquality
@@ -42,6 +44,9 @@ class StringType private(val collationId: Int) extends AtomicType with Serializa
private[sql] def supportsLowercaseEquality: Boolean =
CollationFactory.fetchCollation(collationId).supportsLowercaseEquality
+ private[sql] def isNonCSAI: Boolean =
+ !CollationFactory.isCaseSensitiveAndAccentInsensitive(collationId)
+
private[sql] def isUTF8BinaryCollation: Boolean =
collationId == CollationFactory.UTF8_BINARY_COLLATION_ID
@@ -49,18 +54,18 @@ class StringType private(val collationId: Int) extends AtomicType with Serializa
collationId == CollationFactory.UTF8_LCASE_COLLATION_ID
/**
- * Support for Binary Ordering implies that strings are considered equal only
- * if they are byte for byte equal. E.g. all accent or case-insensitive collations are
- * considered non-binary. Also their ordering does not require calls to ICU library, as
- * it follows spark internal implementation. If this field is true, byte level operations
- * can be used against this datatype (e.g. for equality, hashing and ordering).
+ * Support for Binary Ordering implies that strings are considered equal only if they are byte
+ * for byte equal. E.g. all accent or case-insensitive collations are considered non-binary.
+ * Also their ordering does not require calls to ICU library, as it follows spark internal
+ * implementation. If this field is true, byte level operations can be used against this
+ * datatype (e.g. for equality, hashing and ordering).
*/
private[sql] def supportsBinaryOrdering: Boolean =
CollationFactory.fetchCollation(collationId).supportsBinaryOrdering
/**
- * Type name that is shown to the customer.
- * If this is an UTF8_BINARY collation output is `string` due to backwards compatibility.
+ * Type name that is shown to the customer. If this is an UTF8_BINARY collation output is
+ * `string` due to backwards compatibility.
*/
override def typeName: String =
if (isUTF8BinaryCollation) "string"
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala
index 3ff96fea9ee04..d4e590629921c 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala
@@ -31,11 +31,15 @@ import org.apache.spark.util.SparkSchemaUtils
/**
* A field inside a StructType.
- * @param name The name of this field.
- * @param dataType The data type of this field.
- * @param nullable Indicates if values of this field can be `null` values.
- * @param metadata The metadata of this field. The metadata should be preserved during
- * transformation if the content of the column is not modified, e.g, in selection.
+ * @param name
+ * The name of this field.
+ * @param dataType
+ * The data type of this field.
+ * @param nullable
+ * Indicates if values of this field can be `null` values.
+ * @param metadata
+ * The metadata of this field. The metadata should be preserved during transformation if the
+ * content of the column is not modified, e.g, in selection.
*
* @since 1.3.0
*/
@@ -54,8 +58,9 @@ case class StructField(
stringConcat: StringConcat,
maxDepth: Int): Unit = {
if (maxDepth > 0) {
- stringConcat.append(s"$prefix-- ${SparkSchemaUtils.escapeMetaCharacters(name)}: " +
- s"${dataType.typeName} (nullable = $nullable)\n")
+ stringConcat.append(
+ s"$prefix-- ${SparkSchemaUtils.escapeMetaCharacters(name)}: " +
+ s"${dataType.typeName} (nullable = $nullable)\n")
DataType.buildFormattedString(dataType, s"$prefix |", stringConcat, maxDepth)
}
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/StructType.scala
index 3e637b5110122..4ef1cf400b80e 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -41,10 +41,10 @@ import org.apache.spark.util.SparkCollectionUtils
* {{{
* StructType(fields: Seq[StructField])
* }}}
- * For a [[StructType]] object, one or multiple [[StructField]]s can be extracted by names.
- * If multiple [[StructField]]s are extracted, a [[StructType]] object will be returned.
- * If a provided name does not have a matching field, it will be ignored. For the case
- * of extracting a single [[StructField]], a `null` will be returned.
+ * For a [[StructType]] object, one or multiple [[StructField]]s can be extracted by names. If
+ * multiple [[StructField]]s are extracted, a [[StructType]] object will be returned. If a
+ * provided name does not have a matching field, it will be ignored. For the case of extracting a
+ * single [[StructField]], a `null` will be returned.
*
* Scala Example:
* {{{
@@ -126,8 +126,8 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
override def equals(that: Any): Boolean = {
that match {
case StructType(otherFields) =>
- java.util.Arrays.equals(
- fields.asInstanceOf[Array[AnyRef]], otherFields.asInstanceOf[Array[AnyRef]])
+ java.util.Arrays
+ .equals(fields.asInstanceOf[Array[AnyRef]], otherFields.asInstanceOf[Array[AnyRef]])
case _ => false
}
}
@@ -146,7 +146,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
* .add(StructField("a", IntegerType, true))
* .add(StructField("b", LongType, false))
* .add(StructField("c", StringType, true))
- *}}}
+ * }}}
*/
def add(field: StructField): StructType = {
StructType(fields :+ field)
@@ -155,10 +155,8 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
/**
* Creates a new [[StructType]] by adding a new nullable field with no metadata.
*
- * val struct = (new StructType)
- * .add("a", IntegerType)
- * .add("b", LongType)
- * .add("c", StringType)
+ * val struct = (new StructType) .add("a", IntegerType) .add("b", LongType) .add("c",
+ * StringType)
*/
def add(name: String, dataType: DataType): StructType = {
StructType(fields :+ StructField(name, dataType, nullable = true, Metadata.empty))
@@ -167,10 +165,8 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
/**
* Creates a new [[StructType]] by adding a new field with no metadata.
*
- * val struct = (new StructType)
- * .add("a", IntegerType, true)
- * .add("b", LongType, false)
- * .add("c", StringType, true)
+ * val struct = (new StructType) .add("a", IntegerType, true) .add("b", LongType, false)
+ * .add("c", StringType, true)
*/
def add(name: String, dataType: DataType, nullable: Boolean): StructType = {
StructType(fields :+ StructField(name, dataType, nullable, Metadata.empty))
@@ -185,11 +181,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
* .add("c", StringType, true, Metadata.empty)
* }}}
*/
- def add(
- name: String,
- dataType: DataType,
- nullable: Boolean,
- metadata: Metadata): StructType = {
+ def add(name: String, dataType: DataType, nullable: Boolean, metadata: Metadata): StructType = {
StructType(fields :+ StructField(name, dataType, nullable, metadata))
}
@@ -202,11 +194,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
* .add("c", StringType, true, "comment3")
* }}}
*/
- def add(
- name: String,
- dataType: DataType,
- nullable: Boolean,
- comment: String): StructType = {
+ def add(name: String, dataType: DataType, nullable: Boolean, comment: String): StructType = {
StructType(fields :+ StructField(name, dataType, nullable).withComment(comment))
}
@@ -226,8 +214,8 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
}
/**
- * Creates a new [[StructType]] by adding a new field with no metadata where the
- * dataType is specified as a String.
+ * Creates a new [[StructType]] by adding a new field with no metadata where the dataType is
+ * specified as a String.
*
* {{{
* val struct = (new StructType)
@@ -241,8 +229,8 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
}
/**
- * Creates a new [[StructType]] by adding a new field and specifying metadata where the
- * dataType is specified as a String.
+ * Creates a new [[StructType]] by adding a new field and specifying metadata where the dataType
+ * is specified as a String.
* {{{
* val struct = (new StructType)
* .add("a", "int", true, Metadata.empty)
@@ -250,17 +238,13 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
* .add("c", "string", true, Metadata.empty)
* }}}
*/
- def add(
- name: String,
- dataType: String,
- nullable: Boolean,
- metadata: Metadata): StructType = {
+ def add(name: String, dataType: String, nullable: Boolean, metadata: Metadata): StructType = {
add(name, DataTypeParser.parseDataType(dataType), nullable, metadata)
}
/**
- * Creates a new [[StructType]] by adding a new field and specifying metadata where the
- * dataType is specified as a String.
+ * Creates a new [[StructType]] by adding a new field and specifying metadata where the dataType
+ * is specified as a String.
* {{{
* val struct = (new StructType)
* .add("a", "int", true, "comment1")
@@ -268,21 +252,19 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
* .add("c", "string", true, "comment3")
* }}}
*/
- def add(
- name: String,
- dataType: String,
- nullable: Boolean,
- comment: String): StructType = {
+ def add(name: String, dataType: String, nullable: Boolean, comment: String): StructType = {
add(name, DataTypeParser.parseDataType(dataType), nullable, comment)
}
/**
* Extracts the [[StructField]] with the given name.
*
- * @throws IllegalArgumentException if a field with the given name does not exist
+ * @throws IllegalArgumentException
+ * if a field with the given name does not exist
*/
def apply(name: String): StructField = {
- nameToField.getOrElse(name,
+ nameToField.getOrElse(
+ name,
throw new SparkIllegalArgumentException(
errorClass = "FIELD_NOT_FOUND",
messageParameters = immutable.Map(
@@ -294,7 +276,8 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
* Returns a [[StructType]] containing [[StructField]]s of the given names, preserving the
* original order of fields.
*
- * @throws IllegalArgumentException if at least one given field name does not exist
+ * @throws IllegalArgumentException
+ * if at least one given field name does not exist
*/
def apply(names: Set[String]): StructType = {
val nonExistFields = names -- fieldNamesSet
@@ -312,10 +295,12 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
/**
* Returns the index of a given field.
*
- * @throws IllegalArgumentException if a field with the given name does not exist
+ * @throws IllegalArgumentException
+ * if a field with the given name does not exist
*/
def fieldIndex(name: String): Int = {
- nameToIndex.getOrElse(name,
+ nameToIndex.getOrElse(
+ name,
throw new SparkIllegalArgumentException(
errorClass = "FIELD_NOT_FOUND",
messageParameters = immutable.Map(
@@ -354,10 +339,7 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
} else if (found.isEmpty) {
None
} else {
- findField(
- parent = found.head,
- searchPath = searchPath.tail,
- normalizedPath)
+ findField(parent = found.head, searchPath = searchPath.tail, normalizedPath)
}
}
@@ -433,11 +415,14 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
override def defaultSize: Int = fields.map(_.dataType.defaultSize).sum
override def simpleString: String = {
- val fieldTypes = fields.to(LazyList)
+ val fieldTypes = fields
+ .to(LazyList)
.map(field => s"${field.name}:${field.dataType.simpleString}")
SparkStringUtils.truncatedString(
fieldTypes,
- "struct<", ",", ">",
+ "struct<",
+ ",",
+ ">",
SqlApiConf.get.maxToStringFields)
}
@@ -460,9 +445,9 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
/**
* Returns a string containing a schema in DDL format. For example, the following value:
- * `StructType(Seq(StructField("eventId", IntegerType), StructField("s", StringType)))`
- * will be converted to `eventId` INT, `s` STRING.
- * The returned DDL schema can be used in a table creation.
+ * `StructType(Seq(StructField("eventId", IntegerType), StructField("s", StringType)))` will be
+ * converted to `eventId` INT, `s` STRING. The returned DDL schema can be used in a table
+ * creation.
*
* @since 2.4.0
*/
@@ -470,8 +455,8 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
private[sql] override def simpleString(maxNumberFields: Int): String = {
val builder = new StringBuilder
- val fieldTypes = fields.take(maxNumberFields).map {
- f => s"${f.name}: ${f.dataType.simpleString(maxNumberFields)}"
+ val fieldTypes = fields.take(maxNumberFields).map { f =>
+ s"${f.name}: ${f.dataType.simpleString(maxNumberFields)}"
}
builder.append("struct<")
builder.append(fieldTypes.mkString(", "))
@@ -486,31 +471,29 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
}
/**
- * Merges with another schema (`StructType`). For a struct field A from `this` and a struct field
- * B from `that`,
+ * Merges with another schema (`StructType`). For a struct field A from `this` and a struct
+ * field B from `that`,
*
- * 1. If A and B have the same name and data type, they are merged to a field C with the same name
- * and data type. C is nullable if and only if either A or B is nullable.
- * 2. If A doesn't exist in `that`, it's included in the result schema.
- * 3. If B doesn't exist in `this`, it's also included in the result schema.
- * 4. Otherwise, `this` and `that` are considered as conflicting schemas and an exception would be
- * thrown.
+ * 1. If A and B have the same name and data type, they are merged to a field C with the same
+ * name and data type. C is nullable if and only if either A or B is nullable. 2. If A
+ * doesn't exist in `that`, it's included in the result schema. 3. If B doesn't exist in
+ * `this`, it's also included in the result schema. 4. Otherwise, `this` and `that` are
+ * considered as conflicting schemas and an exception would be thrown.
*/
private[sql] def merge(that: StructType, caseSensitive: Boolean = true): StructType =
StructType.merge(this, that, caseSensitive).asInstanceOf[StructType]
override private[spark] def asNullable: StructType = {
- val newFields = fields.map {
- case StructField(name, dataType, nullable, metadata) =>
- StructField(name, dataType.asNullable, nullable = true, metadata)
+ val newFields = fields.map { case StructField(name, dataType, nullable, metadata) =>
+ StructField(name, dataType.asNullable, nullable = true, metadata)
}
StructType(newFields)
}
/**
- * Returns the same data type but set all nullability fields are true
- * (`StructField.nullable`, `ArrayType.containsNull`, and `MapType.valueContainsNull`).
+ * Returns the same data type but set all nullability fields are true (`StructField.nullable`,
+ * `ArrayType.containsNull`, and `MapType.valueContainsNull`).
*
* @since 4.0.0
*/
@@ -562,7 +545,8 @@ object StructType extends AbstractDataType {
case StructType(fields) =>
val newFields = fields.map { f =>
val mb = new MetadataBuilder()
- f.copy(dataType = removeMetadata(key, f.dataType),
+ f.copy(
+ dataType = removeMetadata(key, f.dataType),
metadata = mb.withMetadata(f.metadata).remove(key).build())
}
StructType(newFields)
@@ -570,39 +554,49 @@ object StructType extends AbstractDataType {
}
/**
- * This leverages `merge` to merge data types for UNION operator by specializing
- * the handling of struct types to follow UNION semantics.
+ * This leverages `merge` to merge data types for UNION operator by specializing the handling of
+ * struct types to follow UNION semantics.
*/
private[sql] def unionLikeMerge(left: DataType, right: DataType): DataType =
- mergeInternal(left, right, (s1: StructType, s2: StructType) => {
- val leftFields = s1.fields
- val rightFields = s2.fields
- require(leftFields.length == rightFields.length, "To merge nullability, " +
- "two structs must have same number of fields.")
-
- val newFields = leftFields.zip(rightFields).map {
- case (leftField, rightField) =>
+ mergeInternal(
+ left,
+ right,
+ (s1: StructType, s2: StructType) => {
+ val leftFields = s1.fields
+ val rightFields = s2.fields
+ require(
+ leftFields.length == rightFields.length,
+ "To merge nullability, " +
+ "two structs must have same number of fields.")
+
+ val newFields = leftFields.zip(rightFields).map { case (leftField, rightField) =>
leftField.copy(
dataType = unionLikeMerge(leftField.dataType, rightField.dataType),
nullable = leftField.nullable || rightField.nullable)
- }
- StructType(newFields)
- })
-
- private[sql] def merge(left: DataType, right: DataType, caseSensitive: Boolean = true): DataType =
- mergeInternal(left, right, (s1: StructType, s2: StructType) => {
- val leftFields = s1.fields
- val rightFields = s2.fields
- val newFields = mutable.ArrayBuffer.empty[StructField]
+ }
+ StructType(newFields)
+ })
- def normalize(name: String): String = {
- if (caseSensitive) name else name.toLowerCase(Locale.ROOT)
- }
+ private[sql] def merge(
+ left: DataType,
+ right: DataType,
+ caseSensitive: Boolean = true): DataType =
+ mergeInternal(
+ left,
+ right,
+ (s1: StructType, s2: StructType) => {
+ val leftFields = s1.fields
+ val rightFields = s2.fields
+ val newFields = mutable.ArrayBuffer.empty[StructField]
+
+ def normalize(name: String): String = {
+ if (caseSensitive) name else name.toLowerCase(Locale.ROOT)
+ }
- val rightMapped = fieldsMap(rightFields, caseSensitive)
- leftFields.foreach {
- case leftField @ StructField(leftName, leftType, leftNullable, _) =>
- rightMapped.get(normalize(leftName))
+ val rightMapped = fieldsMap(rightFields, caseSensitive)
+ leftFields.foreach { case leftField @ StructField(leftName, leftType, leftNullable, _) =>
+ rightMapped
+ .get(normalize(leftName))
.map { case rightField @ StructField(rightName, rightType, rightNullable, _) =>
try {
leftField.copy(
@@ -610,39 +604,40 @@ object StructType extends AbstractDataType {
nullable = leftNullable || rightNullable)
} catch {
case NonFatal(e) =>
- throw DataTypeErrors.cannotMergeIncompatibleDataTypesError(
- leftType, rightType)
+ throw DataTypeErrors.cannotMergeIncompatibleDataTypesError(leftType, rightType)
}
}
.orElse {
Some(leftField)
}
.foreach(newFields += _)
- }
-
- val leftMapped = fieldsMap(leftFields, caseSensitive)
- rightFields
- .filterNot(f => leftMapped.contains(normalize(f.name)))
- .foreach { f =>
- newFields += f
}
- StructType(newFields.toArray)
- })
+ val leftMapped = fieldsMap(leftFields, caseSensitive)
+ rightFields
+ .filterNot(f => leftMapped.contains(normalize(f.name)))
+ .foreach { f =>
+ newFields += f
+ }
+
+ StructType(newFields.toArray)
+ })
private def mergeInternal(
left: DataType,
right: DataType,
mergeStruct: (StructType, StructType) => StructType): DataType =
(left, right) match {
- case (ArrayType(leftElementType, leftContainsNull),
- ArrayType(rightElementType, rightContainsNull)) =>
+ case (
+ ArrayType(leftElementType, leftContainsNull),
+ ArrayType(rightElementType, rightContainsNull)) =>
ArrayType(
mergeInternal(leftElementType, rightElementType, mergeStruct),
leftContainsNull || rightContainsNull)
- case (MapType(leftKeyType, leftValueType, leftContainsNull),
- MapType(rightKeyType, rightValueType, rightContainsNull)) =>
+ case (
+ MapType(leftKeyType, leftValueType, leftContainsNull),
+ MapType(rightKeyType, rightValueType, rightContainsNull)) =>
MapType(
mergeInternal(leftKeyType, rightKeyType, mergeStruct),
mergeInternal(leftValueType, rightValueType, mergeStruct),
@@ -650,17 +645,20 @@ object StructType extends AbstractDataType {
case (s1: StructType, s2: StructType) => mergeStruct(s1, s2)
- case (DecimalType.Fixed(leftPrecision, leftScale),
- DecimalType.Fixed(rightPrecision, rightScale)) =>
+ case (
+ DecimalType.Fixed(leftPrecision, leftScale),
+ DecimalType.Fixed(rightPrecision, rightScale)) =>
if (leftScale == rightScale) {
DecimalType(leftPrecision.max(rightPrecision), leftScale)
} else {
throw DataTypeErrors.cannotMergeDecimalTypesWithIncompatibleScaleError(
- leftScale, rightScale)
+ leftScale,
+ rightScale)
}
case (leftUdt: UserDefinedType[_], rightUdt: UserDefinedType[_])
- if leftUdt.userClass == rightUdt.userClass => leftUdt
+ if leftUdt.userClass == rightUdt.userClass =>
+ leftUdt
case (YearMonthIntervalType(lstart, lend), YearMonthIntervalType(rstart, rend)) =>
YearMonthIntervalType(Math.min(lstart, rstart).toByte, Math.max(lend, rend).toByte)
@@ -706,10 +704,12 @@ object StructType extends AbstractDataType {
// Found a missing field in `source`.
newFields += field
} else if (bothStructType(found.get.dataType, field.dataType) &&
- !found.get.dataType.sameType(field.dataType)) {
+ !found.get.dataType.sameType(field.dataType)) {
// Found a field with same name, but different data type.
- findMissingFields(found.get.dataType.asInstanceOf[StructType],
- field.dataType.asInstanceOf[StructType], resolver).map { missingType =>
+ findMissingFields(
+ found.get.dataType.asInstanceOf[StructType],
+ field.dataType.asInstanceOf[StructType],
+ resolver).map { missingType =>
newFields += found.get.copy(dataType = missingType)
}
}
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala
index 9968d75dd2577..b08d16f0e2c97 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/TimestampNTZType.scala
@@ -20,16 +20,17 @@ package org.apache.spark.sql.types
import org.apache.spark.annotation.Unstable
/**
- * The timestamp without time zone type represents a local time in microsecond precision,
- * which is independent of time zone.
- * Its valid range is [0001-01-01T00:00:00.000000, 9999-12-31T23:59:59.999999].
- * To represent an absolute point in time, use `TimestampType` instead.
+ * The timestamp without time zone type represents a local time in microsecond precision, which is
+ * independent of time zone. Its valid range is [0001-01-01T00:00:00.000000,
+ * 9999-12-31T23:59:59.999999]. To represent an absolute point in time, use `TimestampType`
+ * instead.
*
* Please use the singleton `DataTypes.TimestampNTZType` to refer the type.
* @since 3.4.0
*/
@Unstable
-class TimestampNTZType private() extends DatetimeType {
+class TimestampNTZType private () extends DatetimeType {
+
/**
* The default size of a value of the TimestampNTZType is 8 bytes.
*/
@@ -42,9 +43,9 @@ class TimestampNTZType private() extends DatetimeType {
/**
* The companion case object and its class is separated so the companion object also subclasses
- * the TimestampNTZType class. Otherwise, the companion object would be of type
- * "TimestampNTZType" in byte code. Defined with a private constructor so the companion
- * object is the only possible instantiation.
+ * the TimestampNTZType class. Otherwise, the companion object would be of type "TimestampNTZType"
+ * in byte code. Defined with a private constructor so the companion object is the only possible
+ * instantiation.
*
* @since 3.4.0
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/TimestampType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
index 1185e4a9e32ca..bf869d1f38c57 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
@@ -20,16 +20,16 @@ package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
/**
- * The timestamp type represents a time instant in microsecond precision.
- * Valid range is [0001-01-01T00:00:00.000000Z, 9999-12-31T23:59:59.999999Z] where
- * the left/right-bound is a date and time of the proleptic Gregorian
- * calendar in UTC+00:00.
+ * The timestamp type represents a time instant in microsecond precision. Valid range is
+ * [0001-01-01T00:00:00.000000Z, 9999-12-31T23:59:59.999999Z] where the left/right-bound is a date
+ * and time of the proleptic Gregorian calendar in UTC+00:00.
*
* Please use the singleton `DataTypes.TimestampType` to refer the type.
* @since 1.3.0
*/
@Stable
-class TimestampType private() extends DatetimeType {
+class TimestampType private () extends DatetimeType {
+
/**
* The default size of a value of the TimestampType is 8 bytes.
*/
@@ -40,8 +40,8 @@ class TimestampType private() extends DatetimeType {
/**
* The companion case object and its class is separated so the companion object also subclasses
- * the TimestampType class. Otherwise, the companion object would be of type "TimestampType$"
- * in byte code. Defined with a private constructor so the companion object is the only possible
+ * the TimestampType class. Otherwise, the companion object would be of type "TimestampType$" in
+ * byte code. Defined with a private constructor so the companion object is the only possible
* instantiation.
*
* @since 1.3.0
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/UDTRegistration.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/UDTRegistration.scala
index 9219c1d139b99..85d421a07577b 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/UDTRegistration.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/UDTRegistration.scala
@@ -45,21 +45,26 @@ object UDTRegistration extends Serializable with Logging {
/**
* Queries if a given user class is already registered or not.
- * @param userClassName the name of user class
- * @return boolean value indicates if the given user class is registered or not
+ * @param userClassName
+ * the name of user class
+ * @return
+ * boolean value indicates if the given user class is registered or not
*/
def exists(userClassName: String): Boolean = udtMap.contains(userClassName)
/**
- * Registers an UserDefinedType to an user class. If the user class is already registered
- * with another UserDefinedType, warning log message will be shown.
- * @param userClass the name of user class
- * @param udtClass the name of UserDefinedType class for the given userClass
+ * Registers an UserDefinedType to an user class. If the user class is already registered with
+ * another UserDefinedType, warning log message will be shown.
+ * @param userClass
+ * the name of user class
+ * @param udtClass
+ * the name of UserDefinedType class for the given userClass
*/
def register(userClass: String, udtClass: String): Unit = {
if (udtMap.contains(userClass)) {
- logWarning(log"Cannot register UDT for ${MDC(LogKeys.CLASS_NAME, userClass)}, " +
- log"which is already registered.")
+ logWarning(
+ log"Cannot register UDT for ${MDC(LogKeys.CLASS_NAME, userClass)}, " +
+ log"which is already registered.")
} else {
// When register UDT with class name, we can't check if the UDT class is an UserDefinedType,
// or not. The check is deferred.
@@ -69,8 +74,10 @@ object UDTRegistration extends Serializable with Logging {
/**
* Returns the Class of UserDefinedType for the name of a given user class.
- * @param userClass class name of user class
- * @return Option value of the Class object of UserDefinedType
+ * @param userClass
+ * class name of user class
+ * @return
+ * Option value of the Class object of UserDefinedType
*/
def getUDTFor(userClass: String): Option[Class[_]] = {
udtMap.get(userClass).map { udtClassName =>
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/UpCastRule.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/UpCastRule.scala
index 7ec00bde0b25f..4993e249b3059 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/UpCastRule.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/UpCastRule.scala
@@ -22,13 +22,8 @@ package org.apache.spark.sql.types
private[sql] object UpCastRule {
// See https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types.
// The conversion for integral and floating point types have a linear widening hierarchy:
- val numericPrecedence: IndexedSeq[NumericType] = IndexedSeq(
- ByteType,
- ShortType,
- IntegerType,
- LongType,
- FloatType,
- DoubleType)
+ val numericPrecedence: IndexedSeq[NumericType] =
+ IndexedSeq(ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType)
/**
* Returns true iff we can safely up-cast the `from` type to `to` type without any truncating or
@@ -62,10 +57,9 @@ private[sql] object UpCastRule {
case (StructType(fromFields), StructType(toFields)) =>
fromFields.length == toFields.length &&
- fromFields.zip(toFields).forall {
- case (f1, f2) =>
- resolvableNullability(f1.nullable, f2.nullable) && canUpCast(f1.dataType, f2.dataType)
- }
+ fromFields.zip(toFields).forall { case (f1, f2) =>
+ resolvableNullability(f1.nullable, f2.nullable) && canUpCast(f1.dataType, f2.dataType)
+ }
case (_: DayTimeIntervalType, _: DayTimeIntervalType) => true
case (_: YearMonthIntervalType, _: YearMonthIntervalType) => true
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
index 5cbd876b31e68..dd8ca26c52462 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
@@ -27,15 +27,14 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
/**
* The data type for User Defined Types (UDTs).
*
- * This interface allows a user to make their own classes more interoperable with SparkSQL;
- * e.g., by creating a [[UserDefinedType]] for a class X, it becomes possible to create
- * a `DataFrame` which has class X in the schema.
+ * This interface allows a user to make their own classes more interoperable with SparkSQL; e.g.,
+ * by creating a [[UserDefinedType]] for a class X, it becomes possible to create a `DataFrame`
+ * which has class X in the schema.
*
- * For SparkSQL to recognize UDTs, the UDT must be annotated with
- * [[SQLUserDefinedType]].
+ * For SparkSQL to recognize UDTs, the UDT must be annotated with [[SQLUserDefinedType]].
*
- * The conversion via `serialize` occurs when instantiating a `DataFrame` from another RDD.
- * The conversion via `deserialize` occurs when reading from a `DataFrame`.
+ * The conversion via `serialize` occurs when instantiating a `DataFrame` from another RDD. The
+ * conversion via `deserialize` occurs when reading from a `DataFrame`.
*/
@DeveloperApi
@Since("3.2.0")
@@ -81,7 +80,7 @@ abstract class UserDefinedType[UserType >: Null] extends DataType with Serializa
override private[sql] def acceptsType(dataType: DataType): Boolean = dataType match {
case other: UserDefinedType[_] if this.userClass != null && other.userClass != null =>
this.getClass == other.getClass ||
- this.userClass.isAssignableFrom(other.userClass)
+ this.userClass.isAssignableFrom(other.userClass)
case _ => false
}
@@ -98,6 +97,7 @@ abstract class UserDefinedType[UserType >: Null] extends DataType with Serializa
}
private[spark] object UserDefinedType {
+
/**
* Get the sqlType of a (potential) [[UserDefinedType]].
*/
@@ -115,7 +115,8 @@ private[spark] object UserDefinedType {
private[sql] class PythonUserDefinedType(
val sqlType: DataType,
override val pyUDT: String,
- override val serializedPyClass: String) extends UserDefinedType[Any] {
+ override val serializedPyClass: String)
+ extends UserDefinedType[Any] {
/* The serialization is handled by UDT class in Python */
override def serialize(obj: Any): Any = obj
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/VariantType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/VariantType.scala
index 103fe7a59fc83..4d775c3e1e390 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/VariantType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/VariantType.scala
@@ -20,9 +20,9 @@ package org.apache.spark.sql.types
import org.apache.spark.annotation.Unstable
/**
- * The data type representing semi-structured values with arbitrary hierarchical data structures. It
- * is intended to store parsed JSON values and most other data types in the system (e.g., it cannot
- * store a map with a non-string key type).
+ * The data type representing semi-structured values with arbitrary hierarchical data structures.
+ * It is intended to store parsed JSON values and most other data types in the system (e.g., it
+ * cannot store a map with a non-string key type).
*
* @since 4.0.0
*/
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
index 6532a3b220c5b..f69054f2c1fbc 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
@@ -29,18 +29,19 @@ import org.apache.spark.sql.types.YearMonthIntervalType.fieldToString
*
* `YearMonthIntervalType` represents positive as well as negative year-month intervals.
*
- * @param startField The leftmost field which the type comprises of. Valid values:
- * 0 (YEAR), 1 (MONTH).
- * @param endField The rightmost field which the type comprises of. Valid values:
- * 0 (YEAR), 1 (MONTH).
+ * @param startField
+ * The leftmost field which the type comprises of. Valid values: 0 (YEAR), 1 (MONTH).
+ * @param endField
+ * The rightmost field which the type comprises of. Valid values: 0 (YEAR), 1 (MONTH).
*
* @since 3.2.0
*/
@Unstable
case class YearMonthIntervalType(startField: Byte, endField: Byte) extends AnsiIntervalType {
+
/**
- * Year-month interval values always occupy 4 bytes.
- * The YEAR field is constrained by the upper bound 178956970 to fit to `Int`.
+ * Year-month interval values always occupy 4 bytes. The YEAR field is constrained by the upper
+ * bound 178956970 to fit to `Int`.
*/
override def defaultSize: Int = 4
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala b/sql/api/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
index 6852fe09ef96b..1740cbe2957b8 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
@@ -38,33 +38,33 @@ private[sql] object ArrowUtils {
// todo: support more types.
/** Maps data type from Spark to Arrow. NOTE: timeZoneId required for TimestampTypes */
- def toArrowType(
- dt: DataType, timeZoneId: String, largeVarTypes: Boolean = false): ArrowType = dt match {
- case BooleanType => ArrowType.Bool.INSTANCE
- case ByteType => new ArrowType.Int(8, true)
- case ShortType => new ArrowType.Int(8 * 2, true)
- case IntegerType => new ArrowType.Int(8 * 4, true)
- case LongType => new ArrowType.Int(8 * 8, true)
- case FloatType => new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)
- case DoubleType => new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)
- case _: StringType if !largeVarTypes => ArrowType.Utf8.INSTANCE
- case BinaryType if !largeVarTypes => ArrowType.Binary.INSTANCE
- case _: StringType if largeVarTypes => ArrowType.LargeUtf8.INSTANCE
- case BinaryType if largeVarTypes => ArrowType.LargeBinary.INSTANCE
- case DecimalType.Fixed(precision, scale) => new ArrowType.Decimal(precision, scale, 8 * 16)
- case DateType => new ArrowType.Date(DateUnit.DAY)
- case TimestampType if timeZoneId == null =>
- throw SparkException.internalError("Missing timezoneId where it is mandatory.")
- case TimestampType => new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId)
- case TimestampNTZType =>
- new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)
- case NullType => ArrowType.Null.INSTANCE
- case _: YearMonthIntervalType => new ArrowType.Interval(IntervalUnit.YEAR_MONTH)
- case _: DayTimeIntervalType => new ArrowType.Duration(TimeUnit.MICROSECOND)
- case CalendarIntervalType => new ArrowType.Interval(IntervalUnit.MONTH_DAY_NANO)
- case _ =>
- throw ExecutionErrors.unsupportedDataTypeError(dt)
- }
+ def toArrowType(dt: DataType, timeZoneId: String, largeVarTypes: Boolean = false): ArrowType =
+ dt match {
+ case BooleanType => ArrowType.Bool.INSTANCE
+ case ByteType => new ArrowType.Int(8, true)
+ case ShortType => new ArrowType.Int(8 * 2, true)
+ case IntegerType => new ArrowType.Int(8 * 4, true)
+ case LongType => new ArrowType.Int(8 * 8, true)
+ case FloatType => new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)
+ case DoubleType => new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)
+ case _: StringType if !largeVarTypes => ArrowType.Utf8.INSTANCE
+ case BinaryType if !largeVarTypes => ArrowType.Binary.INSTANCE
+ case _: StringType if largeVarTypes => ArrowType.LargeUtf8.INSTANCE
+ case BinaryType if largeVarTypes => ArrowType.LargeBinary.INSTANCE
+ case DecimalType.Fixed(precision, scale) => new ArrowType.Decimal(precision, scale, 8 * 16)
+ case DateType => new ArrowType.Date(DateUnit.DAY)
+ case TimestampType if timeZoneId == null =>
+ throw SparkException.internalError("Missing timezoneId where it is mandatory.")
+ case TimestampType => new ArrowType.Timestamp(TimeUnit.MICROSECOND, timeZoneId)
+ case TimestampNTZType =>
+ new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)
+ case NullType => ArrowType.Null.INSTANCE
+ case _: YearMonthIntervalType => new ArrowType.Interval(IntervalUnit.YEAR_MONTH)
+ case _: DayTimeIntervalType => new ArrowType.Duration(TimeUnit.MICROSECOND)
+ case CalendarIntervalType => new ArrowType.Interval(IntervalUnit.MONTH_DAY_NANO)
+ case _ =>
+ throw ExecutionErrors.unsupportedDataTypeError(dt)
+ }
def fromArrowType(dt: ArrowType): DataType = dt match {
case ArrowType.Bool.INSTANCE => BooleanType
@@ -73,9 +73,11 @@ private[sql] object ArrowUtils {
case int: ArrowType.Int if int.getIsSigned && int.getBitWidth == 8 * 4 => IntegerType
case int: ArrowType.Int if int.getIsSigned && int.getBitWidth == 8 * 8 => LongType
case float: ArrowType.FloatingPoint
- if float.getPrecision() == FloatingPointPrecision.SINGLE => FloatType
+ if float.getPrecision() == FloatingPointPrecision.SINGLE =>
+ FloatType
case float: ArrowType.FloatingPoint
- if float.getPrecision() == FloatingPointPrecision.DOUBLE => DoubleType
+ if float.getPrecision() == FloatingPointPrecision.DOUBLE =>
+ DoubleType
case ArrowType.Utf8.INSTANCE => StringType
case ArrowType.Binary.INSTANCE => BinaryType
case ArrowType.LargeUtf8.INSTANCE => StringType
@@ -83,13 +85,15 @@ private[sql] object ArrowUtils {
case d: ArrowType.Decimal => DecimalType(d.getPrecision, d.getScale)
case date: ArrowType.Date if date.getUnit == DateUnit.DAY => DateType
case ts: ArrowType.Timestamp
- if ts.getUnit == TimeUnit.MICROSECOND && ts.getTimezone == null => TimestampNTZType
+ if ts.getUnit == TimeUnit.MICROSECOND && ts.getTimezone == null =>
+ TimestampNTZType
case ts: ArrowType.Timestamp if ts.getUnit == TimeUnit.MICROSECOND => TimestampType
case ArrowType.Null.INSTANCE => NullType
- case yi: ArrowType.Interval if yi.getUnit == IntervalUnit.YEAR_MONTH => YearMonthIntervalType()
+ case yi: ArrowType.Interval if yi.getUnit == IntervalUnit.YEAR_MONTH =>
+ YearMonthIntervalType()
case di: ArrowType.Duration if di.getUnit == TimeUnit.MICROSECOND => DayTimeIntervalType()
- case ci: ArrowType.Interval
- if ci.getUnit == IntervalUnit.MONTH_DAY_NANO => CalendarIntervalType
+ case ci: ArrowType.Interval if ci.getUnit == IntervalUnit.MONTH_DAY_NANO =>
+ CalendarIntervalType
case _ => throw ExecutionErrors.unsupportedArrowTypeError(dt)
}
@@ -103,37 +107,54 @@ private[sql] object ArrowUtils {
dt match {
case ArrayType(elementType, containsNull) =>
val fieldType = new FieldType(nullable, ArrowType.List.INSTANCE, null)
- new Field(name, fieldType,
- Seq(toArrowField("element", elementType, containsNull, timeZoneId,
- largeVarTypes)).asJava)
+ new Field(
+ name,
+ fieldType,
+ Seq(
+ toArrowField("element", elementType, containsNull, timeZoneId, largeVarTypes)).asJava)
case StructType(fields) =>
val fieldType = new FieldType(nullable, ArrowType.Struct.INSTANCE, null)
- new Field(name, fieldType,
- fields.map { field =>
- toArrowField(field.name, field.dataType, field.nullable, timeZoneId, largeVarTypes)
- }.toImmutableArraySeq.asJava)
+ new Field(
+ name,
+ fieldType,
+ fields
+ .map { field =>
+ toArrowField(field.name, field.dataType, field.nullable, timeZoneId, largeVarTypes)
+ }
+ .toImmutableArraySeq
+ .asJava)
case MapType(keyType, valueType, valueContainsNull) =>
val mapType = new FieldType(nullable, new ArrowType.Map(false), null)
// Note: Map Type struct can not be null, Struct Type key field can not be null
- new Field(name, mapType,
- Seq(toArrowField(MapVector.DATA_VECTOR_NAME,
- new StructType()
- .add(MapVector.KEY_NAME, keyType, nullable = false)
- .add(MapVector.VALUE_NAME, valueType, nullable = valueContainsNull),
- nullable = false,
- timeZoneId,
- largeVarTypes)).asJava)
+ new Field(
+ name,
+ mapType,
+ Seq(
+ toArrowField(
+ MapVector.DATA_VECTOR_NAME,
+ new StructType()
+ .add(MapVector.KEY_NAME, keyType, nullable = false)
+ .add(MapVector.VALUE_NAME, valueType, nullable = valueContainsNull),
+ nullable = false,
+ timeZoneId,
+ largeVarTypes)).asJava)
case udt: UserDefinedType[_] =>
toArrowField(name, udt.sqlType, nullable, timeZoneId, largeVarTypes)
case _: VariantType =>
- val fieldType = new FieldType(nullable, ArrowType.Struct.INSTANCE, null,
+ val fieldType = new FieldType(
+ nullable,
+ ArrowType.Struct.INSTANCE,
+ null,
Map("variant" -> "true").asJava)
- new Field(name, fieldType,
- Seq(toArrowField("value", BinaryType, false, timeZoneId, largeVarTypes),
+ new Field(
+ name,
+ fieldType,
+ Seq(
+ toArrowField("value", BinaryType, false, timeZoneId, largeVarTypes),
toArrowField("metadata", BinaryType, false, timeZoneId, largeVarTypes)).asJava)
case dataType =>
- val fieldType = new FieldType(nullable, toArrowType(dataType, timeZoneId,
- largeVarTypes), null)
+ val fieldType =
+ new FieldType(nullable, toArrowType(dataType, timeZoneId, largeVarTypes), null)
new Field(name, fieldType, Seq.empty[Field].asJava)
}
}
@@ -149,9 +170,12 @@ private[sql] object ArrowUtils {
val elementField = field.getChildren().get(0)
val elementType = fromArrowField(elementField)
ArrayType(elementType, containsNull = elementField.isNullable)
- case ArrowType.Struct.INSTANCE if field.getMetadata.getOrDefault("variant", "") == "true"
- && field.getChildren.asScala.map(_.getName).asJava
- .containsAll(Seq("value", "metadata").asJava) =>
+ case ArrowType.Struct.INSTANCE
+ if field.getMetadata.getOrDefault("variant", "") == "true"
+ && field.getChildren.asScala
+ .map(_.getName)
+ .asJava
+ .containsAll(Seq("value", "metadata").asJava) =>
VariantType
case ArrowType.Struct.INSTANCE =>
val fields = field.getChildren().asScala.map { child =>
@@ -163,7 +187,9 @@ private[sql] object ArrowUtils {
}
}
- /** Maps schema from Spark to Arrow. NOTE: timeZoneId required for TimestampType in StructType */
+ /**
+ * Maps schema from Spark to Arrow. NOTE: timeZoneId required for TimestampType in StructType
+ */
def toArrowSchema(
schema: StructType,
timeZoneId: String,
@@ -187,14 +213,17 @@ private[sql] object ArrowUtils {
}
private def deduplicateFieldNames(
- dt: DataType, errorOnDuplicatedFieldNames: Boolean): DataType = dt match {
- case udt: UserDefinedType[_] => deduplicateFieldNames(udt.sqlType, errorOnDuplicatedFieldNames)
+ dt: DataType,
+ errorOnDuplicatedFieldNames: Boolean): DataType = dt match {
+ case udt: UserDefinedType[_] =>
+ deduplicateFieldNames(udt.sqlType, errorOnDuplicatedFieldNames)
case st @ StructType(fields) =>
val newNames = if (st.names.toSet.size == st.names.length) {
st.names
} else {
if (errorOnDuplicatedFieldNames) {
- throw ExecutionErrors.duplicatedFieldNameInArrowStructError(st.names.toImmutableArraySeq)
+ throw ExecutionErrors.duplicatedFieldNameInArrowStructError(
+ st.names.toImmutableArraySeq)
}
val genNawName = st.names.groupBy(identity).map {
case (name, names) if names.length > 1 =>
@@ -207,7 +236,10 @@ private[sql] object ArrowUtils {
val newFields =
fields.zip(newNames).map { case (StructField(_, dataType, nullable, metadata), name) =>
StructField(
- name, deduplicateFieldNames(dataType, errorOnDuplicatedFieldNames), nullable, metadata)
+ name,
+ deduplicateFieldNames(dataType, errorOnDuplicatedFieldNames),
+ nullable,
+ metadata)
}
StructType(newFields)
case ArrayType(elementType, containsNull) =>
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
index 07a9409bc57a2..18646f67975c0 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java
@@ -17,20 +17,25 @@
package org.apache.spark.sql.catalyst.expressions;
-import org.apache.spark.SparkBuildInfo;
-import org.apache.spark.sql.errors.QueryExecutionErrors;
-import org.apache.spark.unsafe.types.UTF8String;
-import org.apache.spark.util.VersionUtils;
-
-import javax.crypto.Cipher;
-import javax.crypto.spec.GCMParameterSpec;
-import javax.crypto.spec.IvParameterSpec;
-import javax.crypto.spec.SecretKeySpec;
import java.nio.ByteBuffer;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
import java.security.spec.AlgorithmParameterSpec;
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import javax.crypto.Cipher;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+import org.apache.spark.SparkBuildInfo;
+import org.apache.spark.sql.catalyst.util.ArrayData;
+import org.apache.spark.sql.catalyst.util.GenericArrayData;
+import org.apache.spark.sql.errors.QueryExecutionErrors;
+import org.apache.spark.unsafe.types.UTF8String;
+import org.apache.spark.util.VersionUtils;
/**
* A utility class for constructing expressions.
@@ -272,4 +277,42 @@ private static byte[] aesInternal(
throw QueryExecutionErrors.aesCryptoError(e.getMessage());
}
}
+
+ public static ArrayData getSentences(
+ UTF8String str,
+ UTF8String language,
+ UTF8String country) {
+ if (str == null) return null;
+ Locale locale;
+ if (language != null && country != null) {
+ locale = new Locale(language.toString(), country.toString());
+ } else if (language != null) {
+ locale = new Locale(language.toString());
+ } else {
+ locale = Locale.US;
+ }
+ String sentences = str.toString();
+ BreakIterator sentenceInstance = BreakIterator.getSentenceInstance(locale);
+ sentenceInstance.setText(sentences);
+
+ int sentenceIndex = 0;
+ List res = new ArrayList<>();
+ while (sentenceInstance.next() != BreakIterator.DONE) {
+ String sentence = sentences.substring(sentenceIndex, sentenceInstance.current());
+ sentenceIndex = sentenceInstance.current();
+ BreakIterator wordInstance = BreakIterator.getWordInstance(locale);
+ wordInstance.setText(sentence);
+ int wordIndex = 0;
+ List words = new ArrayList<>();
+ while (wordInstance.next() != BreakIterator.DONE) {
+ String word = sentence.substring(wordIndex, wordInstance.current());
+ wordIndex = wordInstance.current();
+ if (Character.isLetterOrDigit(word.charAt(0))) {
+ words.add(UTF8String.fromString(word));
+ }
+ }
+ res.add(new GenericArrayData(words.toArray(new UTF8String[0])));
+ }
+ return new GenericArrayData(res.toArray(new GenericArrayData[0]));
+ }
}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Column.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Column.java
index b191438dbc3ee..8b32940d7a657 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Column.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Column.java
@@ -53,7 +53,7 @@ static Column create(
boolean nullable,
String comment,
String metadataInJSON) {
- return new ColumnImpl(name, dataType, nullable, comment, null, null, metadataInJSON);
+ return new ColumnImpl(name, dataType, nullable, comment, null, null, null, metadataInJSON);
}
static Column create(
@@ -63,7 +63,8 @@ static Column create(
String comment,
ColumnDefaultValue defaultValue,
String metadataInJSON) {
- return new ColumnImpl(name, dataType, nullable, comment, defaultValue, null, metadataInJSON);
+ return new ColumnImpl(name, dataType, nullable, comment, defaultValue,
+ null, null, metadataInJSON);
}
static Column create(
@@ -74,7 +75,18 @@ static Column create(
String generationExpression,
String metadataInJSON) {
return new ColumnImpl(name, dataType, nullable, comment, null,
- generationExpression, metadataInJSON);
+ generationExpression, null, metadataInJSON);
+ }
+
+ static Column create(
+ String name,
+ DataType dataType,
+ boolean nullable,
+ String comment,
+ IdentityColumnSpec identityColumnSpec,
+ String metadataInJSON) {
+ return new ColumnImpl(name, dataType, nullable, comment, null,
+ null, identityColumnSpec, metadataInJSON);
}
/**
@@ -113,6 +125,12 @@ static Column create(
@Nullable
String generationExpression();
+ /**
+ * Returns the identity column specification of this table column. Null means no identity column.
+ */
+ @Nullable
+ IdentityColumnSpec identityColumnSpec();
+
/**
* Returns the column metadata in JSON format.
*/
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalogCapability.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalogCapability.java
index 5ccb15ff1f0a4..dceac1b484cf2 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalogCapability.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalogCapability.java
@@ -59,5 +59,23 @@ public enum TableCatalogCapability {
* {@link TableCatalog#createTable}.
* See {@link Column#defaultValue()}.
*/
- SUPPORT_COLUMN_DEFAULT_VALUE
+ SUPPORT_COLUMN_DEFAULT_VALUE,
+
+ /**
+ * Signals that the TableCatalog supports defining identity columns upon table creation in SQL.
+ *
+ * Without this capability, any create/replace table statements with an identity column defined
+ * in the table schema will throw an exception during analysis.
+ *
+ * An identity column is defined with syntax:
+ * {@code colName colType GENERATED ALWAYS AS IDENTITY(identityColumnSpec)}
+ * or
+ * {@code colName colType GENERATED BY DEFAULT AS IDENTITY(identityColumnSpec)}
+ * identityColumnSpec is defined with syntax: {@code [START WITH start | INCREMENT BY step]*}
+ *
+ * IdentitySpec is included in the column definition for APIs like
+ * {@link TableCatalog#createTable}.
+ * See {@link Column#identityColumnSpec()}.
+ */
+ SUPPORTS_CREATE_TABLE_WITH_IDENTITY_COLUMNS
}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/ProcedureParameter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/ProcedureParameter.java
index 90d531ae21892..18c76833c5879 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/ProcedureParameter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/ProcedureParameter.java
@@ -32,6 +32,11 @@
*/
@Evolving
public interface ProcedureParameter {
+ /**
+ * A field metadata key that indicates whether an argument is passed by name.
+ */
+ String BY_NAME_METADATA_KEY = "BY_NAME";
+
/**
* Creates a builder for an IN procedure parameter.
*
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/UnboundProcedure.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/UnboundProcedure.java
index ee9a09055243b..1a91fd21bf07e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/UnboundProcedure.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/procedures/UnboundProcedure.java
@@ -35,6 +35,12 @@ public interface UnboundProcedure extends Procedure {
* validate if the input types are compatible while binding or delegate that to Spark. Regardless,
* Spark will always perform the final validation of the arguments and rearrange them as needed
* based on {@link BoundProcedure#parameters() reported parameters}.
+ *
+ * The provided {@code inputType} is based on the procedure arguments. If an argument is passed
+ * by name, its metadata will indicate this with {@link ProcedureParameter#BY_NAME_METADATA_KEY}
+ * set to {@code true}. In such cases, the field name will match the name of the target procedure
+ * parameter. If the argument is not named, {@link ProcedureParameter#BY_NAME_METADATA_KEY} will
+ * not be set and the name will be assigned randomly.
*
* @param inputType the input types to bind to
* @return the bound procedure that is most suitable for the given input types
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
deleted file mode 100644
index 9b95f74db3a49..0000000000000
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
+++ /dev/null
@@ -1,359 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import java.lang.reflect.Modifier
-
-import scala.reflect.{classTag, ClassTag}
-import scala.reflect.runtime.universe.TypeTag
-
-import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal
-import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder}
-import org.apache.spark.sql.catalyst.expressions.{BoundReference, Cast}
-import org.apache.spark.sql.catalyst.expressions.objects.{DecodeUsingSerializer, EncodeUsingSerializer}
-import org.apache.spark.sql.errors.QueryExecutionErrors
-import org.apache.spark.sql.types._
-
-/**
- * Methods for creating an [[Encoder]].
- *
- * @since 1.6.0
- */
-object Encoders {
-
- /**
- * An encoder for nullable boolean type.
- * The Scala primitive encoder is available as [[scalaBoolean]].
- * @since 1.6.0
- */
- def BOOLEAN: Encoder[java.lang.Boolean] = ExpressionEncoder()
-
- /**
- * An encoder for nullable byte type.
- * The Scala primitive encoder is available as [[scalaByte]].
- * @since 1.6.0
- */
- def BYTE: Encoder[java.lang.Byte] = ExpressionEncoder()
-
- /**
- * An encoder for nullable short type.
- * The Scala primitive encoder is available as [[scalaShort]].
- * @since 1.6.0
- */
- def SHORT: Encoder[java.lang.Short] = ExpressionEncoder()
-
- /**
- * An encoder for nullable int type.
- * The Scala primitive encoder is available as [[scalaInt]].
- * @since 1.6.0
- */
- def INT: Encoder[java.lang.Integer] = ExpressionEncoder()
-
- /**
- * An encoder for nullable long type.
- * The Scala primitive encoder is available as [[scalaLong]].
- * @since 1.6.0
- */
- def LONG: Encoder[java.lang.Long] = ExpressionEncoder()
-
- /**
- * An encoder for nullable float type.
- * The Scala primitive encoder is available as [[scalaFloat]].
- * @since 1.6.0
- */
- def FLOAT: Encoder[java.lang.Float] = ExpressionEncoder()
-
- /**
- * An encoder for nullable double type.
- * The Scala primitive encoder is available as [[scalaDouble]].
- * @since 1.6.0
- */
- def DOUBLE: Encoder[java.lang.Double] = ExpressionEncoder()
-
- /**
- * An encoder for nullable string type.
- *
- * @since 1.6.0
- */
- def STRING: Encoder[java.lang.String] = ExpressionEncoder()
-
- /**
- * An encoder for nullable decimal type.
- *
- * @since 1.6.0
- */
- def DECIMAL: Encoder[java.math.BigDecimal] = ExpressionEncoder()
-
- /**
- * An encoder for nullable date type.
- *
- * @since 1.6.0
- */
- def DATE: Encoder[java.sql.Date] = ExpressionEncoder()
-
- /**
- * Creates an encoder that serializes instances of the `java.time.LocalDate` class
- * to the internal representation of nullable Catalyst's DateType.
- *
- * @since 3.0.0
- */
- def LOCALDATE: Encoder[java.time.LocalDate] = ExpressionEncoder()
-
- /**
- * Creates an encoder that serializes instances of the `java.time.LocalDateTime` class
- * to the internal representation of nullable Catalyst's TimestampNTZType.
- *
- * @since 3.4.0
- */
- def LOCALDATETIME: Encoder[java.time.LocalDateTime] = ExpressionEncoder()
-
- /**
- * An encoder for nullable timestamp type.
- *
- * @since 1.6.0
- */
- def TIMESTAMP: Encoder[java.sql.Timestamp] = ExpressionEncoder()
-
- /**
- * Creates an encoder that serializes instances of the `java.time.Instant` class
- * to the internal representation of nullable Catalyst's TimestampType.
- *
- * @since 3.0.0
- */
- def INSTANT: Encoder[java.time.Instant] = ExpressionEncoder()
-
- /**
- * An encoder for arrays of bytes.
- *
- * @since 1.6.1
- */
- def BINARY: Encoder[Array[Byte]] = ExpressionEncoder()
-
- /**
- * Creates an encoder that serializes instances of the `java.time.Duration` class
- * to the internal representation of nullable Catalyst's DayTimeIntervalType.
- *
- * @since 3.2.0
- */
- def DURATION: Encoder[java.time.Duration] = ExpressionEncoder()
-
- /**
- * Creates an encoder that serializes instances of the `java.time.Period` class
- * to the internal representation of nullable Catalyst's YearMonthIntervalType.
- *
- * @since 3.2.0
- */
- def PERIOD: Encoder[java.time.Period] = ExpressionEncoder()
-
- /**
- * Creates an encoder for Java Bean of type T.
- *
- * T must be publicly accessible.
- *
- * supported types for java bean field:
- * - primitive types: boolean, int, double, etc.
- * - boxed types: Boolean, Integer, Double, etc.
- * - String
- * - java.math.BigDecimal, java.math.BigInteger
- * - time related: java.sql.Date, java.sql.Timestamp, java.time.LocalDate, java.time.Instant
- * - collection types: array, java.util.List, and map
- * - nested java bean.
- *
- * @since 1.6.0
- */
- def bean[T](beanClass: Class[T]): Encoder[T] = ExpressionEncoder.javaBean(beanClass)
-
- /**
- * Creates a [[Row]] encoder for schema `schema`.
- *
- * @since 3.5.0
- */
- def row(schema: StructType): Encoder[Row] = ExpressionEncoder(schema)
-
- /**
- * (Scala-specific) Creates an encoder that serializes objects of type T using Kryo.
- * This encoder maps T into a single byte array (binary) field.
- *
- * T must be publicly accessible.
- *
- * @since 1.6.0
- */
- def kryo[T: ClassTag]: Encoder[T] = genericSerializer(useKryo = true)
-
- /**
- * Creates an encoder that serializes objects of type T using Kryo.
- * This encoder maps T into a single byte array (binary) field.
- *
- * T must be publicly accessible.
- *
- * @since 1.6.0
- */
- def kryo[T](clazz: Class[T]): Encoder[T] = kryo(ClassTag[T](clazz))
-
- /**
- * (Scala-specific) Creates an encoder that serializes objects of type T using generic Java
- * serialization. This encoder maps T into a single byte array (binary) field.
- *
- * T must be publicly accessible.
- *
- * @note This is extremely inefficient and should only be used as the last resort.
- *
- * @since 1.6.0
- */
- def javaSerialization[T: ClassTag]: Encoder[T] = genericSerializer(useKryo = false)
-
- /**
- * Creates an encoder that serializes objects of type T using generic Java serialization.
- * This encoder maps T into a single byte array (binary) field.
- *
- * T must be publicly accessible.
- *
- * @note This is extremely inefficient and should only be used as the last resort.
- *
- * @since 1.6.0
- */
- def javaSerialization[T](clazz: Class[T]): Encoder[T] = javaSerialization(ClassTag[T](clazz))
-
- /** Throws an exception if T is not a public class. */
- private def validatePublicClass[T: ClassTag](): Unit = {
- if (!Modifier.isPublic(classTag[T].runtimeClass.getModifiers)) {
- throw QueryExecutionErrors.notPublicClassError(classTag[T].runtimeClass.getName)
- }
- }
-
- /** A way to construct encoders using generic serializers. */
- private def genericSerializer[T: ClassTag](useKryo: Boolean): Encoder[T] = {
- if (classTag[T].runtimeClass.isPrimitive) {
- throw QueryExecutionErrors.primitiveTypesNotSupportedError()
- }
-
- validatePublicClass[T]()
-
- ExpressionEncoder[T](
- objSerializer =
- EncodeUsingSerializer(
- BoundReference(0, ObjectType(classOf[AnyRef]), nullable = true), kryo = useKryo),
- objDeserializer =
- DecodeUsingSerializer[T](
- Cast(GetColumnByOrdinal(0, BinaryType), BinaryType),
- classTag[T],
- kryo = useKryo),
- clsTag = classTag[T]
- )
- }
-
- /**
- * An encoder for 2-ary tuples.
- *
- * @since 1.6.0
- */
- def tuple[T1, T2](
- e1: Encoder[T1],
- e2: Encoder[T2]): Encoder[(T1, T2)] = {
- ExpressionEncoder.tuple(encoderFor(e1), encoderFor(e2))
- }
-
- /**
- * An encoder for 3-ary tuples.
- *
- * @since 1.6.0
- */
- def tuple[T1, T2, T3](
- e1: Encoder[T1],
- e2: Encoder[T2],
- e3: Encoder[T3]): Encoder[(T1, T2, T3)] = {
- ExpressionEncoder.tuple(encoderFor(e1), encoderFor(e2), encoderFor(e3))
- }
-
- /**
- * An encoder for 4-ary tuples.
- *
- * @since 1.6.0
- */
- def tuple[T1, T2, T3, T4](
- e1: Encoder[T1],
- e2: Encoder[T2],
- e3: Encoder[T3],
- e4: Encoder[T4]): Encoder[(T1, T2, T3, T4)] = {
- ExpressionEncoder.tuple(encoderFor(e1), encoderFor(e2), encoderFor(e3), encoderFor(e4))
- }
-
- /**
- * An encoder for 5-ary tuples.
- *
- * @since 1.6.0
- */
- def tuple[T1, T2, T3, T4, T5](
- e1: Encoder[T1],
- e2: Encoder[T2],
- e3: Encoder[T3],
- e4: Encoder[T4],
- e5: Encoder[T5]): Encoder[(T1, T2, T3, T4, T5)] = {
- ExpressionEncoder.tuple(
- encoderFor(e1), encoderFor(e2), encoderFor(e3), encoderFor(e4), encoderFor(e5))
- }
-
- /**
- * An encoder for Scala's product type (tuples, case classes, etc).
- * @since 2.0.0
- */
- def product[T <: Product : TypeTag]: Encoder[T] = ExpressionEncoder()
-
- /**
- * An encoder for Scala's primitive int type.
- * @since 2.0.0
- */
- def scalaInt: Encoder[Int] = ExpressionEncoder()
-
- /**
- * An encoder for Scala's primitive long type.
- * @since 2.0.0
- */
- def scalaLong: Encoder[Long] = ExpressionEncoder()
-
- /**
- * An encoder for Scala's primitive double type.
- * @since 2.0.0
- */
- def scalaDouble: Encoder[Double] = ExpressionEncoder()
-
- /**
- * An encoder for Scala's primitive float type.
- * @since 2.0.0
- */
- def scalaFloat: Encoder[Float] = ExpressionEncoder()
-
- /**
- * An encoder for Scala's primitive byte type.
- * @since 2.0.0
- */
- def scalaByte: Encoder[Byte] = ExpressionEncoder()
-
- /**
- * An encoder for Scala's primitive short type.
- * @since 2.0.0
- */
- def scalaShort: Encoder[Short] = ExpressionEncoder()
-
- /**
- * An encoder for Scala's primitive boolean type.
- * @since 2.0.0
- */
- def scalaBoolean: Encoder[Boolean] = ExpressionEncoder()
-
-}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
index 40b49506b58aa..4752434015375 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
@@ -19,11 +19,11 @@ package org.apache.spark.sql.catalyst
import org.apache.spark.sql.catalyst.{expressions => exprs}
import org.apache.spark.sql.catalyst.analysis.{GetColumnByOrdinal, UnresolvedExtractValue}
-import org.apache.spark.sql.catalyst.encoders.{AgnosticEncoder, AgnosticEncoders, Codec}
+import org.apache.spark.sql.catalyst.encoders.{AgnosticEncoder, AgnosticEncoders, Codec, JavaSerializationCodec, KryoSerializationCodec}
import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{ArrayEncoder, BoxedLeafEncoder, DateEncoder, DayTimeIntervalEncoder, InstantEncoder, IterableEncoder, JavaBeanEncoder, JavaBigIntEncoder, JavaDecimalEncoder, JavaEnumEncoder, LocalDateEncoder, LocalDateTimeEncoder, MapEncoder, OptionEncoder, PrimitiveBooleanEncoder, PrimitiveByteEncoder, PrimitiveDoubleEncoder, PrimitiveFloatEncoder, PrimitiveIntEncoder, PrimitiveLongEncoder, PrimitiveShortEncoder, ProductEncoder, ScalaBigIntEncoder, ScalaDecimalEncoder, ScalaEnumEncoder, StringEncoder, TimestampEncoder, TransformingEncoder, UDTEncoder, YearMonthIntervalEncoder}
import org.apache.spark.sql.catalyst.encoders.EncoderUtils.{externalDataTypeFor, isNativeEncoder}
import org.apache.spark.sql.catalyst.expressions.{Expression, GetStructField, IsNull, Literal, MapKeys, MapValues, UpCast}
-import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, CreateExternalRow, InitializeJavaBean, Invoke, NewInstance, StaticInvoke, UnresolvedCatalystToExternalMap, UnresolvedMapObjects, WrapOption}
+import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, CreateExternalRow, DecodeUsingSerializer, InitializeJavaBean, Invoke, NewInstance, StaticInvoke, UnresolvedCatalystToExternalMap, UnresolvedMapObjects, WrapOption}
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, DateTimeUtils, IntervalUtils}
import org.apache.spark.sql.types._
@@ -411,6 +411,12 @@ object DeserializerBuildHelper {
val result = InitializeJavaBean(newInstance, setters.toMap)
exprs.If(IsNull(path), exprs.Literal.create(null, ObjectType(cls)), result)
+ case TransformingEncoder(tag, _, codec) if codec == JavaSerializationCodec =>
+ DecodeUsingSerializer(path, tag, kryo = false)
+
+ case TransformingEncoder(tag, _, codec) if codec == KryoSerializationCodec =>
+ DecodeUsingSerializer(path, tag, kryo = true)
+
case TransformingEncoder(tag, encoder, provider) =>
Invoke(
Literal.create(provider(), ObjectType(classOf[Codec[_, _]])),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala
index 408bd65333cac..20cf80e88e42a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ProjectingInternalRow.scala
@@ -26,7 +26,8 @@ import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String, VariantVal}
* An [[InternalRow]] that projects particular columns from another [[InternalRow]] without copying
* the underlying data.
*/
-case class ProjectingInternalRow(schema: StructType, colOrdinals: Seq[Int]) extends InternalRow {
+case class ProjectingInternalRow(schema: StructType,
+ colOrdinals: IndexedSeq[Int]) extends InternalRow {
assert(schema.size == colOrdinals.size)
private var row: InternalRow = _
@@ -116,3 +117,9 @@ case class ProjectingInternalRow(schema: StructType, colOrdinals: Seq[Int]) exte
row.get(colOrdinals(ordinal), dataType)
}
}
+
+object ProjectingInternalRow {
+ def apply(schema: StructType, colOrdinals: Seq[Int]): ProjectingInternalRow = {
+ new ProjectingInternalRow(schema, colOrdinals.toIndexedSeq)
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
index 38bf0651d6f1c..daebe15c298f6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
@@ -21,7 +21,7 @@ import scala.language.existentials
import org.apache.spark.sql.catalyst.{expressions => exprs}
import org.apache.spark.sql.catalyst.DeserializerBuildHelper.expressionWithNullSafety
-import org.apache.spark.sql.catalyst.encoders.{AgnosticEncoder, AgnosticEncoders, Codec}
+import org.apache.spark.sql.catalyst.encoders.{AgnosticEncoder, AgnosticEncoders, Codec, JavaSerializationCodec, KryoSerializationCodec}
import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{ArrayEncoder, BoxedBooleanEncoder, BoxedByteEncoder, BoxedDoubleEncoder, BoxedFloatEncoder, BoxedIntEncoder, BoxedLeafEncoder, BoxedLongEncoder, BoxedShortEncoder, DateEncoder, DayTimeIntervalEncoder, InstantEncoder, IterableEncoder, JavaBeanEncoder, JavaBigIntEncoder, JavaDecimalEncoder, JavaEnumEncoder, LocalDateEncoder, LocalDateTimeEncoder, MapEncoder, OptionEncoder, PrimitiveLeafEncoder, ProductEncoder, ScalaBigIntEncoder, ScalaDecimalEncoder, ScalaEnumEncoder, StringEncoder, TimestampEncoder, TransformingEncoder, UDTEncoder, YearMonthIntervalEncoder}
import org.apache.spark.sql.catalyst.encoders.EncoderUtils.{externalDataTypeFor, isNativeEncoder, lenientExternalDataTypeFor}
import org.apache.spark.sql.catalyst.expressions.{BoundReference, CheckOverflow, CreateNamedStruct, Expression, IsNull, KnownNotNull, Literal, UnsafeArrayData}
@@ -398,6 +398,12 @@ object SerializerBuildHelper {
}
createSerializerForObject(input, serializedFields)
+ case TransformingEncoder(_, _, codec) if codec == JavaSerializationCodec =>
+ EncodeUsingSerializer(input, kryo = false)
+
+ case TransformingEncoder(_, _, codec) if codec == KryoSerializationCodec =>
+ EncodeUsingSerializer(input, kryo = true)
+
case TransformingEncoder(_, encoder, codecProvider) =>
val encoded = Invoke(
Literal(codecProvider(), ObjectType(classOf[Codec[_, _]])),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 0164af945ca28..9e5b1d1254c87 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -26,7 +26,7 @@ import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Random, Success, Try}
-import org.apache.spark.{SparkException, SparkUnsupportedOperationException}
+import org.apache.spark.{SparkException, SparkThrowable, SparkUnsupportedOperationException}
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst._
import org.apache.spark.sql.catalyst.catalog._
@@ -50,6 +50,7 @@ import org.apache.spark.sql.connector.catalog.{View => _, _}
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
import org.apache.spark.sql.connector.catalog.TableChange.{After, ColumnPosition}
import org.apache.spark.sql.connector.catalog.functions.{AggregateFunction => V2AggregateFunction, ScalarFunction, UnboundFunction}
+import org.apache.spark.sql.connector.catalog.procedures.{BoundProcedure, ProcedureParameter, UnboundProcedure}
import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform}
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
@@ -310,6 +311,8 @@ class Analyzer(override val catalogManager: CatalogManager) extends RuleExecutor
ExtractGenerator ::
ResolveGenerate ::
ResolveFunctions ::
+ ResolveProcedures ::
+ BindProcedures ::
ResolveTableSpec ::
ResolveAliases ::
ResolveSubquery ::
@@ -2611,6 +2614,66 @@ class Analyzer(override val catalogManager: CatalogManager) extends RuleExecutor
}
}
+ /**
+ * A rule that resolves procedures.
+ */
+ object ResolveProcedures extends Rule[LogicalPlan] {
+ def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
+ _.containsPattern(UNRESOLVED_PROCEDURE), ruleId) {
+ case Call(UnresolvedProcedure(CatalogAndIdentifier(catalog, ident)), args, execute) =>
+ val procedureCatalog = catalog.asProcedureCatalog
+ val procedure = load(procedureCatalog, ident)
+ Call(ResolvedProcedure(procedureCatalog, ident, procedure), args, execute)
+ }
+
+ private def load(catalog: ProcedureCatalog, ident: Identifier): UnboundProcedure = {
+ try {
+ catalog.loadProcedure(ident)
+ } catch {
+ case e: Exception if !e.isInstanceOf[SparkThrowable] =>
+ val nameParts = catalog.name +: ident.asMultipartIdentifier
+ throw QueryCompilationErrors.failedToLoadRoutineError(nameParts, e)
+ }
+ }
+ }
+
+ /**
+ * A rule that binds procedures to the input types and rearranges arguments as needed.
+ */
+ object BindProcedures extends Rule[LogicalPlan] {
+ def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case Call(ResolvedProcedure(catalog, ident, unbound: UnboundProcedure), args, execute)
+ if args.forall(_.resolved) =>
+ val inputType = extractInputType(args)
+ val bound = unbound.bind(inputType)
+ validateParameterModes(bound)
+ val rearrangedArgs = NamedParametersSupport.defaultRearrange(bound, args)
+ Call(ResolvedProcedure(catalog, ident, bound), rearrangedArgs, execute)
+ }
+
+ private def extractInputType(args: Seq[Expression]): StructType = {
+ val fields = args.zipWithIndex.map {
+ case (NamedArgumentExpression(name, value), _) =>
+ StructField(name, value.dataType, value.nullable, byNameMetadata)
+ case (arg, index) =>
+ StructField(s"param$index", arg.dataType, arg.nullable)
+ }
+ StructType(fields)
+ }
+
+ private def byNameMetadata: Metadata = {
+ new MetadataBuilder()
+ .putBoolean(ProcedureParameter.BY_NAME_METADATA_KEY, value = true)
+ .build()
+ }
+
+ private def validateParameterModes(procedure: BoundProcedure): Unit = {
+ procedure.parameters.find(_.mode != ProcedureParameter.Mode.IN).foreach { param =>
+ throw SparkException.internalError(s"Unsupported parameter mode: ${param.mode}")
+ }
+ }
+ }
+
/**
* This rule resolves and rewrites subqueries inside expressions.
*
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala
index 17b1c4e249f57..3afe0ec8e9a7a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala
@@ -77,6 +77,7 @@ object AnsiTypeCoercion extends TypeCoercionBase {
override def typeCoercionRules: List[Rule[LogicalPlan]] =
UnpivotCoercion ::
WidenSetOperationTypes ::
+ ProcedureArgumentCoercion ::
new AnsiCombinedTypeCoercionRule(
CollationTypeCasts ::
InConversion ::
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index a9fbe548ba39e..5a9d5cd87ecc7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -250,6 +250,9 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
context = u.origin.getQueryContext,
summary = u.origin.context.summary)
+ case u: UnresolvedInlineTable if unresolvedInlineTableContainsScalarSubquery(u) =>
+ throw QueryCompilationErrors.inlineTableContainsScalarSubquery(u)
+
case command: V2PartitionCommand =>
command.table match {
case r @ ResolvedTable(_, _, table, _) => table match {
@@ -673,6 +676,14 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
varName,
c.defaultExpr.originalSQL)
+ case c: Call if c.resolved && c.bound && c.checkArgTypes().isFailure =>
+ c.checkArgTypes() match {
+ case mismatch: TypeCheckResult.DataTypeMismatch =>
+ c.dataTypeMismatch("CALL", mismatch)
+ case _ =>
+ throw SparkException.internalError("Invalid input for procedure")
+ }
+
case _ => // Falls back to the following checks
}
@@ -1559,6 +1570,15 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
case _ =>
}
}
+
+ private def unresolvedInlineTableContainsScalarSubquery(
+ unresolvedInlineTable: UnresolvedInlineTable) = {
+ unresolvedInlineTable.rows.exists { row =>
+ row.exists { expression =>
+ expression.exists(_.isInstanceOf[ScalarSubquery])
+ }
+ }
+ }
}
// a heap of the preempted error that only keeps the top priority element, representing the sole
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala
index 0fa11b9c45038..e22a4b941b30c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala
@@ -105,11 +105,15 @@ object DeduplicateRelations extends Rule[LogicalPlan] {
case p: LogicalPlan if p.isStreaming => (plan, false)
case m: MultiInstanceRelation =>
- deduplicateAndRenew[LogicalPlan with MultiInstanceRelation](
- existingRelations,
- m,
- _.output.map(_.exprId.id),
- node => node.newInstance().asInstanceOf[LogicalPlan with MultiInstanceRelation])
+ val planWrapper = RelationWrapper(m.getClass, m.output.map(_.exprId.id))
+ if (existingRelations.contains(planWrapper)) {
+ val newNode = m.newInstance()
+ newNode.copyTagsFrom(m)
+ (newNode, true)
+ } else {
+ existingRelations.add(planWrapper)
+ (m, false)
+ }
case p: Project =>
deduplicateAndRenew[Project](
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index dfe1bd12bb7ff..d03d8114e9976 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -384,7 +384,9 @@ object FunctionRegistry {
expression[Rand]("rand"),
expression[Rand]("random", true, Some("3.0.0")),
expression[Randn]("randn"),
+ expression[RandStr]("randstr"),
expression[Stack]("stack"),
+ expression[Uniform]("uniform"),
expression[ZeroIfNull]("zeroifnull"),
CaseWhen.registryEntry,
@@ -839,6 +841,7 @@ object FunctionRegistry {
expressionBuilder("try_variant_get", TryVariantGetExpressionBuilder),
expression[SchemaOfVariant]("schema_of_variant"),
expression[SchemaOfVariantAgg]("schema_of_variant_agg"),
+ expression[ToVariantObject]("to_variant_object"),
// cast
expression[Cast]("cast"),
@@ -1157,6 +1160,7 @@ object TableFunctionRegistry {
generator[PosExplode]("posexplode"),
generator[PosExplode]("posexplode_outer", outer = true),
generator[Stack]("stack"),
+ generator[Collations]("collations"),
generator[SQLKeywords]("sql_keywords"),
generator[VariantExplode]("variant_explode"),
generator[VariantExplode]("variant_explode_outer", outer = true)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index 08c5b3531b4c8..5983346ff1e27 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.trees.AlwaysProcess
import org.apache.spark.sql.catalyst.types.DataTypeUtils
+import org.apache.spark.sql.connector.catalog.procedures.BoundProcedure
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.types.{AbstractArrayType, AbstractMapType, AbstractStringType, StringTypeAnyCollation}
@@ -202,6 +203,20 @@ abstract class TypeCoercionBase {
}
}
+ /**
+ * A type coercion rule that implicitly casts procedure arguments to expected types.
+ */
+ object ProcedureArgumentCoercion extends Rule[LogicalPlan] {
+ override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case c @ Call(ResolvedProcedure(_, _, procedure: BoundProcedure), args, _) if c.resolved =>
+ val expectedDataTypes = procedure.parameters.map(_.dataType)
+ val coercedArgs = args.zip(expectedDataTypes).map {
+ case (arg, expectedType) => implicitCast(arg, expectedType).getOrElse(arg)
+ }
+ c.copy(args = coercedArgs)
+ }
+ }
+
/**
* Widens the data types of the [[Unpivot]] values.
*/
@@ -838,6 +853,7 @@ object TypeCoercion extends TypeCoercionBase {
override def typeCoercionRules: List[Rule[LogicalPlan]] =
UnpivotCoercion ::
WidenSetOperationTypes ::
+ ProcedureArgumentCoercion ::
new CombinedTypeCoercionRule(
CollationTypeCasts ::
InConversion ::
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/package.scala
index c0689eb121679..daab9e4d78bf5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/package.scala
@@ -67,9 +67,13 @@ package object analysis {
}
def dataTypeMismatch(expr: Expression, mismatch: DataTypeMismatch): Nothing = {
+ dataTypeMismatch(toSQLExpr(expr), mismatch)
+ }
+
+ def dataTypeMismatch(sqlExpr: String, mismatch: DataTypeMismatch): Nothing = {
throw new AnalysisException(
errorClass = s"DATATYPE_MISMATCH.${mismatch.errorSubClass}",
- messageParameters = mismatch.messageParameters + ("sqlExpr" -> toSQLExpr(expr)),
+ messageParameters = mismatch.messageParameters + ("sqlExpr" -> sqlExpr),
origin = t.origin)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 60d979e9c7afb..6f445b1e88d70 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -974,3 +974,13 @@ case object UnresolvedWithinGroup extends LeafExpression with Unevaluable {
override def dataType: DataType = throw new UnresolvedException("dataType")
override lazy val resolved = false
}
+
+case class UnresolvedTranspose(
+ indices: Seq[Expression],
+ child: LogicalPlan
+) extends UnresolvedUnaryNode {
+ final override val nodePatterns: Seq[TreePattern] = Seq(UNRESOLVED_TRANSPOSE)
+
+ override protected def withNewChildInternal(newChild: LogicalPlan): UnresolvedTranspose =
+ copy(child = newChild)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala
index ecdf40e87a894..dee78b8f03af4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/v2ResolutionPlans.scala
@@ -23,13 +23,14 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.catalyst.expressions.{Attribute, LeafExpression, Unevaluable}
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, Statistics}
-import org.apache.spark.sql.catalyst.trees.TreePattern.{TreePattern, UNRESOLVED_FUNC}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{TreePattern, UNRESOLVED_FUNC, UNRESOLVED_PROCEDURE}
import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes
import org.apache.spark.sql.catalyst.util.CharVarcharUtils
-import org.apache.spark.sql.connector.catalog.{CatalogPlugin, FunctionCatalog, Identifier, Table, TableCatalog}
+import org.apache.spark.sql.connector.catalog.{CatalogPlugin, FunctionCatalog, Identifier, ProcedureCatalog, Table, TableCatalog}
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition
import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
+import org.apache.spark.sql.connector.catalog.procedures.Procedure
import org.apache.spark.sql.types.{DataType, StructField}
import org.apache.spark.sql.util.CaseInsensitiveStringMap
import org.apache.spark.util.ArrayImplicits._
@@ -135,6 +136,12 @@ case class UnresolvedFunctionName(
case class UnresolvedIdentifier(nameParts: Seq[String], allowTemp: Boolean = false)
extends UnresolvedLeafNode
+/**
+ * A procedure identifier that should be resolved into [[ResolvedProcedure]].
+ */
+case class UnresolvedProcedure(nameParts: Seq[String]) extends UnresolvedLeafNode {
+ final override val nodePatterns: Seq[TreePattern] = Seq(UNRESOLVED_PROCEDURE)
+}
/**
* A resolved leaf node whose statistics has no meaning.
@@ -192,6 +199,12 @@ case class ResolvedFieldName(path: Seq[String], field: StructField) extends Fiel
case class ResolvedFieldPosition(position: ColumnPosition) extends FieldPosition
+case class ResolvedProcedure(
+ catalog: ProcedureCatalog,
+ ident: Identifier,
+ procedure: Procedure) extends LeafNodeWithoutStats {
+ override def output: Seq[Attribute] = Nil
+}
/**
* A plan containing resolved persistent views.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index 2c27da3cf6e15..5444ab6845867 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -21,12 +21,12 @@ import java.util.Locale
import scala.util.control.Exception.allCatch
+import org.apache.spark.SparkException
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.analysis.TypeCoercion
import org.apache.spark.sql.catalyst.expressions.ExprUtils
import org.apache.spark.sql.catalyst.util.{DateFormatter, TimestampFormatter}
import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
-import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
@@ -138,7 +138,7 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable {
case BooleanType => tryParseBoolean(field)
case StringType => StringType
case other: DataType =>
- throw QueryExecutionErrors.dataTypeUnexpectedError(other)
+ throw SparkException.internalError(s"Unexpected data type $other")
}
compatibleType(typeSoFar, typeElemInfer).getOrElse(StringType)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
index 0b5ce65fed6df..d7d53230470d9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.{DeserializerBuildHelper, InternalRow, Java
import org.apache.spark.sql.catalyst.analysis.{Analyzer, GetColumnByOrdinal, SimpleAnalyzer, UnresolvedAttribute, UnresolvedExtractValue}
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.{Deserializer, Serializer}
import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, InitializeJavaBean, Invoke, NewInstance}
+import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, InitializeJavaBean, NewInstance}
import org.apache.spark.sql.catalyst.optimizer.{ReassignLambdaVariableID, SimplifyCasts}
import org.apache.spark.sql.catalyst.plans.logical.{CatalystSerde, DeserializeToObject, LeafNode, LocalRelation}
import org.apache.spark.sql.catalyst.types.DataTypeUtils
@@ -54,9 +54,9 @@ object ExpressionEncoder {
def apply[T](enc: AgnosticEncoder[T]): ExpressionEncoder[T] = {
new ExpressionEncoder[T](
+ enc,
SerializerBuildHelper.createSerializer(enc),
- DeserializerBuildHelper.createDeserializer(enc),
- enc.clsTag)
+ DeserializerBuildHelper.createDeserializer(enc))
}
def apply(schema: StructType): ExpressionEncoder[Row] = apply(schema, lenient = false)
@@ -70,107 +70,6 @@ object ExpressionEncoder {
apply(JavaTypeInference.encoderFor(beanClass))
}
- /**
- * Given a set of N encoders, constructs a new encoder that produce objects as items in an
- * N-tuple. Note that these encoders should be unresolved so that information about
- * name/positional binding is preserved.
- * When `useNullSafeDeserializer` is true, the deserialization result for a child will be null if
- * the input is null. It is false by default as most deserializers handle null input properly and
- * don't require an extra null check. Some of them are null-tolerant, such as the deserializer for
- * `Option[T]`, and we must not set it to true in this case.
- */
- def tuple(
- encoders: Seq[ExpressionEncoder[_]],
- useNullSafeDeserializer: Boolean = false): ExpressionEncoder[_] = {
- if (encoders.length > 22) {
- throw QueryExecutionErrors.elementsOfTupleExceedLimitError()
- }
-
- encoders.foreach(_.assertUnresolved())
-
- val cls = Utils.getContextOrSparkClassLoader.loadClass(s"scala.Tuple${encoders.size}")
-
- val newSerializerInput = BoundReference(0, ObjectType(cls), nullable = true)
- val serializers = encoders.zipWithIndex.map { case (enc, index) =>
- val boundRefs = enc.objSerializer.collect { case b: BoundReference => b }.distinct
- assert(boundRefs.size == 1, "object serializer should have only one bound reference but " +
- s"there are ${boundRefs.size}")
-
- val originalInputObject = boundRefs.head
- val newInputObject = Invoke(
- newSerializerInput,
- s"_${index + 1}",
- originalInputObject.dataType,
- returnNullable = originalInputObject.nullable)
-
- val newSerializer = enc.objSerializer.transformUp {
- case BoundReference(0, _, _) => newInputObject
- }
-
- Alias(newSerializer, s"_${index + 1}")()
- }
- val newSerializer = CreateStruct(serializers)
-
- def nullSafe(input: Expression, result: Expression): Expression = {
- If(IsNull(input), Literal.create(null, result.dataType), result)
- }
-
- val newDeserializerInput = GetColumnByOrdinal(0, newSerializer.dataType)
- val childrenDeserializers = encoders.zipWithIndex.map { case (enc, index) =>
- val getColExprs = enc.objDeserializer.collect { case c: GetColumnByOrdinal => c }.distinct
- assert(getColExprs.size == 1, "object deserializer should have only one " +
- s"`GetColumnByOrdinal`, but there are ${getColExprs.size}")
-
- val input = GetStructField(newDeserializerInput, index)
- val childDeserializer = enc.objDeserializer.transformUp {
- case GetColumnByOrdinal(0, _) => input
- }
-
- if (useNullSafeDeserializer && enc.objSerializer.nullable) {
- nullSafe(input, childDeserializer)
- } else {
- childDeserializer
- }
- }
- val newDeserializer =
- NewInstance(cls, childrenDeserializers, ObjectType(cls), propagateNull = false)
-
- new ExpressionEncoder[Any](
- nullSafe(newSerializerInput, newSerializer),
- nullSafe(newDeserializerInput, newDeserializer),
- ClassTag(cls))
- }
-
- // Tuple1
- def tuple[T](e: ExpressionEncoder[T]): ExpressionEncoder[Tuple1[T]] =
- tuple(Seq(e)).asInstanceOf[ExpressionEncoder[Tuple1[T]]]
-
- def tuple[T1, T2](
- e1: ExpressionEncoder[T1],
- e2: ExpressionEncoder[T2]): ExpressionEncoder[(T1, T2)] =
- tuple(Seq(e1, e2)).asInstanceOf[ExpressionEncoder[(T1, T2)]]
-
- def tuple[T1, T2, T3](
- e1: ExpressionEncoder[T1],
- e2: ExpressionEncoder[T2],
- e3: ExpressionEncoder[T3]): ExpressionEncoder[(T1, T2, T3)] =
- tuple(Seq(e1, e2, e3)).asInstanceOf[ExpressionEncoder[(T1, T2, T3)]]
-
- def tuple[T1, T2, T3, T4](
- e1: ExpressionEncoder[T1],
- e2: ExpressionEncoder[T2],
- e3: ExpressionEncoder[T3],
- e4: ExpressionEncoder[T4]): ExpressionEncoder[(T1, T2, T3, T4)] =
- tuple(Seq(e1, e2, e3, e4)).asInstanceOf[ExpressionEncoder[(T1, T2, T3, T4)]]
-
- def tuple[T1, T2, T3, T4, T5](
- e1: ExpressionEncoder[T1],
- e2: ExpressionEncoder[T2],
- e3: ExpressionEncoder[T3],
- e4: ExpressionEncoder[T4],
- e5: ExpressionEncoder[T5]): ExpressionEncoder[(T1, T2, T3, T4, T5)] =
- tuple(Seq(e1, e2, e3, e4, e5)).asInstanceOf[ExpressionEncoder[(T1, T2, T3, T4, T5)]]
-
private val anyObjectType = ObjectType(classOf[Any])
/**
@@ -228,6 +127,7 @@ object ExpressionEncoder {
* A generic encoder for JVM objects that uses Catalyst Expressions for a `serializer`
* and a `deserializer`.
*
+ * @param encoder the `AgnosticEncoder` for type `T`.
* @param objSerializer An expression that can be used to encode a raw object to corresponding
* Spark SQL representation that can be a primitive column, array, map or a
* struct. This represents how Spark SQL generally serializes an object of
@@ -236,13 +136,15 @@ object ExpressionEncoder {
* representation. This represents how Spark SQL generally deserializes
* a serialized value in Spark SQL representation back to an object of
* type `T`.
- * @param clsTag A classtag for `T`.
*/
case class ExpressionEncoder[T](
+ encoder: AgnosticEncoder[T],
objSerializer: Expression,
- objDeserializer: Expression,
- clsTag: ClassTag[T])
- extends Encoder[T] {
+ objDeserializer: Expression)
+ extends Encoder[T]
+ with ToAgnosticEncoder[T] {
+
+ override def clsTag: ClassTag[T] = encoder.clsTag
/**
* A sequence of expressions, one for each top-level field that can be used to
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CannotReplaceMissingTableException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/KryoSerializationCodecImpl.scala
similarity index 57%
rename from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CannotReplaceMissingTableException.scala
rename to sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/KryoSerializationCodecImpl.scala
index f3e0c0aca29ca..49c7b41f77472 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CannotReplaceMissingTableException.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/KryoSerializationCodecImpl.scala
@@ -14,20 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.spark.sql.catalyst.encoders
+import java.nio.ByteBuffer
-package org.apache.spark.sql.catalyst.analysis
+import org.apache.spark.sql.catalyst.expressions.objects.SerializerSupport
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.util.quoteNameParts
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.util.ArrayImplicits._
+/**
+ * A codec that uses Kryo to (de)serialize arbitrary objects to and from a byte array.
+ */
+class KryoSerializationCodecImpl extends Codec[Any, Array[Byte]] {
+ private val serializer = SerializerSupport.newSerializer(useKryo = true)
+ override def encode(in: Any): Array[Byte] =
+ serializer.serialize(in).array()
-class CannotReplaceMissingTableException(
- tableIdentifier: Identifier,
- cause: Option[Throwable] = None)
- extends AnalysisException(
- errorClass = "TABLE_OR_VIEW_NOT_FOUND",
- messageParameters = Map("relationName"
- -> quoteNameParts((tableIdentifier.namespace :+ tableIdentifier.name).toImmutableArraySeq)),
- cause = cause)
+ override def decode(out: Array[Byte]): Any =
+ serializer.deserialize(ByteBuffer.wrap(out))
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala
index de1122da646b7..c226e48c6be5e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala
@@ -21,7 +21,7 @@ import org.apache.spark.sql.internal.SQLConf
// scalastyle:off line.size.limit
@ExpressionDescription(
- usage = "Usage: input [NOT] BETWEEN lower AND upper - evaluate if `input` is [not] in between `lower` and `upper`",
+ usage = "input [NOT] _FUNC_ lower AND upper - evaluate if `input` is [not] in between `lower` and `upper`",
examples = """
Examples:
> SELECT 0.5 _FUNC_ 0.1 AND 1.0;
@@ -33,7 +33,7 @@ import org.apache.spark.sql.internal.SQLConf
* lower - Lower bound of the between check.
* upper - Upper bound of the between check.
""",
- since = "4.0.0",
+ since = "1.0.0",
group = "conditional_funcs")
case class Between private(input: Expression, lower: Expression, upper: Expression, replacement: Expression)
extends RuntimeReplaceable with InheritAnalysisRules {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 4a2b4b28e690e..7a2799e99fe2d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -128,7 +128,10 @@ object Cast extends QueryErrorsBase {
case (TimestampType, _: NumericType) => true
case (VariantType, _) => variant.VariantGet.checkDataType(to)
- case (_, VariantType) => variant.VariantGet.checkDataType(from)
+ // Structs and Maps can't be cast to Variants since the Variant spec does not yet contain
+ // lossless equivalents for these types. The `to_variant_object` expression can be used instead
+ // to convert data of these types to Variant Objects.
+ case (_, VariantType) => variant.VariantGet.checkDataType(from, allowStructsAndMaps = false)
case (ArrayType(fromType, fn), ArrayType(toType, tn)) =>
canAnsiCast(fromType, toType) && resolvableNullability(fn, tn)
@@ -237,7 +240,10 @@ object Cast extends QueryErrorsBase {
case (_: NumericType, _: NumericType) => true
case (VariantType, _) => variant.VariantGet.checkDataType(to)
- case (_, VariantType) => variant.VariantGet.checkDataType(from)
+ // Structs and Maps can't be cast to Variants since the Variant spec does not yet contain
+ // lossless equivalents for these types. The `to_variant_object` expression can be used instead
+ // to convert data of these types to Variant Objects.
+ case (_, VariantType) => variant.VariantGet.checkDataType(from, allowStructsAndMaps = false)
case (ArrayType(fromType, fn), ArrayType(toType, tn)) =>
canCast(fromType, toType) &&
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PipeSelect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PipeSelect.scala
new file mode 100644
index 0000000000000..0b5479cc8f0ee
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PipeSelect.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction
+import org.apache.spark.sql.catalyst.trees.TreePattern.{PIPE_OPERATOR_SELECT, RUNTIME_REPLACEABLE, TreePattern}
+import org.apache.spark.sql.errors.QueryCompilationErrors
+
+/**
+ * Represents a SELECT clause when used with the |> SQL pipe operator.
+ * We use this to make sure that no aggregate functions exist in the SELECT expressions.
+ */
+case class PipeSelect(child: Expression)
+ extends UnaryExpression with RuntimeReplaceable {
+ final override val nodePatterns: Seq[TreePattern] = Seq(PIPE_OPERATOR_SELECT, RUNTIME_REPLACEABLE)
+ override def withNewChildInternal(newChild: Expression): Expression = PipeSelect(newChild)
+ override lazy val replacement: Expression = {
+ def visit(e: Expression): Unit = e match {
+ case a: AggregateFunction =>
+ // If we used the pipe operator |> SELECT clause to specify an aggregate function, this is
+ // invalid; return an error message instructing the user to use the pipe operator
+ // |> AGGREGATE clause for this purpose instead.
+ throw QueryCompilationErrors.pipeOperatorSelectContainsAggregateFunction(a)
+ case _: WindowExpression =>
+ // Window functions are allowed in pipe SELECT operators, so do not traverse into children.
+ case _ =>
+ e.children.foreach(visit)
+ }
+ visit(child)
+ child
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
index 4987e31b49911..8ad062ab0e2f9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
@@ -21,6 +21,7 @@ import java.nio.ByteBuffer
import com.google.common.primitives.{Doubles, Ints, Longs}
+import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{DataTypeMismatch, TypeCheckSuccess}
@@ -32,7 +33,6 @@ import org.apache.spark.sql.catalyst.types.PhysicalNumericType
import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
import org.apache.spark.sql.catalyst.util.QuantileSummaries
import org.apache.spark.sql.catalyst.util.QuantileSummaries.{defaultCompressThreshold, Stats}
-import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.types._
import org.apache.spark.util.ArrayImplicits._
@@ -189,7 +189,7 @@ case class ApproximatePercentile(
PhysicalNumericType.numeric(n)
.toDouble(value.asInstanceOf[PhysicalNumericType#InternalType])
case other: DataType =>
- throw QueryExecutionErrors.dataTypeUnexpectedError(other)
+ throw SparkException.internalError(s"Unexpected data type $other")
}
buffer.add(doubleValue)
}
@@ -214,7 +214,7 @@ case class ApproximatePercentile(
case DoubleType => doubleResult
case _: DecimalType => doubleResult.map(Decimal(_))
case other: DataType =>
- throw QueryExecutionErrors.dataTypeUnexpectedError(other)
+ throw SparkException.internalError(s"Unexpected data type $other")
}
if (result.length == 0) {
null
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HistogramNumeric.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HistogramNumeric.scala
index ba26c5a1022d0..eda2c742ab4b5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HistogramNumeric.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HistogramNumeric.scala
@@ -143,7 +143,8 @@ case class HistogramNumeric(
if (buffer.getUsedBins < 1) {
null
} else {
- val result = (0 until buffer.getUsedBins).map { index =>
+ val array = new Array[AnyRef](buffer.getUsedBins)
+ (0 until buffer.getUsedBins).foreach { index =>
// Note that the 'coord.x' and 'coord.y' have double-precision floating point type here.
val coord = buffer.getBin(index)
if (propagateInputType) {
@@ -163,16 +164,16 @@ case class HistogramNumeric(
coord.x.toLong
case _ => coord.x
}
- InternalRow.apply(result, coord.y)
+ array(index) = InternalRow.apply(result, coord.y)
} else {
// Otherwise, just apply the double-precision values in 'coord.x' and 'coord.y' to the
// output row directly. In this case: 'SELECT histogram_numeric(val, 3)
// FROM VALUES (0L), (1L), (2L), (10L) AS tab(col)' returns an array of structs where the
// first field has DoubleType.
- InternalRow.apply(coord.x, coord.y)
+ array(index) = InternalRow.apply(coord.x, coord.y)
}
}
- new GenericArrayData(result)
+ new GenericArrayData(array)
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
index e77622a26d90a..c593c8bfb8341 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
@@ -170,7 +170,7 @@ case class CollectSet(
override def eval(buffer: mutable.HashSet[Any]): Any = {
val array = child.dataType match {
case BinaryType =>
- buffer.iterator.map(_.asInstanceOf[ArrayData].toByteArray()).toArray
+ buffer.iterator.map(_.asInstanceOf[ArrayData].toByteArray()).toArray[Any]
case _ => buffer.toArray
}
new GenericArrayData(array)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 375a2bde59230..5cdd3c7eb62d1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -433,7 +433,7 @@ case class ArraysZip(children: Seq[Expression], names: Seq[Expression])
inputArrays.map(_.numElements()).max
}
- val result = new Array[InternalRow](biggestCardinality)
+ val result = new Array[AnyRef](biggestCardinality)
val zippedArrs: Seq[(ArrayData, Int)] = inputArrays.zipWithIndex
for (i <- 0 until biggestCardinality) {
@@ -1058,20 +1058,26 @@ case class SortArray(base: Expression, ascendingOrder: Expression)
override def checkInputDataTypes(): TypeCheckResult = base.dataType match {
case ArrayType(dt, _) if RowOrdering.isOrderable(dt) =>
- ascendingOrder match {
- case Literal(_: Boolean, BooleanType) =>
- TypeCheckResult.TypeCheckSuccess
- case _ =>
- DataTypeMismatch(
- errorSubClass = "UNEXPECTED_INPUT_TYPE",
- messageParameters = Map(
- "paramIndex" -> ordinalNumber(1),
- "requiredType" -> toSQLType(BooleanType),
- "inputSql" -> toSQLExpr(ascendingOrder),
- "inputType" -> toSQLType(ascendingOrder.dataType))
- )
+ if (!ascendingOrder.foldable) {
+ DataTypeMismatch(
+ errorSubClass = "NON_FOLDABLE_INPUT",
+ messageParameters = Map(
+ "inputName" -> toSQLId("ascendingOrder"),
+ "inputType" -> toSQLType(ascendingOrder.dataType),
+ "inputExpr" -> toSQLExpr(ascendingOrder)))
+ } else if (ascendingOrder.dataType != BooleanType) {
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> ordinalNumber(1),
+ "requiredType" -> toSQLType(BooleanType),
+ "inputSql" -> toSQLExpr(ascendingOrder),
+ "inputType" -> toSQLType(ascendingOrder.dataType))
+ )
+ } else {
+ TypeCheckResult.TypeCheckSuccess
}
- case ArrayType(dt, _) =>
+ case ArrayType(_, _) =>
DataTypeMismatch(
errorSubClass = "INVALID_ORDERING_TYPE",
messageParameters = Map(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index ba1beab28d9a7..b8b47f2763f5b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.catalyst.trees.TreePattern._
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.types.StringTypeAnyCollation
+import org.apache.spark.sql.internal.types.StringTypeNonCSAICollation
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.ArrayImplicits._
@@ -579,7 +579,7 @@ case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: E
override def third: Expression = keyValueDelim
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeAnyCollation, StringTypeAnyCollation, StringTypeAnyCollation)
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation)
override def dataType: DataType = MapType(first.dataType, first.dataType)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
index ca74fefb9c032..56ecbf550e45c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{FunctionRegistryBase, TypeCheckResult, TypeCoercion}
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
+import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.trees.TernaryLike
@@ -188,6 +189,13 @@ case class CaseWhen(
}
override def nullable: Boolean = {
+ if (branches.exists(_._1 == TrueLiteral)) {
+ // if any of the branch is always true
+ // nullability check should only be related to branches
+ // before the TrueLiteral and value of the first TrueLiteral branch
+ val (h, t) = branches.span(_._1 != TrueLiteral)
+ return h.exists(_._2.nullable) || t.head._2.nullable
+ }
// Result is nullable if any of the branch is nullable, or if the else value is nullable
branches.exists(_._2.nullable) || elseValue.map(_.nullable).getOrElse(true)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 2cc88a25f465d..dc58352a1b362 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.catalyst.expressions
import scala.collection.mutable
+import scala.jdk.CollectionConverters.CollectionHasAsScala
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
@@ -28,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.plans.logical.{FunctionSignature, InputParameter}
import org.apache.spark.sql.catalyst.trees.TreePattern.{GENERATOR, TreePattern}
-import org.apache.spark.sql.catalyst.util.{ArrayData, MapData}
+import org.apache.spark.sql.catalyst.util.{ArrayData, CollationFactory, MapData}
import org.apache.spark.sql.catalyst.util.SQLKeywordUtils._
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
import org.apache.spark.sql.internal.SQLConf
@@ -618,3 +619,44 @@ case class SQLKeywords() extends LeafExpression with Generator with CodegenFallb
override def prettyName: String = "sql_keywords"
}
+
+@ExpressionDescription(
+ usage = """_FUNC_() - Get all of the Spark SQL string collations""",
+ examples = """
+ Examples:
+ > SELECT * FROM _FUNC_() WHERE NAME = 'UTF8_BINARY';
+ SYSTEM BUILTIN UTF8_BINARY NULL NULL ACCENT_SENSITIVE CASE_SENSITIVE NO_PAD NULL
+ """,
+ since = "4.0.0",
+ group = "generator_funcs")
+case class Collations() extends LeafExpression with Generator with CodegenFallback {
+ override def elementSchema: StructType = new StructType()
+ .add("CATALOG", StringType, nullable = false)
+ .add("SCHEMA", StringType, nullable = false)
+ .add("NAME", StringType, nullable = false)
+ .add("LANGUAGE", StringType)
+ .add("COUNTRY", StringType)
+ .add("ACCENT_SENSITIVITY", StringType, nullable = false)
+ .add("CASE_SENSITIVITY", StringType, nullable = false)
+ .add("PAD_ATTRIBUTE", StringType, nullable = false)
+ .add("ICU_VERSION", StringType)
+
+ override def eval(input: InternalRow): IterableOnce[InternalRow] = {
+ CollationFactory.listCollations().asScala.map(CollationFactory.loadCollationMeta).map { m =>
+ InternalRow(
+ UTF8String.fromString(m.catalog),
+ UTF8String.fromString(m.schema),
+ UTF8String.fromString(m.collationName),
+ UTF8String.fromString(m.language),
+ UTF8String.fromString(m.country),
+ UTF8String.fromString(
+ if (m.accentSensitivity) "ACCENT_SENSITIVE" else "ACCENT_INSENSITIVE"),
+ UTF8String.fromString(
+ if (m.caseSensitivity) "CASE_SENSITIVE" else "CASE_INSENSITIVE"),
+ UTF8String.fromString(m.padAttribute),
+ UTF8String.fromString(m.icuVersion))
+ }
+ }
+
+ override def prettyName: String = "collations"
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index 7005d663a3f96..2037eb22fede6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -632,7 +632,8 @@ case class JsonToStructs(
schema: DataType,
options: Map[String, String],
child: Expression,
- timeZoneId: Option[String] = None)
+ timeZoneId: Option[String] = None,
+ variantAllowDuplicateKeys: Boolean = SQLConf.get.getConf(SQLConf.VARIANT_ALLOW_DUPLICATE_KEYS))
extends UnaryExpression
with TimeZoneAwareExpression
with CodegenFallback
@@ -719,7 +720,8 @@ case class JsonToStructs(
override def nullSafeEval(json: Any): Any = nullableSchema match {
case _: VariantType =>
- VariantExpressionEvalUtils.parseJson(json.asInstanceOf[UTF8String])
+ VariantExpressionEvalUtils.parseJson(json.asInstanceOf[UTF8String],
+ allowDuplicateKeys = variantAllowDuplicateKeys)
case _ =>
converter(parser.parse(json.asInstanceOf[UTF8String]))
}
@@ -737,6 +739,12 @@ case class JsonToStructs(
copy(child = newChild)
}
+object JsonToStructs {
+ def unapply(
+ j: JsonToStructs): Option[(DataType, Map[String, String], Expression, Option[String])] =
+ Some((j.schema, j.options, j.child, j.timeZoneId))
+}
+
/**
* Converts a [[StructType]], [[ArrayType]] or [[MapType]] to a JSON output string.
*/
@@ -1072,7 +1080,7 @@ case class JsonObjectKeys(child: Expression) extends UnaryExpression with Codege
// skip all the children of inner object or array
parser.skipChildren()
}
- new GenericArrayData(arrayBufferOfKeys.toArray)
+ new GenericArrayData(arrayBufferOfKeys.toArray[Any])
}
override protected def withNewChildInternal(newChild: Expression): JsonObjectKeys =
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
index 00274a16b888b..ddba820414ae4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
@@ -1293,7 +1293,7 @@ sealed trait BitShiftOperation
* @param right number of bits to left shift.
*/
@ExpressionDescription(
- usage = "base << exp - Bitwise left shift.",
+ usage = "base _FUNC_ exp - Bitwise left shift.",
examples = """
Examples:
> SELECT shiftleft(2, 1);
@@ -1322,7 +1322,7 @@ case class ShiftLeft(left: Expression, right: Expression) extends BitShiftOperat
* @param right number of bits to right shift.
*/
@ExpressionDescription(
- usage = "base >> expr - Bitwise (signed) right shift.",
+ usage = "base _FUNC_ expr - Bitwise (signed) right shift.",
examples = """
Examples:
> SELECT shiftright(4, 1);
@@ -1350,7 +1350,7 @@ case class ShiftRight(left: Expression, right: Expression) extends BitShiftOpera
* @param right the number of bits to right shift.
*/
@ExpressionDescription(
- usage = "base >>> expr - Bitwise unsigned right shift.",
+ usage = "base _FUNC_ expr - Bitwise unsigned right shift.",
examples = """
Examples:
> SELECT shiftrightunsigned(4, 1);
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
index f5db972a28643..f329f8346b0de 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala
@@ -17,13 +17,18 @@
package org.apache.spark.sql.catalyst.expressions
+import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.UnresolvedSeed
+import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, UnresolvedSeed}
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
+import org.apache.spark.sql.catalyst.expressions.ExpectsInputTypes.{ordinalNumber, toSQLExpr, toSQLType}
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, FalseLiteral}
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.catalyst.trees.TreePattern.{EXPRESSION_WITH_RANDOM_SEED, TreePattern}
+import org.apache.spark.sql.catalyst.trees.{BinaryLike, TernaryLike, UnaryLike}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{EXPRESSION_WITH_RANDOM_SEED, RUNTIME_REPLACEABLE, TreePattern}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.random.XORShiftRandom
/**
@@ -33,8 +38,7 @@ import org.apache.spark.util.random.XORShiftRandom
*
* Since this expression is stateful, it cannot be a case object.
*/
-abstract class RDG extends UnaryExpression with ExpectsInputTypes with Nondeterministic
- with ExpressionWithRandomSeed {
+trait RDG extends Expression with ExpressionWithRandomSeed {
/**
* Record ID within each partition. By being transient, the Random Number Generator is
* reset every time we serialize and deserialize and initialize it.
@@ -43,12 +47,6 @@ abstract class RDG extends UnaryExpression with ExpectsInputTypes with Nondeterm
override def stateful: Boolean = true
- override protected def initializeInternal(partitionIndex: Int): Unit = {
- rng = new XORShiftRandom(seed + partitionIndex)
- }
-
- override def seedExpression: Expression = child
-
@transient protected lazy val seed: Long = seedExpression match {
case e if e.dataType == IntegerType => e.eval().asInstanceOf[Int]
case e if e.dataType == LongType => e.eval().asInstanceOf[Long]
@@ -57,6 +55,15 @@ abstract class RDG extends UnaryExpression with ExpectsInputTypes with Nondeterm
override def nullable: Boolean = false
override def dataType: DataType = DoubleType
+}
+
+abstract class NondeterministicUnaryRDG
+ extends RDG with UnaryLike[Expression] with Nondeterministic with ExpectsInputTypes {
+ override def seedExpression: Expression = child
+
+ override protected def initializeInternal(partitionIndex: Int): Unit = {
+ rng = new XORShiftRandom(seed + partitionIndex)
+ }
override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(IntegerType, LongType))
}
@@ -74,6 +81,7 @@ trait ExpressionWithRandomSeed extends Expression {
private[catalyst] object ExpressionWithRandomSeed {
def expressionToSeed(e: Expression, source: String): Option[Long] = e match {
+ case IntegerLiteral(seed) => Some(seed)
case LongLiteral(seed) => Some(seed)
case Literal(null, _) => None
case _ => throw QueryCompilationErrors.invalidRandomSeedParameter(source, e)
@@ -99,7 +107,7 @@ private[catalyst] object ExpressionWithRandomSeed {
since = "1.5.0",
group = "math_funcs")
// scalastyle:on line.size.limit
-case class Rand(child: Expression, hideSeed: Boolean = false) extends RDG {
+case class Rand(child: Expression, hideSeed: Boolean = false) extends NondeterministicUnaryRDG {
def this() = this(UnresolvedSeed, true)
@@ -150,7 +158,7 @@ object Rand {
since = "1.5.0",
group = "math_funcs")
// scalastyle:on line.size.limit
-case class Randn(child: Expression, hideSeed: Boolean = false) extends RDG {
+case class Randn(child: Expression, hideSeed: Boolean = false) extends NondeterministicUnaryRDG {
def this() = this(UnresolvedSeed, true)
@@ -181,3 +189,236 @@ case class Randn(child: Expression, hideSeed: Boolean = false) extends RDG {
object Randn {
def apply(seed: Long): Randn = Randn(Literal(seed, LongType))
}
+
+@ExpressionDescription(
+ usage = """
+ _FUNC_(min, max[, seed]) - Returns a random value with independent and identically
+ distributed (i.i.d.) values with the specified range of numbers. The random seed is optional.
+ The provided numbers specifying the minimum and maximum values of the range must be constant.
+ If both of these numbers are integers, then the result will also be an integer. Otherwise if
+ one or both of these are floating-point numbers, then the result will also be a floating-point
+ number.
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_(10, 20, 0) > 0 AS result;
+ true
+ """,
+ since = "4.0.0",
+ group = "math_funcs")
+case class Uniform(min: Expression, max: Expression, seedExpression: Expression)
+ extends RuntimeReplaceable with TernaryLike[Expression] with RDG {
+ def this(min: Expression, max: Expression) = this(min, max, UnresolvedSeed)
+
+ final override lazy val deterministic: Boolean = false
+ override val nodePatterns: Seq[TreePattern] =
+ Seq(RUNTIME_REPLACEABLE, EXPRESSION_WITH_RANDOM_SEED)
+
+ override val dataType: DataType = {
+ val first = min.dataType
+ val second = max.dataType
+ (min.dataType, max.dataType) match {
+ case _ if !seedExpression.resolved || seedExpression.dataType == NullType =>
+ NullType
+ case (_, NullType) | (NullType, _) => NullType
+ case (_, LongType) | (LongType, _)
+ if Seq(first, second).forall(integer) => LongType
+ case (_, IntegerType) | (IntegerType, _)
+ if Seq(first, second).forall(integer) => IntegerType
+ case (_, ShortType) | (ShortType, _)
+ if Seq(first, second).forall(integer) => ShortType
+ case (_, DoubleType) | (DoubleType, _) => DoubleType
+ case (_, FloatType) | (FloatType, _) => FloatType
+ case _ =>
+ throw SparkException.internalError(
+ s"Unexpected argument data types: ${min.dataType}, ${max.dataType}")
+ }
+ }
+
+ private def integer(t: DataType): Boolean = t match {
+ case _: ShortType | _: IntegerType | _: LongType => true
+ case _ => false
+ }
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ var result: TypeCheckResult = TypeCheckResult.TypeCheckSuccess
+ def requiredType = "integer or floating-point"
+ Seq((min, "min", 0),
+ (max, "max", 1),
+ (seedExpression, "seed", 2)).foreach {
+ case (expr: Expression, name: String, index: Int) =>
+ if (result == TypeCheckResult.TypeCheckSuccess) {
+ if (!expr.foldable) {
+ result = DataTypeMismatch(
+ errorSubClass = "NON_FOLDABLE_INPUT",
+ messageParameters = Map(
+ "inputName" -> name,
+ "inputType" -> requiredType,
+ "inputExpr" -> toSQLExpr(expr)))
+ } else expr.dataType match {
+ case _: ShortType | _: IntegerType | _: LongType | _: FloatType | _: DoubleType |
+ _: NullType =>
+ case _ =>
+ result = DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> ordinalNumber(index),
+ "requiredType" -> requiredType,
+ "inputSql" -> toSQLExpr(expr),
+ "inputType" -> toSQLType(expr.dataType)))
+ }
+ }
+ }
+ result
+ }
+
+ override def first: Expression = min
+ override def second: Expression = max
+ override def third: Expression = seedExpression
+
+ override def withNewSeed(newSeed: Long): Expression =
+ Uniform(min, max, Literal(newSeed, LongType))
+
+ override def withNewChildrenInternal(
+ newFirst: Expression, newSecond: Expression, newThird: Expression): Expression =
+ Uniform(newFirst, newSecond, newThird)
+
+ override def replacement: Expression = {
+ if (Seq(min, max, seedExpression).exists(_.dataType == NullType)) {
+ Literal(null)
+ } else {
+ def cast(e: Expression, to: DataType): Expression = if (e.dataType == to) e else Cast(e, to)
+ cast(Add(
+ cast(min, DoubleType),
+ Multiply(
+ Subtract(
+ cast(max, DoubleType),
+ cast(min, DoubleType)),
+ Rand(seed))),
+ dataType)
+ }
+ }
+}
+
+@ExpressionDescription(
+ usage = """
+ _FUNC_(length[, seed]) - Returns a string of the specified length whose characters are chosen
+ uniformly at random from the following pool of characters: 0-9, a-z, A-Z. The random seed is
+ optional. The string length must be a constant two-byte or four-byte integer (SMALLINT or INT,
+ respectively).
+ """,
+ examples =
+ """
+ Examples:
+ > SELECT _FUNC_(3, 0) AS result;
+ ceV
+ """,
+ since = "4.0.0",
+ group = "string_funcs")
+case class RandStr(length: Expression, override val seedExpression: Expression)
+ extends ExpressionWithRandomSeed with BinaryLike[Expression] with Nondeterministic {
+ def this(length: Expression) = this(length, UnresolvedSeed)
+
+ override def nullable: Boolean = false
+ override def dataType: DataType = StringType
+ override def stateful: Boolean = true
+ override def left: Expression = length
+ override def right: Expression = seedExpression
+
+ /**
+ * Record ID within each partition. By being transient, the Random Number Generator is
+ * reset every time we serialize and deserialize and initialize it.
+ */
+ @transient protected var rng: XORShiftRandom = _
+
+ @transient protected lazy val seed: Long = seedExpression match {
+ case e if e.dataType == IntegerType => e.eval().asInstanceOf[Int]
+ case e if e.dataType == LongType => e.eval().asInstanceOf[Long]
+ }
+ override protected def initializeInternal(partitionIndex: Int): Unit = {
+ rng = new XORShiftRandom(seed + partitionIndex)
+ }
+
+ override def withNewSeed(newSeed: Long): Expression = RandStr(length, Literal(newSeed, LongType))
+ override def withNewChildrenInternal(newFirst: Expression, newSecond: Expression): Expression =
+ RandStr(newFirst, newSecond)
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ var result: TypeCheckResult = TypeCheckResult.TypeCheckSuccess
+ def requiredType = "INT or SMALLINT"
+ Seq((length, "length", 0),
+ (seedExpression, "seedExpression", 1)).foreach {
+ case (expr: Expression, name: String, index: Int) =>
+ if (result == TypeCheckResult.TypeCheckSuccess) {
+ if (!expr.foldable) {
+ result = DataTypeMismatch(
+ errorSubClass = "NON_FOLDABLE_INPUT",
+ messageParameters = Map(
+ "inputName" -> name,
+ "inputType" -> requiredType,
+ "inputExpr" -> toSQLExpr(expr)))
+ } else expr.dataType match {
+ case _: ShortType | _: IntegerType =>
+ case _: LongType if index == 1 =>
+ case _ =>
+ result = DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> ordinalNumber(index),
+ "requiredType" -> requiredType,
+ "inputSql" -> toSQLExpr(expr),
+ "inputType" -> toSQLType(expr.dataType)))
+ }
+ }
+ }
+ result
+ }
+
+ override def evalInternal(input: InternalRow): Any = {
+ val numChars = length.eval(input).asInstanceOf[Number].intValue()
+ val bytes = new Array[Byte](numChars)
+ (0 until numChars).foreach { i =>
+ // We generate a random number between 0 and 61, inclusive. Between the 62 different choices
+ // we choose 0-9, a-z, or A-Z, where each category comprises 10 choices, 26 choices, or 26
+ // choices, respectively (10 + 26 + 26 = 62).
+ val num = (rng.nextInt() % 62).abs
+ num match {
+ case _ if num < 10 =>
+ bytes.update(i, ('0' + num).toByte)
+ case _ if num < 36 =>
+ bytes.update(i, ('a' + num - 10).toByte)
+ case _ =>
+ bytes.update(i, ('A' + num - 36).toByte)
+ }
+ }
+ val result: UTF8String = UTF8String.fromBytes(bytes.toArray)
+ result
+ }
+
+ override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+ val className = classOf[XORShiftRandom].getName
+ val rngTerm = ctx.addMutableState(className, "rng")
+ ctx.addPartitionInitializationStatement(
+ s"$rngTerm = new $className(${seed}L + partitionIndex);")
+ val eval = length.genCode(ctx)
+ ev.copy(code =
+ code"""
+ |${eval.code}
+ |int length = (int)(${eval.value});
+ |char[] chars = new char[length];
+ |for (int i = 0; i < length; i++) {
+ | int v = Math.abs($rngTerm.nextInt() % 62);
+ | if (v < 10) {
+ | chars[i] = (char)('0' + v);
+ | } else if (v < 36) {
+ | chars[i] = (char)('a' + (v - 10));
+ | } else {
+ | chars[i] = (char)('A' + (v - 36));
+ | }
+ |}
+ |UTF8String ${ev.value} = UTF8String.fromString(new String(chars));
+ |boolean ${ev.isNull} = false;
+ |""".stripMargin,
+ isNull = FalseLiteral)
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 6ccd5a451eafc..da6d786efb4e3 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.expressions
import java.nio.{ByteBuffer, CharBuffer}
import java.nio.charset.CharacterCodingException
-import java.text.{BreakIterator, DecimalFormat, DecimalFormatSymbols}
+import java.text.{DecimalFormat, DecimalFormatSymbols}
import java.util.{Base64 => JBase64, HashMap, Locale, Map => JMap}
import scala.collection.mutable.ArrayBuffer
@@ -38,7 +38,7 @@ import org.apache.spark.sql.catalyst.trees.TreePattern.{TreePattern, UPPER_OR_LO
import org.apache.spark.sql.catalyst.util.{ArrayData, CharsetProvider, CollationFactory, CollationSupport, GenericArrayData, TypeUtils}
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.types.{AbstractArrayType, StringTypeAnyCollation}
+import org.apache.spark.sql.internal.types.{AbstractArrayType, StringTypeAnyCollation, StringTypeNonCSAICollation}
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.UTF8StringBuilder
import org.apache.spark.unsafe.array.ByteArrayMethods
@@ -609,6 +609,8 @@ case class Contains(left: Expression, right: Expression) extends StringPredicate
defineCodeGen(ctx, ev, (c1, c2) =>
CollationSupport.Contains.genCode(c1, c2, collationId))
}
+ override def inputTypes : Seq[AbstractDataType] =
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): Contains = copy(left = newLeft, right = newRight)
}
@@ -650,6 +652,10 @@ case class StartsWith(left: Expression, right: Expression) extends StringPredica
defineCodeGen(ctx, ev, (c1, c2) =>
CollationSupport.StartsWith.genCode(c1, c2, collationId))
}
+
+ override def inputTypes : Seq[AbstractDataType] =
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): StartsWith = copy(left = newLeft, right = newRight)
}
@@ -691,6 +697,10 @@ case class EndsWith(left: Expression, right: Expression) extends StringPredicate
defineCodeGen(ctx, ev, (c1, c2) =>
CollationSupport.EndsWith.genCode(c1, c2, collationId))
}
+
+ override def inputTypes : Seq[AbstractDataType] =
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): EndsWith = copy(left = newLeft, right = newRight)
}
@@ -919,7 +929,7 @@ case class StringReplace(srcExpr: Expression, searchExpr: Expression, replaceExp
override def dataType: DataType = srcExpr.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeAnyCollation, StringTypeAnyCollation, StringTypeAnyCollation)
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation)
override def first: Expression = srcExpr
override def second: Expression = searchExpr
override def third: Expression = replaceExpr
@@ -1167,7 +1177,7 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac
override def dataType: DataType = srcExpr.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeAnyCollation, StringTypeAnyCollation, StringTypeAnyCollation)
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, StringTypeNonCSAICollation)
override def first: Expression = srcExpr
override def second: Expression = matchingExpr
override def third: Expression = replaceExpr
@@ -1394,6 +1404,9 @@ case class StringTrim(srcStr: Expression, trimStr: Option[Expression] = None)
override def doEval(srcString: UTF8String, trimString: UTF8String): UTF8String =
CollationSupport.StringTrim.exec(srcString, trimString, collationId)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+
override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
copy(
srcStr = newChildren.head,
@@ -1501,6 +1514,9 @@ case class StringTrimLeft(srcStr: Expression, trimStr: Option[Expression] = None
override def doEval(srcString: UTF8String, trimString: UTF8String): UTF8String =
CollationSupport.StringTrimLeft.exec(srcString, trimString, collationId)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+
override protected def withNewChildrenInternal(
newChildren: IndexedSeq[Expression]): StringTrimLeft =
copy(
@@ -1561,6 +1577,9 @@ case class StringTrimRight(srcStr: Expression, trimStr: Option[Expression] = Non
override def doEval(srcString: UTF8String, trimString: UTF8String): UTF8String =
CollationSupport.StringTrimRight.exec(srcString, trimString, collationId)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
+
override protected def withNewChildrenInternal(
newChildren: IndexedSeq[Expression]): StringTrimRight =
copy(
@@ -1595,7 +1614,7 @@ case class StringInstr(str: Expression, substr: Expression)
override def right: Expression = substr
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeAnyCollation, StringTypeAnyCollation)
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation)
override def nullSafeEval(string: Any, sub: Any): Any = {
CollationSupport.StringInstr.
@@ -1643,7 +1662,7 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr:
override def dataType: DataType = strExpr.dataType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeAnyCollation, StringTypeAnyCollation, IntegerType)
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, IntegerType)
override def first: Expression = strExpr
override def second: Expression = delimExpr
override def third: Expression = countExpr
@@ -1701,7 +1720,7 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
override def nullable: Boolean = substr.nullable || str.nullable
override def dataType: DataType = IntegerType
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeAnyCollation, StringTypeAnyCollation, IntegerType)
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, IntegerType)
override def eval(input: InternalRow): Any = {
val s = start.eval(input)
@@ -1969,13 +1988,10 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC
if (pattern == null) {
null
} else {
- val sb = new StringBuffer()
- val formatter = new java.util.Formatter(sb, Locale.US)
-
+ val formatter = new java.util.Formatter(Locale.US)
val arglist = children.tail.map(_.eval(input).asInstanceOf[AnyRef])
- formatter.format(pattern.asInstanceOf[UTF8String].toString, arglist: _*)
-
- UTF8String.fromString(sb.toString)
+ UTF8String.fromString(
+ formatter.format(pattern.asInstanceOf[UTF8String].toString, arglist: _*).toString)
}
}
@@ -2006,19 +2022,16 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC
val form = ctx.freshName("formatter")
val formatter = classOf[java.util.Formatter].getName
- val sb = ctx.freshName("sb")
- val stringBuffer = classOf[StringBuffer].getName
ev.copy(code = code"""
${pattern.code}
boolean ${ev.isNull} = ${pattern.isNull};
${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
if (!${ev.isNull}) {
- $stringBuffer $sb = new $stringBuffer();
- $formatter $form = new $formatter($sb, ${classOf[Locale].getName}.US);
+ $formatter $form = new $formatter(${classOf[Locale].getName}.US);
Object[] $argList = new Object[$numArgLists];
$argListCodes
- $form.format(${pattern.value}.toString(), $argList);
- ${ev.value} = UTF8String.fromString($sb.toString());
+ ${ev.value} = UTF8String.fromString(
+ $form.format(${pattern.value}.toString(), $argList).toString());
}""")
}
@@ -3333,14 +3346,37 @@ case class FormatNumber(x: Expression, d: Expression)
/**
* Splits a string into arrays of sentences, where each sentence is an array of words.
- * The 'lang' and 'country' arguments are optional, and if omitted, the default locale is used.
+ * The `lang` and `country` arguments are optional, their default values are all '',
+ * - When they are omitted:
+ * 1. If they are both omitted, the `Locale.ROOT - locale(language='', country='')` is used.
+ * The `Locale.ROOT` is regarded as the base locale of all locales, and is used as the
+ * language/country neutral locale for the locale sensitive operations.
+ * 2. If the `country` is omitted, the `locale(language, country='')` is used.
+ * - When they are null:
+ * 1. If they are both `null`, the `Locale.US - locale(language='en', country='US')` is used.
+ * 2. If the `language` is null and the `country` is not null,
+ * the `Locale.US - locale(language='en', country='US')` is used.
+ * 3. If the `language` is not null and the `country` is null, the `locale(language)` is used.
+ * 4. If neither is `null`, the `locale(language, country)` is used.
*/
@ExpressionDescription(
- usage = "_FUNC_(str[, lang, country]) - Splits `str` into an array of array of words.",
+ usage = "_FUNC_(str[, lang[, country]]) - Splits `str` into an array of array of words.",
+ arguments = """
+ Arguments:
+ * str - A STRING expression to be parsed.
+ * lang - An optional STRING expression with a language code from ISO 639 Alpha-2 (e.g. 'DE'),
+ Alpha-3, or a language subtag of up to 8 characters.
+ * country - An optional STRING expression with a country code from ISO 3166 alpha-2 country
+ code or a UN M.49 numeric-3 area code.
+ """,
examples = """
Examples:
> SELECT _FUNC_('Hi there! Good morning.');
[["Hi","there"],["Good","morning"]]
+ > SELECT _FUNC_('Hi there! Good morning.', 'en');
+ [["Hi","there"],["Good","morning"]]
+ > SELECT _FUNC_('Hi there! Good morning.', 'en', 'US');
+ [["Hi","there"],["Good","morning"]]
""",
since = "2.0.0",
group = "string_funcs")
@@ -3348,7 +3384,9 @@ case class Sentences(
str: Expression,
language: Expression = Literal(""),
country: Expression = Literal(""))
- extends TernaryExpression with ImplicitCastInputTypes with CodegenFallback {
+ extends TernaryExpression
+ with ImplicitCastInputTypes
+ with RuntimeReplaceable {
def this(str: Expression) = this(str, Literal(""), Literal(""))
def this(str: Expression, language: Expression) = this(str, language, Literal(""))
@@ -3362,49 +3400,18 @@ case class Sentences(
override def second: Expression = language
override def third: Expression = country
- override def eval(input: InternalRow): Any = {
- val string = str.eval(input)
- if (string == null) {
- null
- } else {
- val languageStr = language.eval(input).asInstanceOf[UTF8String]
- val countryStr = country.eval(input).asInstanceOf[UTF8String]
- val locale = if (languageStr != null && countryStr != null) {
- new Locale(languageStr.toString, countryStr.toString)
- } else {
- Locale.US
- }
- getSentences(string.asInstanceOf[UTF8String].toString, locale)
- }
- }
-
- private def getSentences(sentences: String, locale: Locale) = {
- val bi = BreakIterator.getSentenceInstance(locale)
- bi.setText(sentences)
- var idx = 0
- val result = new ArrayBuffer[GenericArrayData]
- while (bi.next != BreakIterator.DONE) {
- val sentence = sentences.substring(idx, bi.current)
- idx = bi.current
-
- val wi = BreakIterator.getWordInstance(locale)
- var widx = 0
- wi.setText(sentence)
- val words = new ArrayBuffer[UTF8String]
- while (wi.next != BreakIterator.DONE) {
- val word = sentence.substring(widx, wi.current)
- widx = wi.current
- if (Character.isLetterOrDigit(word.charAt(0))) words += UTF8String.fromString(word)
- }
- result += new GenericArrayData(words)
- }
- new GenericArrayData(result)
- }
+ override def replacement: Expression =
+ StaticInvoke(
+ classOf[ExpressionImplUtils],
+ dataType,
+ "getSentences",
+ Seq(str, language, country),
+ inputTypes,
+ propagateNull = false)
override protected def withNewChildrenInternal(
newFirst: Expression, newSecond: Expression, newThird: Expression): Sentences =
copy(str = newFirst, language = newSecond, country = newThird)
-
}
/**
@@ -3475,7 +3482,7 @@ case class SplitPart (
false)
override def nodeName: String = "split_part"
override def inputTypes: Seq[AbstractDataType] =
- Seq(StringTypeAnyCollation, StringTypeAnyCollation, IntegerType)
+ Seq(StringTypeNonCSAICollation, StringTypeNonCSAICollation, IntegerType)
def children: Seq[Expression] = Seq(str, delimiter, partNum)
protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = {
copy(str = newChildren.apply(0), delimiter = newChildren.apply(1),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
index a34a2c740876e..ad9610ea0c78a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromProtobufSqlFunctions.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.util.ArrayBasedMapData
import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.types.{MapType, NullType, StringType}
+import org.apache.spark.sql.types.{BinaryType, MapType, NullType, StringType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
@@ -88,6 +88,27 @@ case class FromProtobuf(
messageName: Expression,
descFilePath: Expression,
options: Expression) extends QuaternaryExpression with RuntimeReplaceable {
+
+ def this(data: Expression, messageName: Expression, descFilePathOrOptions: Expression) = {
+ this(
+ data,
+ messageName,
+ descFilePathOrOptions match {
+ case lit: Literal
+ if lit.dataType == StringType || lit.dataType == BinaryType => descFilePathOrOptions
+ case _ => Literal(null)
+ },
+ descFilePathOrOptions.dataType match {
+ case _: MapType => descFilePathOrOptions
+ case _ => Literal(null)
+ }
+ )
+ }
+
+ def this(data: Expression, messageName: Expression) = {
+ this(data, messageName, Literal(null), Literal(null))
+ }
+
override def first: Expression = data
override def second: Expression = messageName
override def third: Expression = descFilePath
@@ -110,11 +131,11 @@ case class FromProtobuf(
"representing the Protobuf message name"))
}
val descFilePathCheck = descFilePath.dataType match {
- case _: StringType if descFilePath.foldable => None
+ case _: StringType | BinaryType | NullType if descFilePath.foldable => None
case _ =>
Some(TypeCheckResult.TypeCheckFailure(
"The third argument of the FROM_PROTOBUF SQL function must be a constant string " +
- "representing the Protobuf descriptor file path"))
+ "or binary data representing the Protobuf descriptor file path"))
}
val optionsCheck = options.dataType match {
case MapType(StringType, StringType, _) |
@@ -141,7 +162,10 @@ case class FromProtobuf(
s.toString
}
val descFilePathValue: Option[Array[Byte]] = descFilePath.eval() match {
+ case s: UTF8String if s.toString.isEmpty => None
case s: UTF8String => Some(ProtobufHelper.readDescriptorFileContent(s.toString))
+ case bytes: Array[Byte] if bytes.isEmpty => None
+ case bytes: Array[Byte] => Some(bytes)
case null => None
}
val optionsValue: Map[String, String] = options.eval() match {
@@ -201,6 +225,27 @@ case class ToProtobuf(
messageName: Expression,
descFilePath: Expression,
options: Expression) extends QuaternaryExpression with RuntimeReplaceable {
+
+ def this(data: Expression, messageName: Expression, descFilePathOrOptions: Expression) = {
+ this(
+ data,
+ messageName,
+ descFilePathOrOptions match {
+ case lit: Literal
+ if lit.dataType == StringType || lit.dataType == BinaryType => descFilePathOrOptions
+ case _ => Literal(null)
+ },
+ descFilePathOrOptions.dataType match {
+ case _: MapType => descFilePathOrOptions
+ case _ => Literal(null)
+ }
+ )
+ }
+
+ def this(data: Expression, messageName: Expression) = {
+ this(data, messageName, Literal(null), Literal(null))
+ }
+
override def first: Expression = data
override def second: Expression = messageName
override def third: Expression = descFilePath
@@ -223,11 +268,11 @@ case class ToProtobuf(
"representing the Protobuf message name"))
}
val descFilePathCheck = descFilePath.dataType match {
- case _: StringType if descFilePath.foldable => None
+ case _: StringType | BinaryType | NullType if descFilePath.foldable => None
case _ =>
Some(TypeCheckResult.TypeCheckFailure(
"The third argument of the TO_PROTOBUF SQL function must be a constant string " +
- "representing the Protobuf descriptor file path"))
+ "or binary data representing the Protobuf descriptor file path"))
}
val optionsCheck = options.dataType match {
case MapType(StringType, StringType, _) |
@@ -256,6 +301,7 @@ case class ToProtobuf(
}
val descFilePathValue: Option[Array[Byte]] = descFilePath.eval() match {
case s: UTF8String => Some(ProtobufHelper.readDescriptorFileContent(s.toString))
+ case bytes: Array[Byte] => Some(bytes)
case null => None
}
val optionsValue: Map[String, String] = options.eval() match {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala
index bbf554d384b12..487985b4770ee 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala
@@ -126,7 +126,7 @@ object VariantExpressionEvalUtils {
buildVariant(builder, element, elementType)
}
builder.finishWritingArray(start, offsets)
- case MapType(StringType, valueType, _) =>
+ case MapType(_: StringType, valueType, _) =>
val data = input.asInstanceOf[MapData]
val keys = data.keyArray()
val values = data.valueArray()
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
index bd956fa5c00e1..2c8ca1e8bb2bb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
import org.apache.spark.sql.catalyst.json.JsonInferSchema
import org.apache.spark.sql.catalyst.trees.TreePattern.{TreePattern, VARIANT_GET}
import org.apache.spark.sql.catalyst.trees.UnaryLike
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, QuotingUtils}
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryErrorsBase, QueryExecutionErrors}
import org.apache.spark.sql.internal.SQLConf
@@ -117,6 +117,73 @@ case class IsVariantNull(child: Expression) extends UnaryExpression
copy(child = newChild)
}
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = "_FUNC_(expr) - Convert a nested input (array/map/struct) into a variant where maps and structs are converted to variant objects which are unordered unlike SQL structs. Input maps can only have string keys.",
+ examples = """
+ Examples:
+ > SELECT _FUNC_(named_struct('a', 1, 'b', 2));
+ {"a":1,"b":2}
+ > SELECT _FUNC_(array(1, 2, 3));
+ [1,2,3]
+ > SELECT _FUNC_(array(named_struct('a', 1)));
+ [{"a":1}]
+ > SELECT _FUNC_(array(map("a", 2)));
+ [{"a":2}]
+ """,
+ since = "4.0.0",
+ group = "variant_funcs")
+// scalastyle:on line.size.limit
+case class ToVariantObject(child: Expression)
+ extends UnaryExpression
+ with NullIntolerant
+ with QueryErrorsBase {
+
+ override val dataType: DataType = VariantType
+
+ // Only accept nested types at the root but any types can be nested inside.
+ override def checkInputDataTypes(): TypeCheckResult = {
+ val checkResult: Boolean = child.dataType match {
+ case _: StructType | _: ArrayType | _: MapType =>
+ VariantGet.checkDataType(child.dataType, allowStructsAndMaps = true)
+ case _ => false
+ }
+ if (!checkResult) {
+ DataTypeMismatch(
+ errorSubClass = "CAST_WITHOUT_SUGGESTION",
+ messageParameters =
+ Map("srcType" -> toSQLType(child.dataType), "targetType" -> toSQLType(VariantType)))
+ } else {
+ TypeCheckResult.TypeCheckSuccess
+ }
+ }
+
+ override def prettyName: String = "to_variant_object"
+
+ override protected def withNewChildInternal(newChild: Expression): ToVariantObject =
+ copy(child = newChild)
+
+ protected override def nullSafeEval(input: Any): Any =
+ VariantExpressionEvalUtils.castToVariant(input, child.dataType)
+
+ override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+ val childCode = child.genCode(ctx)
+ val cls = variant.VariantExpressionEvalUtils.getClass.getName.stripSuffix("$")
+ val fromArg = ctx.addReferenceObj("from", child.dataType)
+ val javaType = JavaCode.javaType(VariantType)
+ val code =
+ code"""
+ ${childCode.code}
+ boolean ${ev.isNull} = ${childCode.isNull};
+ $javaType ${ev.value} = ${CodeGenerator.defaultValue(VariantType)};
+ if (!${childCode.isNull}) {
+ ${ev.value} = $cls.castToVariant(${childCode.value}, $fromArg);
+ }
+ """
+ ev.copy(code = code)
+ }
+}
+
object VariantPathParser extends RegexParsers {
// A path segment in the `VariantGet` expression represents either an object key access or an
// array index access.
@@ -260,13 +327,16 @@ case object VariantGet {
* Returns whether a data type can be cast into/from variant. For scalar types, we allow a subset
* of them. For nested types, we reject map types with a non-string key type.
*/
- def checkDataType(dataType: DataType): Boolean = dataType match {
+ def checkDataType(dataType: DataType, allowStructsAndMaps: Boolean = true): Boolean =
+ dataType match {
case _: NumericType | BooleanType | _: StringType | BinaryType | _: DatetimeType |
VariantType | _: DayTimeIntervalType | _: YearMonthIntervalType =>
true
- case ArrayType(elementType, _) => checkDataType(elementType)
- case MapType(_: StringType, valueType, _) => checkDataType(valueType)
- case StructType(fields) => fields.forall(f => checkDataType(f.dataType))
+ case ArrayType(elementType, _) => checkDataType(elementType, allowStructsAndMaps)
+ case MapType(_: StringType, valueType, _) if allowStructsAndMaps =>
+ checkDataType(valueType, allowStructsAndMaps)
+ case StructType(fields) if allowStructsAndMaps =>
+ fields.forall(f => checkDataType(f.dataType, allowStructsAndMaps))
case _ => false
}
@@ -635,7 +705,7 @@ object VariantExplode {
> SELECT _FUNC_(parse_json('null'));
VOID
> SELECT _FUNC_(parse_json('[{"b":true,"a":0}]'));
- ARRAY>
+ ARRAY>
""",
since = "4.0.0",
group = "variant_funcs"
@@ -666,7 +736,24 @@ object SchemaOfVariant {
/** The actual implementation of the `SchemaOfVariant` expression. */
def schemaOfVariant(input: VariantVal): UTF8String = {
val v = new Variant(input.getValue, input.getMetadata)
- UTF8String.fromString(schemaOf(v).sql)
+ UTF8String.fromString(printSchema(schemaOf(v)))
+ }
+
+ /**
+ * Similar to `dataType.sql`. The only difference is that `StructType` is shown as
+ * `OBJECT<...>` rather than `STRUCT<...>`.
+ * SchemaOfVariant expressions use the Struct DataType to denote the Object type in the variant
+ * spec. However, the Object type is not equivalent to the struct type as an Object represents an
+ * unordered bag of key-value pairs while the Struct type is ordered.
+ */
+ def printSchema(dataType: DataType): String = dataType match {
+ case StructType(fields) =>
+ def printField(f: StructField): String =
+ s"${QuotingUtils.quoteIfNeeded(f.name)}: ${printSchema(f.dataType)}"
+
+ s"OBJECT<${fields.map(printField).mkString(", ")}>"
+ case ArrayType(elementType, _) => s"ARRAY<${printSchema(elementType)}>"
+ case _ => dataType.sql
}
/**
@@ -731,7 +818,7 @@ object SchemaOfVariant {
> SELECT _FUNC_(parse_json(j)) FROM VALUES ('1'), ('2'), ('3') AS tab(j);
BIGINT
> SELECT _FUNC_(parse_json(j)) FROM VALUES ('{"a": 1}'), ('{"b": true}'), ('{"c": 1.23}') AS tab(j);
- STRUCT
+ OBJECT
""",
since = "4.0.0",
group = "variant_funcs")
@@ -767,7 +854,8 @@ case class SchemaOfVariantAgg(
override def merge(buffer: DataType, input: DataType): DataType =
SchemaOfVariant.mergeSchema(buffer, input)
- override def eval(buffer: DataType): Any = UTF8String.fromString(buffer.sql)
+ override def eval(buffer: DataType): Any =
+ UTF8String.fromString(SchemaOfVariant.printSchema(buffer))
override def serialize(buffer: DataType): Array[Byte] = buffer.json.getBytes("UTF-8")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
index 5b06741a2f54e..31e65cf0abc95 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
@@ -255,7 +255,7 @@ case class XPathList(xml: Expression, path: Expression) extends XPathExtract {
override def nullSafeEval(xml: Any, path: Any): Any = {
val nodeList = xpathUtil.evalNodeList(xml.asInstanceOf[UTF8String].toString, pathString)
if (nodeList ne null) {
- val ret = new Array[UTF8String](nodeList.getLength)
+ val ret = new Array[AnyRef](nodeList.getLength)
var i = 0
while (i < nodeList.getLength) {
ret(i) = UTF8String.fromString(nodeList.item(i).getNodeValue)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index 26de4cc7ad1c8..13129d44fe0c2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -215,7 +215,7 @@ class JacksonParser(
)
}
- Some(InternalRow(new GenericArrayData(res.toArray)))
+ Some(InternalRow(new GenericArrayData(res.toArray[Any])))
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuery.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuery.scala
index 424f4b96271d3..6c0d7189862d3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuery.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/DecorrelateInnerQuery.scala
@@ -1064,7 +1064,7 @@ object DecorrelateInnerQuery extends PredicateHelper {
// Project, they could get added at the beginning or the end of the output columns
// depending on the child plan.
// The inner expressions for the domain are the values of newOuterReferenceMap.
- val domainProjections = collectedChildOuterReferences.map(newOuterReferenceMap(_))
+ val domainProjections = newOuterReferences.map(newOuterReferenceMap(_))
val newChild = Project(child.output ++ domainProjections, decorrelatedChild)
(newChild, newJoinCond, newOuterReferenceMap)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index a2a26924885c0..8e14537c6a5b4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -303,7 +303,8 @@ abstract class Optimizer(catalogManager: CatalogManager)
ReplaceCurrentLike(catalogManager),
SpecialDatetimeValues,
RewriteAsOfJoin,
- EvalInlineTables
+ EvalInlineTables,
+ ReplaceTranspose
)
override def apply(plan: LogicalPlan): LogicalPlan = {
@@ -1722,15 +1723,18 @@ object EliminateSorts extends Rule[LogicalPlan] {
* 3) by eliminating the always-true conditions given the constraints on the child's output.
*/
object PruneFilters extends Rule[LogicalPlan] with PredicateHelper {
+ private def shouldApply(child: LogicalPlan): Boolean =
+ SQLConf.get.getConf(SQLConf.PRUNE_FILTERS_CAN_PRUNE_STREAMING_SUBPLAN) || !child.isStreaming
+
def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(
_.containsPattern(FILTER), ruleId) {
// If the filter condition always evaluate to true, remove the filter.
case Filter(Literal(true, BooleanType), child) => child
// If the filter condition always evaluate to null or false,
// replace the input with an empty relation.
- case Filter(Literal(null, _), child) =>
+ case Filter(Literal(null, _), child) if shouldApply(child) =>
LocalRelation(child.output, data = Seq.empty, isStreaming = child.isStreaming)
- case Filter(Literal(false, BooleanType), child) =>
+ case Filter(Literal(false, BooleanType), child) if shouldApply(child) =>
LocalRelation(child.output, data = Seq.empty, isStreaming = child.isStreaming)
// If any deterministic condition is guaranteed to be true given the constraints on the child's
// output, remove the condition
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
index 801bd2693af42..5aef82b64ed32 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
@@ -400,13 +400,14 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
(distinctAggOperatorMap.flatMap(_._2) ++
regularAggOperatorMap.map(e => (e._1, e._3))).toMap
+ val groupByMapNonFoldable = groupByMap.filter(!_._1.foldable)
val patchedAggExpressions = a.aggregateExpressions.map { e =>
e.transformDown {
case e: Expression =>
// The same GROUP BY clauses can have different forms (different names for instance) in
// the groupBy and aggregate expressions of an aggregate. This makes a map lookup
// tricky. So we do a linear search for a semantically equal group by expression.
- groupByMap
+ groupByMapNonFoldable
.find(ge => e.semanticEquals(ge._1))
.map(_._2)
.getOrElse(transformations.getOrElse(e, e))
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index d0ee9f2d110d5..3cdde622d51f7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -79,7 +79,7 @@ object ConstantFolding extends Rule[LogicalPlan] {
// Fold expressions that are foldable.
case e if e.foldable =>
try {
- Literal.create(e.eval(EmptyRow), e.dataType)
+ Literal.create(e.freshCopyIfContainsStatefulExpression().eval(EmptyRow), e.dataType)
} catch {
case NonFatal(_) if isConditionalBranch =>
// When doing constant folding inside conditional expressions, we should not fail
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
index 55f222d880844..a524acc19aea8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
@@ -184,3 +184,10 @@ object SpecialDatetimeValues extends Rule[LogicalPlan] {
}
}
}
+
+object ReplaceTranspose extends Rule[LogicalPlan] {
+ def apply(plan: LogicalPlan): LogicalPlan = plan.transform {
+ case t @ Transpose(output, data) =>
+ LocalRelation(output, data)
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index b0922542c5629..674005caaf1b2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -24,7 +24,7 @@ import scala.collection.mutable.{ArrayBuffer, ListBuffer, Set}
import scala.jdk.CollectionConverters._
import scala.util.{Left, Right}
-import org.antlr.v4.runtime.{ParserRuleContext, Token}
+import org.antlr.v4.runtime.{ParserRuleContext, RuleContext, Token}
import org.antlr.v4.runtime.misc.Interval
import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode}
@@ -45,7 +45,7 @@ import org.apache.spark.sql.catalyst.trees.TreePattern.PARAMETER
import org.apache.spark.sql.catalyst.types.DataTypeUtils
import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, DateTimeUtils, IntervalUtils}
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{convertSpecialDate, convertSpecialTimestamp, convertSpecialTimestampNTZ, getZoneId, stringToDate, stringToTimestamp, stringToTimestampWithoutTimeZone}
-import org.apache.spark.sql.connector.catalog.{CatalogV2Util, SupportsNamespaces, TableCatalog, TableWritePrivilege}
+import org.apache.spark.sql.connector.catalog.{CatalogV2Util, IdentityColumnSpec, SupportsNamespaces, TableCatalog, TableWritePrivilege}
import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition
import org.apache.spark.sql.connector.expressions.{ApplyTransform, BucketTransform, DaysTransform, Expression => V2Expression, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, Transform, YearsTransform}
import org.apache.spark.sql.errors.{DataTypeErrorsBase, QueryCompilationErrors, QueryParsingErrors, SqlScriptingErrors}
@@ -173,14 +173,10 @@ class AstBuilder extends DataTypeAstBuilder
case Some(c: CreateVariable) =>
if (allowVarDeclare) {
throw SqlScriptingErrors.variableDeclarationOnlyAtBeginning(
- c.origin,
- toSQLId(c.name.asInstanceOf[UnresolvedIdentifier].nameParts),
- c.origin.line.get.toString)
+ c.origin, c.name.asInstanceOf[UnresolvedIdentifier].nameParts)
} else {
throw SqlScriptingErrors.variableDeclarationNotAllowedInScope(
- c.origin,
- toSQLId(c.name.asInstanceOf[UnresolvedIdentifier].nameParts),
- c.origin.line.get.toString)
+ c.origin, c.name.asInstanceOf[UnresolvedIdentifier].nameParts)
}
case _ =>
}
@@ -200,7 +196,9 @@ class AstBuilder extends DataTypeAstBuilder
el.multipartIdentifier().getText.toLowerCase(Locale.ROOT) =>
withOrigin(bl) {
throw SqlScriptingErrors.labelsMismatch(
- CurrentOrigin.get, bl.multipartIdentifier().getText, el.multipartIdentifier().getText)
+ CurrentOrigin.get,
+ bl.multipartIdentifier().getText,
+ el.multipartIdentifier().getText)
}
case (None, Some(el: EndLabelContext)) =>
withOrigin(el) {
@@ -261,6 +259,120 @@ class AstBuilder extends DataTypeAstBuilder
WhileStatement(condition, body, Some(labelText))
}
+ override def visitSearchedCaseStatement(ctx: SearchedCaseStatementContext): CaseStatement = {
+ val conditions = ctx.conditions.asScala.toList.map(boolExpr => withOrigin(boolExpr) {
+ SingleStatement(
+ Project(
+ Seq(Alias(expression(boolExpr), "condition")()),
+ OneRowRelation()))
+ })
+ val conditionalBodies =
+ ctx.conditionalBodies.asScala.toList.map(body => visitCompoundBody(body))
+
+ if (conditions.length != conditionalBodies.length) {
+ throw SparkException.internalError(
+ s"Mismatched number of conditions ${conditions.length} and condition bodies" +
+ s" ${conditionalBodies.length} in case statement")
+ }
+
+ CaseStatement(
+ conditions = conditions,
+ conditionalBodies = conditionalBodies,
+ elseBody = Option(ctx.elseBody).map(body => visitCompoundBody(body)))
+ }
+
+ override def visitSimpleCaseStatement(ctx: SimpleCaseStatementContext): CaseStatement = {
+ // uses EqualTo to compare the case variable(the main case expression)
+ // to the WHEN clause expressions
+ val conditions = ctx.conditionExpressions.asScala.toList.map(expr => withOrigin(expr) {
+ SingleStatement(
+ Project(
+ Seq(Alias(EqualTo(expression(ctx.caseVariable), expression(expr)), "condition")()),
+ OneRowRelation()))
+ })
+ val conditionalBodies =
+ ctx.conditionalBodies.asScala.toList.map(body => visitCompoundBody(body))
+
+ if (conditions.length != conditionalBodies.length) {
+ throw SparkException.internalError(
+ s"Mismatched number of conditions ${conditions.length} and condition bodies" +
+ s" ${conditionalBodies.length} in case statement")
+ }
+
+ CaseStatement(
+ conditions = conditions,
+ conditionalBodies = conditionalBodies,
+ elseBody = Option(ctx.elseBody).map(body => visitCompoundBody(body)))
+ }
+
+ override def visitRepeatStatement(ctx: RepeatStatementContext): RepeatStatement = {
+ val labelText = generateLabelText(Option(ctx.beginLabel()), Option(ctx.endLabel()))
+ val boolExpr = ctx.booleanExpression()
+
+ val condition = withOrigin(boolExpr) {
+ SingleStatement(
+ Project(
+ Seq(Alias(expression(boolExpr), "condition")()),
+ OneRowRelation()))}
+ val body = visitCompoundBody(ctx.compoundBody())
+
+ RepeatStatement(condition, body, Some(labelText))
+ }
+
+ private def leaveOrIterateContextHasLabel(
+ ctx: RuleContext, label: String, isIterate: Boolean): Boolean = {
+ ctx match {
+ case c: BeginEndCompoundBlockContext
+ if Option(c.beginLabel()).isDefined &&
+ c.beginLabel().multipartIdentifier().getText.toLowerCase(Locale.ROOT).equals(label) =>
+ if (isIterate) {
+ throw SqlScriptingErrors.invalidIterateLabelUsageForCompound(CurrentOrigin.get, label)
+ }
+ true
+ case c: WhileStatementContext
+ if Option(c.beginLabel()).isDefined &&
+ c.beginLabel().multipartIdentifier().getText.toLowerCase(Locale.ROOT).equals(label)
+ => true
+ case c: RepeatStatementContext
+ if Option(c.beginLabel()).isDefined &&
+ c.beginLabel().multipartIdentifier().getText.toLowerCase(Locale.ROOT).equals(label)
+ => true
+ case _ => false
+ }
+ }
+
+ override def visitLeaveStatement(ctx: LeaveStatementContext): LeaveStatement =
+ withOrigin(ctx) {
+ val labelText = ctx.multipartIdentifier().getText.toLowerCase(Locale.ROOT)
+ var parentCtx = ctx.parent
+
+ while (Option(parentCtx).isDefined) {
+ if (leaveOrIterateContextHasLabel(parentCtx, labelText, isIterate = false)) {
+ return LeaveStatement(labelText)
+ }
+ parentCtx = parentCtx.parent
+ }
+
+ throw SqlScriptingErrors.labelDoesNotExist(
+ CurrentOrigin.get, labelText, "LEAVE")
+ }
+
+ override def visitIterateStatement(ctx: IterateStatementContext): IterateStatement =
+ withOrigin(ctx) {
+ val labelText = ctx.multipartIdentifier().getText.toLowerCase(Locale.ROOT)
+ var parentCtx = ctx.parent
+
+ while (Option(parentCtx).isDefined) {
+ if (leaveOrIterateContextHasLabel(parentCtx, labelText, isIterate = true)) {
+ return IterateStatement(labelText)
+ }
+ parentCtx = parentCtx.parent
+ }
+
+ throw SqlScriptingErrors.labelDoesNotExist(
+ CurrentOrigin.get, labelText, "ITERATE")
+ }
+
override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = withOrigin(ctx) {
Option(ctx.statement().asInstanceOf[ParserRuleContext])
.orElse(Option(ctx.setResetStatement().asInstanceOf[ParserRuleContext]))
@@ -357,7 +469,8 @@ class AstBuilder extends DataTypeAstBuilder
ctx.aggregationClause,
ctx.havingClause,
ctx.windowClause,
- plan
+ plan,
+ isPipeOperatorSelect = false
)
}
}
@@ -945,7 +1058,8 @@ class AstBuilder extends DataTypeAstBuilder
ctx.aggregationClause,
ctx.havingClause,
ctx.windowClause,
- from
+ from,
+ isPipeOperatorSelect = false
)
}
@@ -1032,7 +1146,8 @@ class AstBuilder extends DataTypeAstBuilder
aggregationClause,
havingClause,
windowClause,
- isDistinct = false)
+ isDistinct = false,
+ isPipeOperatorSelect = false)
ScriptTransformation(
string(visitStringLit(transformClause.script)),
@@ -1053,6 +1168,8 @@ class AstBuilder extends DataTypeAstBuilder
* Add a regular (SELECT) query specification to a logical plan. The query specification
* is the core of the logical plan, this is where sourcing (FROM clause), projection (SELECT),
* aggregation (GROUP BY ... HAVING ...) and filtering (WHERE) takes place.
+ * If 'isPipeOperatorSelect' is true, wraps each projected expression with a [[PipeSelect]]
+ * expression for future validation of the expressions during analysis.
*
* Note that query hints are ignored (both by the parser and the builder).
*/
@@ -1064,7 +1181,8 @@ class AstBuilder extends DataTypeAstBuilder
aggregationClause: AggregationClauseContext,
havingClause: HavingClauseContext,
windowClause: WindowClauseContext,
- relation: LogicalPlan): LogicalPlan = withOrigin(ctx) {
+ relation: LogicalPlan,
+ isPipeOperatorSelect: Boolean): LogicalPlan = withOrigin(ctx) {
val isDistinct = selectClause.setQuantifier() != null &&
selectClause.setQuantifier().DISTINCT() != null
@@ -1076,7 +1194,8 @@ class AstBuilder extends DataTypeAstBuilder
aggregationClause,
havingClause,
windowClause,
- isDistinct)
+ isDistinct,
+ isPipeOperatorSelect)
// Hint
selectClause.hints.asScala.foldRight(plan)(withHints)
@@ -1090,7 +1209,8 @@ class AstBuilder extends DataTypeAstBuilder
aggregationClause: AggregationClauseContext,
havingClause: HavingClauseContext,
windowClause: WindowClauseContext,
- isDistinct: Boolean): LogicalPlan = {
+ isDistinct: Boolean,
+ isPipeOperatorSelect: Boolean): LogicalPlan = {
// Add lateral views.
val withLateralView = lateralView.asScala.foldLeft(relation)(withGenerate)
@@ -1104,7 +1224,20 @@ class AstBuilder extends DataTypeAstBuilder
}
def createProject() = if (namedExpressions.nonEmpty) {
- Project(namedExpressions, withFilter)
+ val newProjectList: Seq[NamedExpression] = if (isPipeOperatorSelect) {
+ // If this is a pipe operator |> SELECT clause, add a [[PipeSelect]] expression wrapping
+ // each alias in the project list, so the analyzer can check invariants later.
+ namedExpressions.map {
+ case a: Alias =>
+ a.withNewChildren(Seq(PipeSelect(a.child)))
+ .asInstanceOf[NamedExpression]
+ case other =>
+ other
+ }
+ } else {
+ namedExpressions
+ }
+ Project(newProjectList, withFilter)
} else {
withFilter
}
@@ -3398,6 +3531,14 @@ class AstBuilder extends DataTypeAstBuilder
throw QueryParsingErrors.fromToIntervalUnsupportedError(from, to, ctx)
}
} catch {
+ // Keep error class of SparkIllegalArgumentExceptions and enrich it with query context
+ case se: SparkIllegalArgumentException =>
+ val pe = new ParseException(
+ errorClass = se.getErrorClass,
+ messageParameters = se.getMessageParameters.asScala.toMap,
+ ctx)
+ pe.setStackTrace(se.getStackTrace)
+ throw pe
// Handle Exceptions thrown by CalendarInterval
case e: IllegalArgumentException =>
val pe = new ParseException(
@@ -3478,13 +3619,19 @@ class AstBuilder extends DataTypeAstBuilder
}
}
+ val dataType = typedVisit[DataType](ctx.dataType)
ColumnDefinition(
name = name,
- dataType = typedVisit[DataType](ctx.dataType),
+ dataType = dataType,
nullable = nullable,
comment = commentSpec.map(visitCommentSpec),
defaultValue = defaultExpression.map(visitDefaultExpression),
- generationExpression = generationExpression.map(visitGenerationExpression)
+ generationExpression = generationExpression.collect {
+ case ctx: GeneratedColumnContext => visitGeneratedColumn(ctx)
+ },
+ identityColumnSpec = generationExpression.collect {
+ case ctx: IdentityColumnContext => visitIdentityColumn(ctx, dataType)
+ }
)
}
@@ -3540,11 +3687,63 @@ class AstBuilder extends DataTypeAstBuilder
/**
* Create a generation expression string.
*/
- override def visitGenerationExpression(ctx: GenerationExpressionContext): String =
+ override def visitGeneratedColumn(ctx: GeneratedColumnContext): String =
withOrigin(ctx) {
getDefaultExpression(ctx.expression(), "GENERATED").originalSQL
}
+ /**
+ * Parse and verify IDENTITY column definition.
+ *
+ * @param ctx The parser context.
+ * @param dataType The data type of column defined as IDENTITY column. Used for verification.
+ * @return Tuple containing start, step and allowExplicitInsert.
+ */
+ protected def visitIdentityColumn(
+ ctx: IdentityColumnContext,
+ dataType: DataType): IdentityColumnSpec = {
+ if (dataType != LongType && dataType != IntegerType) {
+ throw QueryParsingErrors.identityColumnUnsupportedDataType(ctx, dataType.toString)
+ }
+ // We support two flavors of syntax:
+ // (1) GENERATED ALWAYS AS IDENTITY (...)
+ // (2) GENERATED BY DEFAULT AS IDENTITY (...)
+ // (1) forbids explicit inserts, while (2) allows.
+ val allowExplicitInsert = ctx.BY() != null && ctx.DEFAULT() != null
+ val (start, step) = visitIdentityColSpec(ctx.identityColSpec())
+
+ new IdentityColumnSpec(start, step, allowExplicitInsert)
+ }
+
+ override def visitIdentityColSpec(ctx: IdentityColSpecContext): (Long, Long) = {
+ val defaultStart = 1
+ val defaultStep = 1
+ if (ctx == null) {
+ return (defaultStart, defaultStep)
+ }
+ var (start, step): (Option[Long], Option[Long]) = (None, None)
+ ctx.sequenceGeneratorOption().asScala.foreach { option =>
+ if (option.start != null) {
+ if (start.isDefined) {
+ throw QueryParsingErrors.identityColumnDuplicatedSequenceGeneratorOption(ctx, "START")
+ }
+ start = Some(option.start.getText.toLong)
+ } else if (option.step != null) {
+ if (step.isDefined) {
+ throw QueryParsingErrors.identityColumnDuplicatedSequenceGeneratorOption(ctx, "STEP")
+ }
+ step = Some(option.step.getText.toLong)
+ if (step.get == 0L) {
+ throw QueryParsingErrors.identityColumnIllegalStep(ctx)
+ }
+ } else {
+ throw SparkException
+ .internalError(s"Invalid identity column sequence generator option: ${option.getText}")
+ }
+ }
+ (start.getOrElse(defaultStart), step.getOrElse(defaultStep))
+ }
+
/**
* Create an optional comment string.
*/
@@ -5031,6 +5230,13 @@ class AstBuilder extends DataTypeAstBuilder
val options = Option(ctx.options).map(visitPropertyKeyValues).getOrElse(Map.empty)
val isLazy = ctx.LAZY != null
if (query.isDefined) {
+ // Disallow parameter markers in the query of the cache.
+ // We need this limitation because we store the original query text, pre substitution.
+ // To lift this we would need to reconstitute the query with parameter markers replaced with
+ // the values given at CACHE TABLE time, or we would need to store the parameter values
+ // alongside the text.
+ // The same rule can be found in CREATE VIEW builder.
+ checkInvalidParameter(query.get, "the query of CACHE TABLE")
CacheTableAsSelect(ident.head, query.get, source(ctx.query()), isLazy, options)
} else {
CacheTable(
@@ -5491,6 +5697,28 @@ class AstBuilder extends DataTypeAstBuilder
ctx.EXISTS != null)
}
+ /**
+ * Creates a plan for invoking a procedure.
+ *
+ * For example:
+ * {{{
+ * CALL multi_part_name(v1, v2, ...);
+ * CALL multi_part_name(v1, param2 => v2, ...);
+ * CALL multi_part_name(param1 => v1, param2 => v2, ...);
+ * }}}
+ */
+ override def visitCall(ctx: CallContext): LogicalPlan = withOrigin(ctx) {
+ val procedure = withIdentClause(ctx.identifierReference, UnresolvedProcedure)
+ val args = ctx.functionArgument.asScala.map {
+ case expr if expr.namedArgumentExpression != null =>
+ val namedExpr = expr.namedArgumentExpression
+ NamedArgumentExpression(namedExpr.key.getText, expression(namedExpr.value))
+ case expr =>
+ expression(expr)
+ }.toSeq
+ Call(procedure, args)
+ }
+
/**
* Create a TimestampAdd expression.
*/
@@ -5627,4 +5855,59 @@ class AstBuilder extends DataTypeAstBuilder
withOrigin(ctx) {
visitSetVariableImpl(ctx.query(), ctx.multipartIdentifierList(), ctx.assignmentList())
}
+
+ override def visitOperatorPipeStatement(ctx: OperatorPipeStatementContext): LogicalPlan = {
+ visitOperatorPipeRightSide(ctx.operatorPipeRightSide(), plan(ctx.left))
+ }
+
+ private def visitOperatorPipeRightSide(
+ ctx: OperatorPipeRightSideContext, left: LogicalPlan): LogicalPlan = {
+ if (!SQLConf.get.getConf(SQLConf.OPERATOR_PIPE_SYNTAX_ENABLED)) {
+ operationNotAllowed("Operator pipe SQL syntax using |>", ctx)
+ }
+ Option(ctx.selectClause).map { c =>
+ withSelectQuerySpecification(
+ ctx = ctx,
+ selectClause = c,
+ lateralView = new java.util.ArrayList[LateralViewContext](),
+ whereClause = null,
+ aggregationClause = null,
+ havingClause = null,
+ windowClause = null,
+ relation = left,
+ isPipeOperatorSelect = true)
+ }.getOrElse(Option(ctx.whereClause).map { c =>
+ // Add a table subquery boundary between the new filter and the input plan if one does not
+ // already exist. This helps the analyzer behave as if we had added the WHERE clause after a
+ // table subquery containing the input plan.
+ val withSubqueryAlias = left match {
+ case s: SubqueryAlias =>
+ s
+ case u: UnresolvedRelation =>
+ u
+ case _ =>
+ SubqueryAlias(SubqueryAlias.generateSubqueryName(), left)
+ }
+ withWhereClause(c, withSubqueryAlias)
+ }.get)
+ }
+
+ /**
+ * Check plan for any parameters.
+ * If it finds any throws UNSUPPORTED_FEATURE.PARAMETER_MARKER_IN_UNEXPECTED_STATEMENT.
+ * This method is used to ban parameters in some contexts.
+ */
+ protected def checkInvalidParameter(plan: LogicalPlan, statement: String): Unit = {
+ plan.foreach { p =>
+ p.expressions.foreach { expr =>
+ if (expr.containsPattern(PARAMETER)) {
+ throw QueryParsingErrors.parameterMarkerNotAllowed(statement, p.origin)
+ }
+ }
+ }
+ plan.children.foreach(p => checkInvalidParameter(p, statement))
+ plan.innerChildren.collect {
+ case child: LogicalPlan => checkInvalidParameter(child, statement)
+ }
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingLogicalOperators.scala
index 4a5259f09a8a3..ed40a5fd734b6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingLogicalOperators.scala
@@ -89,3 +89,52 @@ case class WhileStatement(
condition: SingleStatement,
body: CompoundBody,
label: Option[String]) extends CompoundPlanStatement
+
+/**
+ * Logical operator for REPEAT statement.
+ * @param condition Any expression evaluating to a Boolean.
+ * Body is executed as long as the condition evaluates to false
+ * @param body Compound body is a collection of statements that are executed once no matter what,
+ * and then as long as condition is false.
+ * @param label An optional label for the loop which is unique amongst all labels for statements
+ * within which the LOOP statement is contained.
+ * If an end label is specified it must match the beginning label.
+ * The label can be used to LEAVE or ITERATE the loop.
+ */
+case class RepeatStatement(
+ condition: SingleStatement,
+ body: CompoundBody,
+ label: Option[String]) extends CompoundPlanStatement
+
+
+/**
+ * Logical operator for LEAVE statement.
+ * The statement can be used both for compounds or any kind of loops.
+ * When used, the corresponding body/loop execution is skipped and the execution continues
+ * with the next statement after the body/loop.
+ * @param label Label of the compound or loop to leave.
+ */
+case class LeaveStatement(label: String) extends CompoundPlanStatement
+
+/**
+ * Logical operator for ITERATE statement.
+ * The statement can be used only for loops.
+ * When used, the rest of the loop is skipped and the loop execution continues
+ * with the next iteration.
+ * @param label Label of the loop to iterate.
+ */
+case class IterateStatement(label: String) extends CompoundPlanStatement
+
+/**
+ * Logical operator for CASE statement.
+ * @param conditions Collection of conditions which correspond to WHEN clauses.
+ * @param conditionalBodies Collection of bodies that have a corresponding condition,
+ * in WHEN branches.
+ * @param elseBody Body that is executed if none of the conditions are met, i.e. ELSE branch.
+ */
+case class CaseStatement(
+ conditions: Seq[SingleStatement],
+ conditionalBodies: Seq[CompoundBody],
+ elseBody: Option[CompoundBody]) extends CompoundPlanStatement {
+ assert(conditions.length == conditionalBodies.length)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala
index 83e50aa33c70d..043214711ccf9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala
@@ -21,10 +21,10 @@ import org.apache.spark.SparkException
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.expressions.{Expression, Literal, UnaryExpression, Unevaluable}
import org.apache.spark.sql.catalyst.parser.ParserInterface
-import org.apache.spark.sql.catalyst.util.GeneratedColumn
+import org.apache.spark.sql.catalyst.util.{GeneratedColumn, IdentityColumn}
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.validateDefaultValueExpr
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumnsUtils.{CURRENT_DEFAULT_COLUMN_METADATA_KEY, EXISTS_DEFAULT_COLUMN_METADATA_KEY}
-import org.apache.spark.sql.connector.catalog.{Column => V2Column, ColumnDefaultValue}
+import org.apache.spark.sql.connector.catalog.{Column => V2Column, ColumnDefaultValue, IdentityColumnSpec}
import org.apache.spark.sql.connector.expressions.LiteralValue
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.internal.connector.ColumnImpl
@@ -41,7 +41,11 @@ case class ColumnDefinition(
comment: Option[String] = None,
defaultValue: Option[DefaultValueExpression] = None,
generationExpression: Option[String] = None,
+ identityColumnSpec: Option[IdentityColumnSpec] = None,
metadata: Metadata = Metadata.empty) extends Expression with Unevaluable {
+ assert(
+ generationExpression.isEmpty || identityColumnSpec.isEmpty,
+ "A ColumnDefinition cannot contain both a generation expression and an identity column spec.")
override def children: Seq[Expression] = defaultValue.toSeq
@@ -58,6 +62,7 @@ case class ColumnDefinition(
comment.orNull,
defaultValue.map(_.toV2(statement, name)).orNull,
generationExpression.orNull,
+ identityColumnSpec.orNull,
if (metadata == Metadata.empty) null else metadata.json)
}
@@ -75,8 +80,19 @@ case class ColumnDefinition(
generationExpression.foreach { generationExpr =>
metadataBuilder.putString(GeneratedColumn.GENERATION_EXPRESSION_METADATA_KEY, generationExpr)
}
+ encodeIdentityColumnSpec(metadataBuilder)
StructField(name, dataType, nullable, metadataBuilder.build())
}
+
+ private def encodeIdentityColumnSpec(metadataBuilder: MetadataBuilder): Unit = {
+ identityColumnSpec.foreach { spec: IdentityColumnSpec =>
+ metadataBuilder.putLong(IdentityColumn.IDENTITY_INFO_START, spec.getStart)
+ metadataBuilder.putLong(IdentityColumn.IDENTITY_INFO_STEP, spec.getStep)
+ metadataBuilder.putBoolean(
+ IdentityColumn.IDENTITY_INFO_ALLOW_EXPLICIT_INSERT,
+ spec.isAllowExplicitInsert)
+ }
+ }
}
object ColumnDefinition {
@@ -87,6 +103,9 @@ object ColumnDefinition {
metadataBuilder.remove(CURRENT_DEFAULT_COLUMN_METADATA_KEY)
metadataBuilder.remove(EXISTS_DEFAULT_COLUMN_METADATA_KEY)
metadataBuilder.remove(GeneratedColumn.GENERATION_EXPRESSION_METADATA_KEY)
+ metadataBuilder.remove(IdentityColumn.IDENTITY_INFO_START)
+ metadataBuilder.remove(IdentityColumn.IDENTITY_INFO_STEP)
+ metadataBuilder.remove(IdentityColumn.IDENTITY_INFO_ALLOW_EXPLICIT_INSERT)
val hasDefaultValue = col.getCurrentDefaultValue().isDefined &&
col.getExistenceDefaultValue().isDefined
@@ -97,6 +116,15 @@ object ColumnDefinition {
None
}
val generationExpr = GeneratedColumn.getGenerationExpression(col)
+ val identityColumnSpec = if (col.metadata.contains(IdentityColumn.IDENTITY_INFO_START)) {
+ Some(new IdentityColumnSpec(
+ col.metadata.getLong(IdentityColumn.IDENTITY_INFO_START),
+ col.metadata.getLong(IdentityColumn.IDENTITY_INFO_STEP),
+ col.metadata.getBoolean(IdentityColumn.IDENTITY_INFO_ALLOW_EXPLICIT_INSERT)
+ ))
+ } else {
+ None
+ }
ColumnDefinition(
col.name,
col.dataType,
@@ -104,6 +132,7 @@ object ColumnDefinition {
col.getComment(),
defaultValue,
generationExpr,
+ identityColumnSpec,
metadataBuilder.build()
)
}
@@ -124,18 +153,8 @@ object ColumnDefinition {
s"Command $cmd should not have column default value expression.")
}
cmd.columns.foreach { col =>
- if (col.defaultValue.isDefined && col.generationExpression.isDefined) {
- throw new AnalysisException(
- errorClass = "GENERATED_COLUMN_WITH_DEFAULT_VALUE",
- messageParameters = Map(
- "colName" -> col.name,
- "defaultValue" -> col.defaultValue.get.originalSQL,
- "genExpr" -> col.generationExpression.get
- )
- )
- }
-
col.defaultValue.foreach { default =>
+ checkDefaultColumnConflicts(col)
validateDefaultValueExpr(default, statement, col.name, col.dataType)
}
}
@@ -143,6 +162,29 @@ object ColumnDefinition {
case _ =>
}
}
+
+ private def checkDefaultColumnConflicts(col: ColumnDefinition): Unit = {
+ if (col.generationExpression.isDefined) {
+ throw new AnalysisException(
+ errorClass = "GENERATED_COLUMN_WITH_DEFAULT_VALUE",
+ messageParameters = Map(
+ "colName" -> col.name,
+ "defaultValue" -> col.defaultValue.get.originalSQL,
+ "genExpr" -> col.generationExpression.get
+ )
+ )
+ }
+ if (col.identityColumnSpec.isDefined) {
+ throw new AnalysisException(
+ errorClass = "IDENTITY_COLUMN_WITH_DEFAULT_VALUE",
+ messageParameters = Map(
+ "colName" -> col.name,
+ "defaultValue" -> col.defaultValue.get.originalSQL,
+ "identityColumnSpec" -> col.identityColumnSpec.get.toString
+ )
+ )
+ }
+ }
}
/**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ExecutableDuringAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ExecutableDuringAnalysis.scala
new file mode 100644
index 0000000000000..dc8dbf701f6a9
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ExecutableDuringAnalysis.scala
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+/**
+ * A logical plan node that requires execution during analysis.
+ */
+trait ExecutableDuringAnalysis extends LogicalPlan {
+ /**
+ * Returns the logical plan node that should be used for EXPLAIN.
+ */
+ def stageForExplain(): LogicalPlan
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/FunctionBuilderBase.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/FunctionBuilderBase.scala
index 4701f4ea1e172..75b2fcd3a5f34 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/FunctionBuilderBase.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/FunctionBuilderBase.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.sql.catalyst.expressions.{Expression, NamedArgumentExpression}
+import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns
+import org.apache.spark.sql.connector.catalog.procedures.{BoundProcedure, ProcedureParameter}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.util.ArrayImplicits._
@@ -122,12 +124,32 @@ object NamedParametersSupport {
functionSignature: FunctionSignature,
args: Seq[Expression],
functionName: String): Seq[Expression] = {
- val parameters: Seq[InputParameter] = functionSignature.parameters
+ defaultRearrange(functionName, functionSignature.parameters, args)
+ }
+
+ final def defaultRearrange(procedure: BoundProcedure, args: Seq[Expression]): Seq[Expression] = {
+ defaultRearrange(
+ procedure.name,
+ procedure.parameters.map(toInputParameter).toSeq,
+ args)
+ }
+
+ private def toInputParameter(param: ProcedureParameter): InputParameter = {
+ val defaultValue = Option(param.defaultValueExpression).map { expr =>
+ ResolveDefaultColumns.analyze(param.name, param.dataType, expr, "CALL")
+ }
+ InputParameter(param.name, defaultValue)
+ }
+
+ private def defaultRearrange(
+ routineName: String,
+ parameters: Seq[InputParameter],
+ args: Seq[Expression]): Seq[Expression] = {
if (parameters.dropWhile(_.default.isEmpty).exists(_.default.isEmpty)) {
- throw QueryCompilationErrors.unexpectedRequiredParameter(functionName, parameters)
+ throw QueryCompilationErrors.unexpectedRequiredParameter(routineName, parameters)
}
- val (positionalArgs, namedArgs) = splitAndCheckNamedArguments(args, functionName)
+ val (positionalArgs, namedArgs) = splitAndCheckNamedArguments(args, routineName)
val namedParameters: Seq[InputParameter] = parameters.drop(positionalArgs.size)
// The following loop checks for the following:
@@ -140,12 +162,12 @@ object NamedParametersSupport {
namedArgs.foreach { namedArg =>
val parameterName = namedArg.key
if (!parameterNamesSet.contains(parameterName)) {
- throw QueryCompilationErrors.unrecognizedParameterName(functionName, namedArg.key,
+ throw QueryCompilationErrors.unrecognizedParameterName(routineName, namedArg.key,
parameterNamesSet.toSeq)
}
if (positionalParametersSet.contains(parameterName)) {
throw QueryCompilationErrors.positionalAndNamedArgumentDoubleReference(
- functionName, namedArg.key)
+ routineName, namedArg.key)
}
}
@@ -154,7 +176,7 @@ object NamedParametersSupport {
val validParameterSizes =
Array.range(parameters.count(_.default.isEmpty), parameters.size + 1).toImmutableArraySeq
throw QueryCompilationErrors.wrongNumArgsError(
- functionName, validParameterSizes, args.length)
+ routineName, validParameterSizes, args.length)
}
// This constructs a map from argument name to value for argument rearrangement.
@@ -168,7 +190,7 @@ object NamedParametersSupport {
namedArgMap.getOrElse(
param.name,
if (param.default.isEmpty) {
- throw QueryCompilationErrors.requiredParameterNotFound(functionName, param.name, index)
+ throw QueryCompilationErrors.requiredParameterNotFound(routineName, param.name, index)
} else {
param.default.get
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MultiResult.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MultiResult.scala
new file mode 100644
index 0000000000000..f249e5c87eba2
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MultiResult.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.plans.logical
+
+import org.apache.spark.sql.catalyst.expressions.Attribute
+
+case class MultiResult(children: Seq[LogicalPlan]) extends LogicalPlan {
+
+ override def output: Seq[Attribute] = children.lastOption.map(_.output).getOrElse(Nil)
+
+ override protected def withNewChildrenInternal(
+ newChildren: IndexedSeq[LogicalPlan]): MultiResult = {
+ copy(children = newChildren)
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index e784e6695dbd0..90af6333b2e0b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.catalyst.plans.logical
-import org.apache.spark.sql.catalyst.{AliasIdentifier, SQLConfHelper}
+import org.apache.spark.sql.catalyst.{AliasIdentifier, InternalRow, SQLConfHelper}
import org.apache.spark.sql.catalyst.analysis.{AnsiTypeCoercion, MultiInstanceRelation, Resolver, TypeCoercion, TypeCoercionBase, UnresolvedUnaryNode}
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable}
import org.apache.spark.sql.catalyst.catalog.CatalogTable.VIEW_STORING_ANALYZED_PLAN
@@ -992,12 +992,18 @@ object Range {
castAndEval[Int](expression, IntegerType, paramIndex, paramName)
}
+// scalastyle:off line.size.limit
@ExpressionDescription(
usage = """
- _FUNC_(start: long, end: long, step: long, numSlices: integer)
- _FUNC_(start: long, end: long, step: long)
- _FUNC_(start: long, end: long)
- _FUNC_(end: long)""",
+ _FUNC_(start[, end[, step[, numSlices]]]) / _FUNC_(end) - Returns a table of values within a specified range.
+ """,
+ arguments = """
+ Arguments:
+ * start - An optional BIGINT literal defaulted to 0, marking the first value generated.
+ * end - A BIGINT literal marking endpoint (exclusive) of the number generation.
+ * step - An optional BIGINT literal defaulted to 1, specifying the increment used when generating values.
+ * numParts - An optional INTEGER literal specifying how the production of rows is spread across partitions.
+ """,
examples = """
Examples:
> SELECT * FROM _FUNC_(1);
@@ -1023,6 +1029,7 @@ object Range {
""",
since = "2.0.0",
group = "table_funcs")
+// scalastyle:on line.size.limit
case class Range(
start: Long,
end: Long,
@@ -1440,7 +1447,7 @@ case class Offset(offsetExpr: Expression, child: LogicalPlan) extends OrderPrese
}
/**
- * A constructor for creating a pivot, which will later be converted to a [[Project]]
+ * A logical plan node for creating a pivot, which will later be converted to a [[Project]]
* or an [[Aggregate]] during the query analysis.
*
* @param groupByExprsOpt A sequence of group by expressions. This field should be None if coming
@@ -1474,9 +1481,27 @@ case class Pivot(
override protected def withNewChildInternal(newChild: LogicalPlan): Pivot = copy(child = newChild)
}
+/**
+ * A logical plan node for transpose, which will later be converted to a [[LocalRelation]]
+ * at ReplaceTranspose during the query optimization.
+ *
+ * The result of the transpose operation is held in the `data` field, and the corresponding
+ * schema is stored in the `output` field. The `Transpose` node does not depend on any child
+ * logical plans after the data has been collected and transposed.
+ *
+ * @param output A sequence of output attributes representing the schema of the transposed data.
+ * @param data A sequence of [[InternalRow]] containing the transposed data.
+ */
+case class Transpose(
+ output: Seq[Attribute],
+ data: Seq[InternalRow] = Nil
+) extends LeafNode {
+ final override val nodePatterns: Seq[TreePattern] = Seq(TRANSPOSE)
+}
+
/**
- * A constructor for creating an Unpivot, which will later be converted to an [[Expand]]
+ * A logical plan node for creating an Unpivot, which will later be converted to an [[Expand]]
* during the query analysis.
*
* Either ids or values array must be set. The ids array can be empty,
@@ -1582,7 +1607,7 @@ case class Unpivot(
}
/**
- * A constructor for creating a logical limit, which is split into two separate logical nodes:
+ * A logical plan node for creating a logical limit, which is split into two separate logical nodes:
* a [[LocalLimit]], which is a partition local limit, followed by a [[GlobalLimit]].
*
* This muds the water for clean logical/physical separation, and is done for better limit pushdown.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index 05628d7b1c98e..b465e0e11612f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -19,17 +19,22 @@ package org.apache.spark.sql.catalyst.plans.logical
import org.apache.spark.{SparkIllegalArgumentException, SparkUnsupportedOperationException}
import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.analysis.{AnalysisContext, AssignmentUtils, EliminateSubqueryAliases, FieldName, NamedRelation, PartitionSpec, ResolvedIdentifier, UnresolvedException, ViewSchemaMode}
+import org.apache.spark.sql.catalyst.analysis.{AnalysisContext, AssignmentUtils, EliminateSubqueryAliases, FieldName, NamedRelation, PartitionSpec, ResolvedIdentifier, ResolvedProcedure, TypeCheckResult, UnresolvedException, UnresolvedProcedure, ViewSchemaMode}
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{DataTypeMismatch, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.catalyst.catalog.FunctionResource
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, Expression, MetadataAttribute, NamedExpression, UnaryExpression, Unevaluable, V2ExpressionUtils}
import org.apache.spark.sql.catalyst.plans.DescribeCommandSchema
import org.apache.spark.sql.catalyst.trees.BinaryLike
-import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, RowDeltaUtils, WriteDeltaProjections}
+import org.apache.spark.sql.catalyst.util.{quoteIfNeeded, truncatedString, CharVarcharUtils, RowDeltaUtils, WriteDeltaProjections}
+import org.apache.spark.sql.catalyst.util.TypeUtils.{ordinalNumber, toSQLExpr}
import org.apache.spark.sql.connector.catalog._
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.{IdentifierHelper, MultipartIdentifierHelper}
+import org.apache.spark.sql.connector.catalog.procedures.BoundProcedure
import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.connector.expressions.filter.Predicate
import org.apache.spark.sql.connector.write.{DeltaWrite, RowLevelOperation, RowLevelOperationTable, SupportsDelta, Write}
+import org.apache.spark.sql.errors.DataTypeErrors.toSQLType
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
import org.apache.spark.sql.types.{BooleanType, DataType, IntegerType, MapType, MetadataBuilder, StringType, StructField, StructType}
import org.apache.spark.util.ArrayImplicits._
@@ -796,9 +801,9 @@ case class MergeIntoTable(
object MergeIntoTable {
def getWritePrivileges(
- matchedActions: Seq[MergeAction],
- notMatchedActions: Seq[MergeAction],
- notMatchedBySourceActions: Seq[MergeAction]): Seq[TableWritePrivilege] = {
+ matchedActions: Iterable[MergeAction],
+ notMatchedActions: Iterable[MergeAction],
+ notMatchedBySourceActions: Iterable[MergeAction]): Seq[TableWritePrivilege] = {
val privileges = scala.collection.mutable.HashSet.empty[TableWritePrivilege]
(matchedActions.iterator ++ notMatchedActions ++ notMatchedBySourceActions).foreach {
case _: DeleteAction => privileges.add(TableWritePrivilege.DELETE)
@@ -1571,3 +1576,61 @@ case class SetVariable(
override protected def withNewChildInternal(newChild: LogicalPlan): SetVariable =
copy(sourceQuery = newChild)
}
+
+/**
+ * The logical plan of the CALL statement.
+ */
+case class Call(
+ procedure: LogicalPlan,
+ args: Seq[Expression],
+ execute: Boolean = true)
+ extends UnaryNode with ExecutableDuringAnalysis {
+
+ override def output: Seq[Attribute] = Nil
+
+ override def child: LogicalPlan = procedure
+
+ def bound: Boolean = procedure match {
+ case ResolvedProcedure(_, _, _: BoundProcedure) => true
+ case _ => false
+ }
+
+ def checkArgTypes(): TypeCheckResult = {
+ require(resolved && bound, "can check arg types only after resolution and binding")
+
+ val params = procedure match {
+ case ResolvedProcedure(_, _, bound: BoundProcedure) => bound.parameters
+ }
+ require(args.length == params.length, "number of args and params must match after binding")
+
+ args.zip(params).zipWithIndex.collectFirst {
+ case ((arg, param), idx)
+ if !DataType.equalsIgnoreCompatibleNullability(arg.dataType, param.dataType) =>
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> ordinalNumber(idx),
+ "requiredType" -> toSQLType(param.dataType),
+ "inputSql" -> toSQLExpr(arg),
+ "inputType" -> toSQLType(arg.dataType)))
+ }.getOrElse(TypeCheckSuccess)
+ }
+
+ override def simpleString(maxFields: Int): String = {
+ val name = procedure match {
+ case ResolvedProcedure(catalog, ident, _) =>
+ s"${quoteIfNeeded(catalog.name)}.${ident.quoted}"
+ case UnresolvedProcedure(nameParts) =>
+ nameParts.quoted
+ }
+ val argsString = truncatedString(args, ", ", maxFields)
+ s"Call $name($argsString)"
+ }
+
+ override def stageForExplain(): Call = {
+ copy(execute = false)
+ }
+
+ override protected def withNewChildInternal(newChild: LogicalPlan): Call =
+ copy(procedure = newChild)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
index 8be7aac7bebf5..b5556cbae7cd9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
@@ -54,6 +54,7 @@ object RuleIdCollection {
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveDeserializer" ::
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveEncodersInUDF" ::
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions" ::
+ "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveProcedures" ::
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveGenerate" ::
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveGroupingAnalytics" ::
"org.apache.spark.sql.catalyst.analysis.ResolveHigherOrderFunctions" ::
@@ -70,6 +71,7 @@ object RuleIdCollection {
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveSubqueryColumnAliases" ::
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveTables" ::
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveTempViews" ::
+ "org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveTranspose" ::
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveUnpivot" ::
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveUpCast" ::
"org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveUserSpecifiedColumns" ::
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
index 6258bd615b440..0f1c98b53e0b3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
@@ -72,6 +72,7 @@ object TreePattern extends Enumeration {
val NOT: Value = Value
val NULL_CHECK: Value = Value
val NULL_LITERAL: Value = Value
+ val PIPE_OPERATOR_SELECT: Value = Value
val SERIALIZE_FROM_OBJECT: Value = Value
val OR: Value = Value
val OUTER_REFERENCE: Value = Value
@@ -92,6 +93,7 @@ object TreePattern extends Enumeration {
val SUM: Value = Value
val TIME_WINDOW: Value = Value
val TIME_ZONE_AWARE_EXPRESSION: Value = Value
+ val TRANSPOSE: Value = Value
val TRUE_OR_FALSE_LITERAL: Value = Value
val VARIANT_GET: Value = Value
val WINDOW_EXPRESSION: Value = Value
@@ -155,8 +157,10 @@ object TreePattern extends Enumeration {
// Unresolved Plan patterns (Alphabetically ordered)
val UNRESOLVED_FUNC: Value = Value
+ val UNRESOLVED_PROCEDURE: Value = Value
val UNRESOLVED_SUBQUERY_COLUMN_ALIAS: Value = Value
val UNRESOLVED_TABLE_VALUED_FUNCTION: Value = Value
+ val UNRESOLVED_TRANSPOSE: Value = Value
val UNRESOLVED_TVF_ALIASES: Value = Value
// Execution expression patterns (alphabetically ordered)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index f1c36f2f5c28f..e27ce29fc2318 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -389,7 +389,7 @@ object DateTimeUtils extends SparkDateTimeUtils {
case "SA" | "SAT" | "SATURDAY" => SATURDAY
case _ =>
throw new SparkIllegalArgumentException(
- errorClass = "_LEGACY_ERROR_TEMP_3209",
+ errorClass = "ILLEGAL_DAY_OF_WEEK",
messageParameters = Map("string" -> string.toString))
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IdentityColumn.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IdentityColumn.scala
new file mode 100644
index 0000000000000..26a3cb026d317
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IdentityColumn.scala
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.sql.connector.catalog.{Identifier, IdentityColumnSpec, TableCatalog, TableCatalogCapability}
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.types.{StructField, StructType}
+
+/**
+ * This object contains utility methods and values for Identity Columns
+ */
+object IdentityColumn {
+ val IDENTITY_INFO_START = "identity.start"
+ val IDENTITY_INFO_STEP = "identity.step"
+ val IDENTITY_INFO_ALLOW_EXPLICIT_INSERT = "identity.allowExplicitInsert"
+
+ /**
+ * If `schema` contains any generated columns, check whether the table catalog supports identity
+ * columns. Otherwise throw an error.
+ */
+ def validateIdentityColumn(
+ schema: StructType,
+ catalog: TableCatalog,
+ ident: Identifier): Unit = {
+ if (hasIdentityColumns(schema)) {
+ if (!catalog
+ .capabilities()
+ .contains(TableCatalogCapability.SUPPORTS_CREATE_TABLE_WITH_IDENTITY_COLUMNS)) {
+ throw QueryCompilationErrors.unsupportedTableOperationError(
+ catalog, ident, operation = "identity column"
+ )
+ }
+ }
+ }
+
+ /**
+ * Whether the given `field` is an identity column
+ */
+ def isIdentityColumn(field: StructField): Boolean = {
+ field.metadata.contains(IDENTITY_INFO_START)
+ }
+
+ /**
+ * Returns the identity information stored in the column metadata if it exists
+ */
+ def getIdentityInfo(field: StructField): Option[IdentityColumnSpec] = {
+ if (isIdentityColumn(field)) {
+ Some(new IdentityColumnSpec(
+ field.metadata.getString(IDENTITY_INFO_START).toLong,
+ field.metadata.getString(IDENTITY_INFO_STEP).toLong,
+ field.metadata.getString(IDENTITY_INFO_ALLOW_EXPLICIT_INSERT).toBoolean))
+ } else {
+ None
+ }
+ }
+
+ /**
+ * Whether the `schema` has one or more identity columns
+ */
+ def hasIdentityColumns(schema: StructType): Boolean = {
+ schema.exists(isIdentityColumn)
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 0067114e36fdd..90c802b7e28df 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -105,16 +105,18 @@ object IntervalUtils extends SparkIntervalUtils {
endField: Byte,
intervalStr: String,
typeName: String,
- fallBackNotice: Option[String] = None) = {
+ fallBackNotice: Boolean = false) = {
throw new SparkIllegalArgumentException(
- errorClass = "_LEGACY_ERROR_TEMP_3214",
+ errorClass = {
+ if (fallBackNotice) "INVALID_INTERVAL_FORMAT.UNMATCHED_FORMAT_STRING_WITH_NOTICE"
+ else "INVALID_INTERVAL_FORMAT.UNMATCHED_FORMAT_STRING"
+ },
messageParameters = Map(
"intervalStr" -> intervalStr,
"supportedFormat" -> supportedFormat((intervalStr, startFiled, endField))
.map(format => s"`$format`").mkString(", "),
"typeName" -> typeName,
- "input" -> input.toString,
- "fallBackNotice" -> fallBackNotice.map(s => s", $s").getOrElse("")))
+ "input" -> input.toString))
}
val supportedFormat = Map(
@@ -145,14 +147,15 @@ object IntervalUtils extends SparkIntervalUtils {
def checkTargetType(targetStartField: Byte, targetEndField: Byte): Boolean =
startField == targetStartField && endField == targetEndField
- input.trimAll().toString match {
+ val trimmedInput = input.trimAll().toString
+ trimmedInput match {
case yearMonthRegex(sign, year, month) if checkTargetType(YM.YEAR, YM.MONTH) =>
- toYMInterval(year, month, finalSign(sign))
+ toYMInterval(year, month, trimmedInput, finalSign(sign))
case yearMonthLiteralRegex(firstSign, secondSign, year, month)
if checkTargetType(YM.YEAR, YM.MONTH) =>
- toYMInterval(year, month, finalSign(firstSign, secondSign))
+ toYMInterval(year, month, trimmedInput, finalSign(firstSign, secondSign))
case yearMonthIndividualRegex(firstSign, value) =>
- safeToInterval("year-month") {
+ safeToInterval("year-month", trimmedInput) {
val sign = finalSign(firstSign)
if (endField == YM.YEAR) {
sign * Math.toIntExact(value.toLong * MONTHS_PER_YEAR)
@@ -164,7 +167,7 @@ object IntervalUtils extends SparkIntervalUtils {
}
}
case yearMonthIndividualLiteralRegex(firstSign, secondSign, value, unit) =>
- safeToInterval("year-month") {
+ safeToInterval("year-month", trimmedInput) {
val sign = finalSign(firstSign, secondSign)
unit.toUpperCase(Locale.ROOT) match {
case "YEAR" if checkTargetType(YM.YEAR, YM.YEAR) =>
@@ -202,21 +205,21 @@ object IntervalUtils extends SparkIntervalUtils {
new CalendarInterval(months, 0, 0)
}
- private def safeToInterval[T](interval: String)(f: => T): T = {
+ private def safeToInterval[T](interval: String, input: String)(f: => T): T = {
try {
f
} catch {
case e: SparkThrowable => throw e
case NonFatal(e) =>
throw new SparkIllegalArgumentException(
- errorClass = "_LEGACY_ERROR_TEMP_3213",
- messageParameters = Map("interval" -> interval, "msg" -> e.getMessage),
+ errorClass = "INVALID_INTERVAL_FORMAT.INTERVAL_PARSING",
+ messageParameters = Map("input" -> input, "interval" -> interval),
cause = e)
}
}
- private def toYMInterval(year: String, month: String, sign: Int): Int = {
- safeToInterval("year-month") {
+ private def toYMInterval(year: String, month: String, input: String, sign: Int): Int = {
+ safeToInterval("year-month", input) {
val years = toLongWithRange(yearStr, year, 0, Integer.MAX_VALUE / MONTHS_PER_YEAR)
val totalMonths =
sign * (years * MONTHS_PER_YEAR + toLongWithRange(monthStr, month, 0, 11))
@@ -285,7 +288,8 @@ object IntervalUtils extends SparkIntervalUtils {
def checkTargetType(targetStartField: Byte, targetEndField: Byte): Boolean =
startField == targetStartField && endField == targetEndField
- input.trimAll().toString match {
+ val trimmedInput = input.trimAll().toString
+ trimmedInput match {
case dayHourRegex(sign, day, hour) if checkTargetType(DT.DAY, DT.HOUR) =>
toDTInterval(day, hour, "0", "0", finalSign(sign))
case dayHourLiteralRegex(firstSign, secondSign, day, hour)
@@ -324,7 +328,7 @@ object IntervalUtils extends SparkIntervalUtils {
toDTInterval(minute, secondAndMicro(second, micro), finalSign(firstSign, secondSign))
case dayTimeIndividualRegex(firstSign, value, suffix) =>
- safeToInterval("day-time") {
+ safeToInterval("day-time", trimmedInput) {
val sign = finalSign(firstSign)
(startField, endField) match {
case (DT.DAY, DT.DAY) if suffix == null && value.length <= 9 =>
@@ -339,11 +343,11 @@ object IntervalUtils extends SparkIntervalUtils {
case -1 => parseSecondNano(s"-${secondAndMicro(value, suffix)}")
}
case (_, _) => throwIllegalIntervalFormatException(input, startField, endField,
- "day-time", DT(startField, endField).typeName, Some(fallbackNotice))
+ "day-time", DT(startField, endField).typeName, true)
}
}
case dayTimeIndividualLiteralRegex(firstSign, secondSign, value, suffix, unit) =>
- safeToInterval("day-time") {
+ safeToInterval("day-time", trimmedInput) {
val sign = finalSign(firstSign, secondSign)
unit.toUpperCase(Locale.ROOT) match {
case "DAY" if suffix == null && value.length <= 9 && checkTargetType(DT.DAY, DT.DAY) =>
@@ -360,11 +364,11 @@ object IntervalUtils extends SparkIntervalUtils {
case -1 => parseSecondNano(s"-${secondAndMicro(value, suffix)}")
}
case _ => throwIllegalIntervalFormatException(input, startField, endField,
- "day-time", DT(startField, endField).typeName, Some(fallbackNotice))
+ "day-time", DT(startField, endField).typeName, true)
}
}
case _ => throwIllegalIntervalFormatException(input, startField, endField,
- "day-time", DT(startField, endField).typeName, Some(fallbackNotice))
+ "day-time", DT(startField, endField).typeName, true)
}
}
@@ -512,7 +516,7 @@ object IntervalUtils extends SparkIntervalUtils {
case DT.SECOND =>
// No-op
case _ => throw new SparkIllegalArgumentException(
- errorClass = "_LEGACY_ERROR_TEMP_3212",
+ errorClass = "INVALID_INTERVAL_FORMAT.UNSUPPORTED_FROM_TO_EXPRESSION",
messageParameters = Map(
"input" -> input,
"from" -> DT.fieldToString(from),
@@ -524,10 +528,14 @@ object IntervalUtils extends SparkIntervalUtils {
micros = Math.addExact(micros, Math.multiplyExact(seconds, MICROS_PER_SECOND))
new CalendarInterval(0, sign * days, sign * micros)
} catch {
+ // Bypass SparkIllegalArgumentExceptions
+ case se: SparkIllegalArgumentException => throw se
case e: Exception =>
throw new SparkIllegalArgumentException(
- errorClass = "_LEGACY_ERROR_TEMP_3211",
- messageParameters = Map("msg" -> e.getMessage),
+ errorClass = "INVALID_INTERVAL_FORMAT.DAY_TIME_PARSING",
+ messageParameters = Map(
+ "msg" -> e.getMessage,
+ "input" -> input),
cause = e)
}
}
@@ -564,7 +572,9 @@ object IntervalUtils extends SparkIntervalUtils {
case Array(secondsStr, nanosStr) =>
val seconds = parseSeconds(secondsStr)
Math.addExact(seconds, parseNanos(nanosStr, seconds < 0))
- case _ => throw new SparkIllegalArgumentException("_LEGACY_ERROR_TEMP_3210")
+ case _ => throw new SparkIllegalArgumentException(
+ errorClass = "INVALID_INTERVAL_FORMAT.SECOND_NANO_FORMAT",
+ messageParameters = Map("input" -> secondNano))
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
index 8b7392e71249e..693ac8d94dbcf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.util
import scala.collection.mutable.ArrayBuffer
-import org.apache.spark.{SparkThrowable, SparkUnsupportedOperationException}
+import org.apache.spark.{SparkException, SparkThrowable, SparkUnsupportedOperationException}
import org.apache.spark.internal.{Logging, MDC}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.sql.AnalysisException
@@ -412,8 +412,11 @@ object ResolveDefaultColumns extends QueryErrorsBase
case _: ExprLiteral | _: Cast => expr
}
} catch {
- case _: AnalysisException | _: MatchError =>
- throw QueryCompilationErrors.failedToParseExistenceDefaultAsLiteral(field.name, text)
+ // AnalysisException thrown from analyze is already formatted, throw it directly.
+ case ae: AnalysisException => throw ae
+ case _: MatchError =>
+ throw SparkException.internalError(s"parse existence default as literal err," +
+ s" field name: ${field.name}, value: $text")
}
// The expression should be a literal value by this point, possibly wrapped in a cast
// function. This is enforced by the execution of commands that assign default values.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
index 65bdae85be12a..282350dda67d3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
@@ -126,6 +126,13 @@ private[sql] object CatalogV2Implicits {
case _ =>
throw QueryCompilationErrors.missingCatalogAbilityError(plugin, "functions")
}
+
+ def asProcedureCatalog: ProcedureCatalog = plugin match {
+ case procedureCatalog: ProcedureCatalog =>
+ procedureCatalog
+ case _ =>
+ throw QueryCompilationErrors.missingCatalogAbilityError(plugin, "procedures")
+ }
}
implicit class NamespaceHelper(namespace: Array[String]) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
index 6698f0a021400..9b7f68070a1a4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.analysis.{AsOfTimestamp, AsOfVersion, Named
import org.apache.spark.sql.catalyst.catalog.ClusterBySpec
import org.apache.spark.sql.catalyst.expressions.Literal
import org.apache.spark.sql.catalyst.plans.logical.{SerdeInfo, TableSpec}
-import org.apache.spark.sql.catalyst.util.GeneratedColumn
+import org.apache.spark.sql.catalyst.util.{GeneratedColumn, IdentityColumn}
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._
import org.apache.spark.sql.connector.catalog.TableChange._
import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
@@ -579,18 +579,10 @@ private[sql] object CatalogV2Util {
val isDefaultColumn = f.getCurrentDefaultValue().isDefined &&
f.getExistenceDefaultValue().isDefined
val isGeneratedColumn = GeneratedColumn.isGeneratedColumn(f)
- if (isDefaultColumn && isGeneratedColumn) {
- throw new AnalysisException(
- errorClass = "GENERATED_COLUMN_WITH_DEFAULT_VALUE",
- messageParameters = Map(
- "colName" -> f.name,
- "defaultValue" -> f.getCurrentDefaultValue().get,
- "genExpr" -> GeneratedColumn.getGenerationExpression(f).get
- )
- )
- }
-
+ val isIdentityColumn = IdentityColumn.isIdentityColumn(f)
if (isDefaultColumn) {
+ checkDefaultColumnConflicts(f)
+
val e = analyze(
f,
statementType = "Column analysis",
@@ -611,10 +603,41 @@ private[sql] object CatalogV2Util {
Seq("comment", GeneratedColumn.GENERATION_EXPRESSION_METADATA_KEY))
Column.create(f.name, f.dataType, f.nullable, f.getComment().orNull,
GeneratedColumn.getGenerationExpression(f).get, metadataAsJson(cleanedMetadata))
+ } else if (isIdentityColumn) {
+ val cleanedMetadata = metadataWithKeysRemoved(
+ Seq("comment",
+ IdentityColumn.IDENTITY_INFO_START,
+ IdentityColumn.IDENTITY_INFO_STEP,
+ IdentityColumn.IDENTITY_INFO_ALLOW_EXPLICIT_INSERT))
+ Column.create(f.name, f.dataType, f.nullable, f.getComment().orNull,
+ IdentityColumn.getIdentityInfo(f).get, metadataAsJson(cleanedMetadata))
} else {
val cleanedMetadata = metadataWithKeysRemoved(Seq("comment"))
Column.create(f.name, f.dataType, f.nullable, f.getComment().orNull,
metadataAsJson(cleanedMetadata))
}
}
+
+ private def checkDefaultColumnConflicts(f: StructField): Unit = {
+ if (GeneratedColumn.isGeneratedColumn(f)) {
+ throw new AnalysisException(
+ errorClass = "GENERATED_COLUMN_WITH_DEFAULT_VALUE",
+ messageParameters = Map(
+ "colName" -> f.name,
+ "defaultValue" -> f.getCurrentDefaultValue().get,
+ "genExpr" -> GeneratedColumn.getGenerationExpression(f).get
+ )
+ )
+ }
+ if (IdentityColumn.isIdentityColumn(f)) {
+ throw new AnalysisException(
+ errorClass = "IDENTITY_COLUMN_WITH_DEFAULT_VALUE",
+ messageParameters = Map(
+ "colName" -> f.name,
+ "defaultValue" -> f.getCurrentDefaultValue().get,
+ "identityColumnSpec" -> IdentityColumn.getIdentityInfo(f).get.toString
+ )
+ )
+ }
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index fa8ea2f5289fa..0b5255e95f073 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -853,6 +853,13 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat
origin = origin)
}
+ def failedToLoadRoutineError(nameParts: Seq[String], e: Exception): Throwable = {
+ new AnalysisException(
+ errorClass = "FAILED_TO_LOAD_ROUTINE",
+ messageParameters = Map("routineName" -> toSQLId(nameParts)),
+ cause = Some(e))
+ }
+
def unresolvedRoutineError(name: FunctionIdentifier, searchPath: Seq[String]): Throwable = {
new AnalysisException(
errorClass = "UNRESOLVED_ROUTINE",
@@ -3516,29 +3523,21 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat
"cond" -> toSQLExpr(cond)))
}
- def failedToParseExistenceDefaultAsLiteral(fieldName: String, defaultValue: String): Throwable = {
- new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1344",
- messageParameters = Map(
- "fieldName" -> fieldName,
- "defaultValue" -> defaultValue))
- }
-
def defaultReferencesNotAllowedInDataSource(
statementType: String, dataSource: String): Throwable = {
new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1345",
+ errorClass = "DEFAULT_UNSUPPORTED",
messageParameters = Map(
- "statementType" -> statementType,
+ "statementType" -> toSQLStmt(statementType),
"dataSource" -> dataSource))
}
def addNewDefaultColumnToExistingTableNotAllowed(
statementType: String, dataSource: String): Throwable = {
new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1346",
+ errorClass = "ADD_DEFAULT_UNSUPPORTED",
messageParameters = Map(
- "statementType" -> statementType,
+ "statementType" -> toSQLStmt(statementType),
"dataSource" -> dataSource))
}
@@ -3959,6 +3958,14 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat
messageParameters = Map("provider" -> name))
}
+ def externalDataSourceException(cause: Throwable): Throwable = {
+ new AnalysisException(
+ errorClass = "DATA_SOURCE_EXTERNAL_ERROR",
+ messageParameters = Map(),
+ cause = Some(cause)
+ )
+ }
+
def foundMultipleDataSources(provider: String): Throwable = {
new AnalysisException(
errorClass = "FOUND_MULTIPLE_DATA_SOURCES",
@@ -4090,10 +4097,33 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat
)
}
+ def avroOptionsException(optionName: String, message: String): Throwable = {
+ new AnalysisException(
+ errorClass = "STDS_INVALID_OPTION_VALUE.WITH_MESSAGE",
+ messageParameters = Map("optionName" -> optionName, "message" -> message)
+ )
+ }
+
def protobufNotLoadedSqlFunctionsUnusable(functionName: String): Throwable = {
new AnalysisException(
errorClass = "PROTOBUF_NOT_LOADED_SQL_FUNCTIONS_UNUSABLE",
messageParameters = Map("functionName" -> functionName)
)
}
+
+ def pipeOperatorSelectContainsAggregateFunction(expr: Expression): Throwable = {
+ new AnalysisException(
+ errorClass = "PIPE_OPERATOR_SELECT_CONTAINS_AGGREGATE_FUNCTION",
+ messageParameters = Map(
+ "expr" -> expr.toString),
+ origin = expr.origin)
+ }
+
+ def inlineTableContainsScalarSubquery(inlineTable: LogicalPlan): Throwable = {
+ new AnalysisException(
+ errorClass = "UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.SCALAR_SUBQUERY_IN_VALUES",
+ messageParameters = Map.empty,
+ origin = inlineTable.origin
+ )
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index d6e23fcc65cd4..4bc071155012b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -79,8 +79,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
messageParameters = Map(
"value" -> toSQLValue(t, from),
"sourceType" -> toSQLType(from),
- "targetType" -> toSQLType(to),
- "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)),
+ "targetType" -> toSQLType(to)),
context = Array.empty,
summary = "")
}
@@ -124,8 +123,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
messageParameters = Map(
"expression" -> toSQLValue(s, StringType),
"sourceType" -> toSQLType(StringType),
- "targetType" -> toSQLType(BooleanType),
- "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)),
+ "targetType" -> toSQLType(BooleanType)),
context = getQueryContext(context),
summary = getSummary(context))
}
@@ -139,8 +137,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
messageParameters = Map(
"expression" -> toSQLValue(s, StringType),
"sourceType" -> toSQLType(StringType),
- "targetType" -> toSQLType(to),
- "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)),
+ "targetType" -> toSQLType(to)),
context = getQueryContext(context),
summary = getSummary(context))
}
@@ -387,18 +384,6 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
s"The aggregate window function ${toSQLId(funcName)} does not support merging.")
}
- def dataTypeUnexpectedError(dataType: DataType): SparkUnsupportedOperationException = {
- new SparkUnsupportedOperationException(
- errorClass = "_LEGACY_ERROR_TEMP_2011",
- messageParameters = Map("dataType" -> dataType.catalogString))
- }
-
- def typeUnsupportedError(dataType: DataType): SparkIllegalArgumentException = {
- new SparkIllegalArgumentException(
- errorClass = "_LEGACY_ERROR_TEMP_2011",
- messageParameters = Map("dataType" -> dataType.toString()))
- }
-
def negativeValueUnexpectedError(
frequencyExpression : Expression): SparkIllegalArgumentException = {
new SparkIllegalArgumentException(
@@ -1310,10 +1295,6 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
messageParameters = Map("badRecord" -> badRecord))
}
- def elementsOfTupleExceedLimitError(): SparkUnsupportedOperationException = {
- new SparkUnsupportedOperationException("_LEGACY_ERROR_TEMP_2150")
- }
-
def expressionDecodingError(e: Exception, expressions: Seq[Expression]): SparkRuntimeException = {
new SparkRuntimeException(
errorClass = "EXPRESSION_DECODING_FAILED",
@@ -1895,17 +1876,6 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
cause = null)
}
- def notPublicClassError(name: String): SparkUnsupportedOperationException = {
- new SparkUnsupportedOperationException(
- errorClass = "_LEGACY_ERROR_TEMP_2229",
- messageParameters = Map(
- "name" -> name))
- }
-
- def primitiveTypesNotSupportedError(): SparkUnsupportedOperationException = {
- new SparkUnsupportedOperationException(errorClass = "_LEGACY_ERROR_TEMP_2230")
- }
-
def onlySupportDataSourcesProvidingFileFormatError(providingClass: String): Throwable = {
new SparkException(
errorClass = "_LEGACY_ERROR_TEMP_2233",
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/SqlScriptingErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/SqlScriptingErrors.scala
index 61661b1d32f39..7f13dc334e06e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/SqlScriptingErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/SqlScriptingErrors.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.errors
import org.apache.spark.sql.catalyst.trees.Origin
+import org.apache.spark.sql.errors.DataTypeErrors.toSQLId
import org.apache.spark.sql.errors.QueryExecutionErrors.toSQLStmt
import org.apache.spark.sql.exceptions.SqlScriptingException
@@ -32,7 +33,7 @@ private[sql] object SqlScriptingErrors {
origin = origin,
errorClass = "LABELS_MISMATCH",
cause = null,
- messageParameters = Map("beginLabel" -> beginLabel, "endLabel" -> endLabel))
+ messageParameters = Map("beginLabel" -> toSQLId(beginLabel), "endLabel" -> toSQLId(endLabel)))
}
def endLabelWithoutBeginLabel(origin: Origin, endLabel: String): Throwable = {
@@ -40,29 +41,27 @@ private[sql] object SqlScriptingErrors {
origin = origin,
errorClass = "END_LABEL_WITHOUT_BEGIN_LABEL",
cause = null,
- messageParameters = Map("endLabel" -> endLabel))
+ messageParameters = Map("endLabel" -> toSQLId(endLabel)))
}
def variableDeclarationNotAllowedInScope(
origin: Origin,
- varName: String,
- lineNumber: String): Throwable = {
+ varName: Seq[String]): Throwable = {
new SqlScriptingException(
origin = origin,
errorClass = "INVALID_VARIABLE_DECLARATION.NOT_ALLOWED_IN_SCOPE",
cause = null,
- messageParameters = Map("varName" -> varName, "lineNumber" -> lineNumber))
+ messageParameters = Map("varName" -> toSQLId(varName)))
}
def variableDeclarationOnlyAtBeginning(
origin: Origin,
- varName: String,
- lineNumber: String): Throwable = {
+ varName: Seq[String]): Throwable = {
new SqlScriptingException(
origin = origin,
errorClass = "INVALID_VARIABLE_DECLARATION.ONLY_AT_BEGINNING",
cause = null,
- messageParameters = Map("varName" -> varName, "lineNumber" -> lineNumber))
+ messageParameters = Map("varName" -> toSQLId(varName)))
}
def invalidBooleanStatement(
@@ -84,4 +83,27 @@ private[sql] object SqlScriptingErrors {
cause = null,
messageParameters = Map("invalidStatement" -> toSQLStmt(stmt)))
}
+
+ def labelDoesNotExist(
+ origin: Origin,
+ labelName: String,
+ statementType: String): Throwable = {
+ new SqlScriptingException(
+ origin = origin,
+ errorClass = "INVALID_LABEL_USAGE.DOES_NOT_EXIST",
+ cause = null,
+ messageParameters = Map(
+ "labelName" -> toSQLStmt(labelName),
+ "statementType" -> statementType))
+ }
+
+ def invalidIterateLabelUsageForCompound(
+ origin: Origin,
+ labelName: String): Throwable = {
+ new SqlScriptingException(
+ origin = origin,
+ errorClass = "INVALID_LABEL_USAGE.ITERATE_IN_COMPOUND",
+ cause = null,
+ messageParameters = Map("labelName" -> toSQLStmt(labelName)))
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/exceptions/SqlScriptingException.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/exceptions/SqlScriptingException.scala
index 4354e7e3635e4..f0c28c95046eb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/exceptions/SqlScriptingException.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/exceptions/SqlScriptingException.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.exceptions.SqlScriptingException.errorMessageWithLin
class SqlScriptingException (
errorClass: String,
cause: Throwable,
- origin: Origin,
+ val origin: Origin,
messageParameters: Map[String, String] = Map.empty)
extends Exception(
errorMessageWithLineNumber(Option(origin), errorClass, messageParameters),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 72915f0e5c256..6c3e9bac1cfe5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1843,6 +1843,13 @@ object SQLConf {
.intConf
.createWithDefault(10000)
+ val DATAFRAME_TRANSPOSE_MAX_VALUES = buildConf("spark.sql.transposeMaxValues")
+ .doc("When doing a transpose without specifying values for the index column this is" +
+ " the maximum number of values that will be transposed without error.")
+ .version("4.0.0")
+ .intConf
+ .createWithDefault(500)
+
val RUN_SQL_ON_FILES = buildConf("spark.sql.runSQLOnFiles")
.internal()
.doc("When true, we could use `datasource`.`path` as table in SQL query.")
@@ -3162,6 +3169,29 @@ object SQLConf {
.version("4.0.0")
.fallbackConf(Python.PYTHON_WORKER_FAULTHANLDER_ENABLED)
+ val PYSPARK_PLOT_MAX_ROWS =
+ buildConf("spark.sql.pyspark.plotting.max_rows")
+ .doc(
+ "The visual limit on top-n-based plots. If set to 1000, the first 1000 data points " +
+ "will be used for plotting.")
+ .version("4.0.0")
+ .intConf
+ .createWithDefault(1000)
+
+ val PYSPARK_PLOT_SAMPLE_RATIO =
+ buildConf("spark.sql.pyspark.plotting.sample_ratio")
+ .doc(
+ "The proportion of data that will be plotted for sample-based plots. It is determined " +
+ "based on spark.sql.pyspark.plotting.max_rows if not explicitly set."
+ )
+ .version("4.0.0")
+ .doubleConf
+ .checkValue(
+ ratio => ratio >= 0.0 && ratio <= 1.0,
+ "The value should be between 0.0 and 1.0 inclusive."
+ )
+ .createOptional
+
val ARROW_SPARKR_EXECUTION_ENABLED =
buildConf("spark.sql.execution.arrow.sparkr.enabled")
.doc("When true, make use of Apache Arrow for columnar data transfers in SparkR. " +
@@ -3820,6 +3850,15 @@ object SQLConf {
.intConf
.createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
+ val PRUNE_FILTERS_CAN_PRUNE_STREAMING_SUBPLAN =
+ buildConf("spark.databricks.sql.optimizer.pruneFiltersCanPruneStreamingSubplan")
+ .internal()
+ .doc("Allow PruneFilters to remove streaming subplans when we encounter a false filter. " +
+ "This flag is to restore prior buggy behavior for broken pipelines.")
+ .version("4.0.0")
+ .booleanConf
+ .createWithDefault(false)
+
object Deprecated {
val MAPRED_REDUCE_TASKS = "mapred.reduce.tasks"
}
@@ -4982,6 +5021,15 @@ object SQLConf {
.stringConf
.createWithDefault("versionAsOf")
+ val OPERATOR_PIPE_SYNTAX_ENABLED =
+ buildConf("spark.sql.operatorPipeSyntaxEnabled")
+ .doc("If true, enable operator pipe syntax for Apache Spark SQL. This uses the operator " +
+ "pipe marker |> to indicate separation between clauses of SQL in a manner that describes " +
+ "the sequence of steps that the query performs in a composable fashion.")
+ .version("4.0.0")
+ .booleanConf
+ .createWithDefault(Utils.isTesting)
+
val LEGACY_PERCENTILE_DISC_CALCULATION = buildConf("spark.sql.legacy.percentileDiscCalculation")
.internal()
.doc("If true, the old bogus percentile_disc calculation is used. The old calculation " +
@@ -5712,6 +5760,8 @@ class SQLConf extends Serializable with Logging with SqlApiConf {
def dataFramePivotMaxValues: Int = getConf(DATAFRAME_PIVOT_MAX_VALUES)
+ def dataFrameTransposeMaxValues: Int = getConf(DATAFRAME_TRANSPOSE_MAX_VALUES)
+
def runSQLonFile: Boolean = getConf(RUN_SQL_ON_FILES)
def enableTwoLevelAggMap: Boolean = getConf(ENABLE_TWOLEVEL_AGG_MAP)
@@ -5846,6 +5896,10 @@ class SQLConf extends Serializable with Logging with SqlApiConf {
def pythonUDFWorkerFaulthandlerEnabled: Boolean = getConf(PYTHON_UDF_WORKER_FAULTHANLDER_ENABLED)
+ def pysparkPlotMaxRows: Int = getConf(PYSPARK_PLOT_MAX_ROWS)
+
+ def pysparkPlotSampleRatio: Option[Double] = getConf(PYSPARK_PLOT_SAMPLE_RATIO)
+
def arrowSparkREnabled: Boolean = getConf(ARROW_SPARKR_EXECUTION_ENABLED)
def arrowPySparkFallbackEnabled: Boolean = getConf(ARROW_PYSPARK_FALLBACK_ENABLED)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ColumnImpl.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ColumnImpl.scala
index 2a67ffc4bbef5..47889410561e3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ColumnImpl.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/ColumnImpl.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.internal.connector
-import org.apache.spark.sql.connector.catalog.{Column, ColumnDefaultValue}
+import org.apache.spark.sql.connector.catalog.{Column, ColumnDefaultValue, IdentityColumnSpec}
import org.apache.spark.sql.types.DataType
// The standard concrete implementation of data source V2 column.
@@ -28,4 +28,5 @@ case class ColumnImpl(
comment: String,
defaultValue: ColumnDefaultValue,
generationExpression: String,
+ identityColumnSpec: IdentityColumnSpec,
metadataInJSON: String) extends Column
diff --git a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtilsSuite.scala b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtilsSuite.scala
index 40e6182e587b3..50e933ba97ae6 100644
--- a/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtilsSuite.scala
+++ b/sql/catalyst/src/test/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtilsSuite.scala
@@ -349,7 +349,7 @@ class ExpressionImplUtilsSuite extends SparkFunSuite {
exception = intercept[SparkRuntimeException] {
f(t)
},
- errorClass = t.expectedErrorClassOpt.get,
+ condition = t.expectedErrorClassOpt.get,
parameters = t.errorParamsMap
)
}
@@ -361,7 +361,7 @@ class ExpressionImplUtilsSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
ExpressionImplUtils.validateUTF8String(str)
},
- errorClass = "INVALID_UTF8_STRING",
+ condition = "INVALID_UTF8_STRING",
parameters = Map(
"str" -> str.getBytes.map(byte => f"\\x$byte%02X").mkString
)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowJsonSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowJsonSuite.scala
index 3e72dc7da24b7..cf50063baa13c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RowJsonSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RowJsonSuite.scala
@@ -136,7 +136,7 @@ class RowJsonSuite extends SparkFunSuite {
new StructType().add("a", ObjectType(classOf[(Int, Int)])))
row.jsonValue
},
- errorClass = "FAILED_ROW_TO_JSON",
+ condition = "FAILED_ROW_TO_JSON",
parameters = Map(
"value" -> toSQLValue("(1,2)"),
"class" -> "class scala.Tuple2$mcII$sp",
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystTypeConvertersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystTypeConvertersSuite.scala
index 562aac766fc33..7572843f44a19 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystTypeConvertersSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystTypeConvertersSuite.scala
@@ -108,7 +108,7 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
CatalystTypeConverters.createToCatalystConverter(structType)("test")
},
- errorClass = "_LEGACY_ERROR_TEMP_3219",
+ condition = "_LEGACY_ERROR_TEMP_3219",
parameters = Map(
"other" -> "test",
"otherClass" -> "java.lang.String",
@@ -121,7 +121,7 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
CatalystTypeConverters.createToCatalystConverter(mapType)("test")
},
- errorClass = "_LEGACY_ERROR_TEMP_3221",
+ condition = "_LEGACY_ERROR_TEMP_3221",
parameters = Map(
"other" -> "test",
"otherClass" -> "java.lang.String",
@@ -135,7 +135,7 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
CatalystTypeConverters.createToCatalystConverter(arrayType)("test")
},
- errorClass = "_LEGACY_ERROR_TEMP_3220",
+ condition = "_LEGACY_ERROR_TEMP_3220",
parameters = Map(
"other" -> "test",
"otherClass" -> "java.lang.String",
@@ -148,7 +148,7 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
CatalystTypeConverters.createToCatalystConverter(decimalType)("test")
},
- errorClass = "_LEGACY_ERROR_TEMP_3219",
+ condition = "_LEGACY_ERROR_TEMP_3219",
parameters = Map(
"other" -> "test",
"otherClass" -> "java.lang.String",
@@ -160,7 +160,7 @@ class CatalystTypeConvertersSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
CatalystTypeConverters.createToCatalystConverter(StringType)(0.1)
},
- errorClass = "_LEGACY_ERROR_TEMP_3219",
+ condition = "_LEGACY_ERROR_TEMP_3219",
parameters = Map(
"other" -> "0.1",
"otherClass" -> "java.lang.Double",
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index daa8d12613f2c..a09dadbcd4816 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -494,7 +494,7 @@ class ScalaReflectionSuite extends SparkFunSuite {
exception = intercept[SparkUnsupportedOperationException] {
schemaFor[TraitProductWithoutCompanion]
},
- errorClass = "_LEGACY_ERROR_TEMP_2144",
+ condition = "_LEGACY_ERROR_TEMP_2144",
parameters = Map("tpe" -> "org.apache.spark.sql.catalyst.TraitProductWithoutCompanion"))
}
@@ -503,7 +503,7 @@ class ScalaReflectionSuite extends SparkFunSuite {
exception = intercept[SparkUnsupportedOperationException] {
schemaFor[TraitProductWithNoConstructorCompanion]
},
- errorClass = "_LEGACY_ERROR_TEMP_2144",
+ condition = "_LEGACY_ERROR_TEMP_2144",
parameters = Map("tpe" ->
"org.apache.spark.sql.catalyst.TraitProductWithNoConstructorCompanion"))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ShuffleSpecSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ShuffleSpecSuite.scala
index 8a71496607466..fc5d39fd9c2bb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ShuffleSpecSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ShuffleSpecSuite.scala
@@ -474,7 +474,7 @@ class ShuffleSpecSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkUnsupportedOperationException] {
RangeShuffleSpec(10, distribution).createPartitioning(distribution.clustering)
},
- errorClass = "UNSUPPORTED_CALL.WITHOUT_SUGGESTION",
+ condition = "UNSUPPORTED_CALL.WITHOUT_SUGGESTION",
parameters = Map(
"methodName" -> "createPartitioning$",
"className" -> "org.apache.spark.sql.catalyst.plans.physical.ShuffleSpec"))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index cfe08e1895363..70cc50a23a6a2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -110,14 +110,15 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
}
- def errorClassTest(
+ def errorConditionTest(
name: String,
plan: LogicalPlan,
- errorClass: String,
+ condition: String,
messageParameters: Map[String, String],
caseSensitive: Boolean = true): Unit = {
test(name) {
- assertAnalysisErrorClass(plan, errorClass, messageParameters, caseSensitive = caseSensitive)
+ assertAnalysisErrorCondition(
+ plan, condition, messageParameters, caseSensitive = caseSensitive)
}
}
@@ -134,10 +135,10 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
testRelation.select(ScalarSubquery(LocalRelation()).as("a")),
"Scalar subquery must return only one column, but got 0" :: Nil)
- errorClassTest(
+ errorConditionTest(
"single invalid type, single arg",
testRelation.select(TestFunction(dateLit :: Nil, IntegerType :: Nil).as("a")),
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
messageParameters = Map(
"sqlExpr" -> "\"testfunction(NULL)\"",
"paramIndex" -> "first",
@@ -145,11 +146,11 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"inputType" -> "\"DATE\"",
"requiredType" -> "\"INT\""))
- errorClassTest(
+ errorConditionTest(
"single invalid type, second arg",
testRelation.select(
TestFunction(dateLit :: dateLit :: Nil, DateType :: IntegerType :: Nil).as("a")),
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
messageParameters = Map(
"sqlExpr" -> "\"testfunction(NULL, NULL)\"",
"paramIndex" -> "second",
@@ -157,11 +158,11 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"inputType" -> "\"DATE\"",
"requiredType" -> "\"INT\""))
- errorClassTest(
+ errorConditionTest(
"multiple invalid type",
testRelation.select(
TestFunction(dateLit :: dateLit :: Nil, IntegerType :: IntegerType :: Nil).as("a")),
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
messageParameters = Map(
"sqlExpr" -> "\"testfunction(NULL, NULL)\"",
"paramIndex" -> "first",
@@ -169,17 +170,17 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"inputType" -> "\"DATE\"",
"requiredType" -> "\"INT\""))
- errorClassTest(
+ errorConditionTest(
"SPARK-44477: type check failure",
testRelation.select(
TestFunctionWithTypeCheckFailure(dateLit :: Nil, BinaryType :: Nil).as("a")),
- errorClass = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
+ condition = "DATATYPE_MISMATCH.TYPE_CHECK_FAILURE_WITH_HINT",
messageParameters = Map(
"sqlExpr" -> "\"testfunctionwithtypecheckfailure(NULL)\"",
"msg" -> "Expression must be a binary",
"hint" -> ""))
- errorClassTest(
+ errorConditionTest(
"invalid window function",
testRelation2.select(
WindowExpression(
@@ -188,10 +189,10 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
UnresolvedAttribute("a") :: Nil,
SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
UnspecifiedFrame)).as("window")),
- errorClass = "UNSUPPORTED_EXPR_FOR_WINDOW",
+ condition = "UNSUPPORTED_EXPR_FOR_WINDOW",
messageParameters = Map("sqlExpr" -> "\"0\""))
- errorClassTest(
+ errorConditionTest(
"distinct aggregate function in window",
testRelation2.select(
WindowExpression(
@@ -200,7 +201,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
UnresolvedAttribute("a") :: Nil,
SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
UnspecifiedFrame)).as("window")),
- errorClass = "DISTINCT_WINDOW_FUNCTION_UNSUPPORTED",
+ condition = "DISTINCT_WINDOW_FUNCTION_UNSUPPORTED",
messageParameters = Map("windowExpr" ->
s"""
|"count(DISTINCT b) OVER (PARTITION BY a ORDER BY b ASC NULLS FIRST
@@ -221,9 +222,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
test("distinct function") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CatalystSqlParser.parsePlan("SELECT hex(DISTINCT a) FROM TaBlE"),
- expectedErrorClass = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
+ expectedErrorCondition = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
expectedMessageParameters = Map(
"prettyName" -> toSQLId("hex"),
"syntax" -> toSQLStmt("DISTINCT")),
@@ -231,9 +232,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
test("non aggregate function with filter predicate") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CatalystSqlParser.parsePlan("SELECT hex(a) FILTER (WHERE c = 1) FROM TaBlE2"),
- expectedErrorClass = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
+ expectedErrorCondition = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
expectedMessageParameters = Map(
"prettyName" -> toSQLId("hex"),
"syntax" -> toSQLStmt("FILTER CLAUSE")),
@@ -241,9 +242,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
test("distinct window function") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CatalystSqlParser.parsePlan("SELECT percent_rank(DISTINCT a) OVER () FROM TaBlE"),
- expectedErrorClass = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
+ expectedErrorCondition = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
expectedMessageParameters = Map(
"prettyName" -> toSQLId("percent_rank"),
"syntax" -> toSQLStmt("DISTINCT")),
@@ -251,10 +252,10 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
test("window function with filter predicate") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CatalystSqlParser.parsePlan(
"SELECT percent_rank(a) FILTER (WHERE c > 1) OVER () FROM TaBlE2"),
- expectedErrorClass = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
+ expectedErrorCondition = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
expectedMessageParameters = Map(
"prettyName" -> toSQLId("percent_rank"),
"syntax" -> toSQLStmt("FILTER CLAUSE")),
@@ -262,7 +263,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
test("window specification error") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = CatalystSqlParser.parsePlan(
"""
|WITH sample_data AS (
@@ -274,17 +275,17 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
|FROM sample_data
|GROUP BY a, b;
|""".stripMargin),
- expectedErrorClass = "MISSING_WINDOW_SPECIFICATION",
+ expectedErrorCondition = "MISSING_WINDOW_SPECIFICATION",
expectedMessageParameters = Map(
"windowName" -> "b",
"docroot" -> SPARK_DOC_ROOT))
}
test("higher order function with filter predicate") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CatalystSqlParser.parsePlan("SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x) " +
"FILTER (WHERE c > 1)"),
- expectedErrorClass = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
+ expectedErrorCondition = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
expectedMessageParameters = Map(
"prettyName" -> toSQLId("aggregate"),
"syntax" -> toSQLStmt("FILTER CLAUSE")),
@@ -293,9 +294,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
test("function don't support ignore nulls") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CatalystSqlParser.parsePlan("SELECT hex(a) IGNORE NULLS FROM TaBlE2"),
- expectedErrorClass = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
+ expectedErrorCondition = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
expectedMessageParameters = Map(
"prettyName" -> toSQLId("hex"),
"syntax" -> toSQLStmt("IGNORE NULLS")),
@@ -303,9 +304,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
test("some window function don't support ignore nulls") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CatalystSqlParser.parsePlan("SELECT percent_rank(a) IGNORE NULLS FROM TaBlE2"),
- expectedErrorClass = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
+ expectedErrorCondition = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
expectedMessageParameters = Map(
"prettyName" -> toSQLId("percent_rank"),
"syntax" -> toSQLStmt("IGNORE NULLS")),
@@ -313,9 +314,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
test("aggregate function don't support ignore nulls") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CatalystSqlParser.parsePlan("SELECT count(a) IGNORE NULLS FROM TaBlE2"),
- expectedErrorClass = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
+ expectedErrorCondition = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
expectedMessageParameters = Map(
"prettyName" -> toSQLId("count"),
"syntax" -> toSQLStmt("IGNORE NULLS")),
@@ -323,10 +324,10 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
test("higher order function don't support ignore nulls") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CatalystSqlParser.parsePlan(
"SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x) IGNORE NULLS"),
- expectedErrorClass = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
+ expectedErrorCondition = "INVALID_SQL_SYNTAX.FUNCTION_WITH_UNSUPPORTED_SYNTAX",
expectedMessageParameters = Map(
"prettyName" -> toSQLId("aggregate"),
"syntax" -> toSQLStmt("IGNORE NULLS")),
@@ -334,11 +335,11 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x) IGNORE NULLS", 7, 68)))
}
- errorClassTest(
+ errorConditionTest(
name = "nested aggregate functions",
testRelation.groupBy($"a")(
Max(Count(Literal(1)).toAggregateExpression()).toAggregateExpression()),
- errorClass = "NESTED_AGGREGATE_FUNCTION",
+ condition = "NESTED_AGGREGATE_FUNCTION",
messageParameters = Map.empty
)
@@ -353,7 +354,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
SpecifiedWindowFrame(RangeFrame, Literal(1), Literal(2)))).as("window")),
"Cannot specify window frame for lead function" :: Nil)
- errorClassTest(
+ errorConditionTest(
"the offset of nth_value window function is negative or zero",
testRelation2.select(
WindowExpression(
@@ -362,14 +363,14 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
UnresolvedAttribute("a") :: Nil,
SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
SpecifiedWindowFrame(RowFrame, Literal(0), Literal(0)))).as("window")),
- errorClass = "DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE",
+ condition = "DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE",
messageParameters = Map(
"sqlExpr" -> "\"nth_value(b, 0)\"",
"exprName" -> "offset",
"valueRange" -> "(0, 9223372036854775807]",
"currentValue" -> "0L"))
- errorClassTest(
+ errorConditionTest(
"the offset of nth_value window function is not int literal",
testRelation2.select(
WindowExpression(
@@ -378,7 +379,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
UnresolvedAttribute("a") :: Nil,
SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
SpecifiedWindowFrame(RowFrame, Literal(0), Literal(0)))).as("window")),
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
messageParameters = Map(
"sqlExpr" -> "\"nth_value(b, true)\"",
"paramIndex" -> "second",
@@ -386,7 +387,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"inputType" -> "\"BOOLEAN\"",
"requiredType" -> "\"INT\""))
- errorClassTest(
+ errorConditionTest(
"the buckets of ntile window function is not foldable",
testRelation2.select(
WindowExpression(
@@ -395,7 +396,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
UnresolvedAttribute("a") :: Nil,
SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
UnspecifiedFrame)).as("window")),
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
messageParameters = Map(
"sqlExpr" -> "\"ntile(99.9)\"",
"paramIndex" -> "first",
@@ -404,7 +405,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"requiredType" -> "\"INT\""))
- errorClassTest(
+ errorConditionTest(
"the buckets of ntile window function is not int literal",
testRelation2.select(
WindowExpression(
@@ -413,20 +414,20 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
UnresolvedAttribute("a") :: Nil,
SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
UnspecifiedFrame)).as("window")),
- errorClass = "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
+ condition = "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
messageParameters = Map(
"sqlExpr" -> "\"ntile(b)\"",
"inputName" -> "`buckets`",
"inputExpr" -> "\"b\"",
"inputType" -> "\"INT\""))
- errorClassTest(
+ errorConditionTest(
"unresolved attributes",
testRelation.select($"abcd"),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`abcd`", "proposal" -> "`a`"))
- errorClassTest(
+ errorConditionTest(
"unresolved attributes with a generated name",
testRelation2.groupBy($"a")(max($"b"))
.where(sum($"b") > 0)
@@ -434,41 +435,41 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`havingCondition`", "proposal" -> "`max(b)`"))
- errorClassTest(
+ errorConditionTest(
"unresolved star expansion in max",
testRelation2.groupBy($"a")(sum(UnresolvedStar(None))),
- errorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ condition = "INVALID_USAGE_OF_STAR_OR_REGEX",
messageParameters = Map("elem" -> "'*'", "prettyName" -> "expression `sum`")
)
- errorClassTest(
+ errorConditionTest(
"sorting by unsupported column types",
mapRelation.orderBy($"map".asc),
- errorClass = "DATATYPE_MISMATCH.INVALID_ORDERING_TYPE",
+ condition = "DATATYPE_MISMATCH.INVALID_ORDERING_TYPE",
messageParameters = Map(
"sqlExpr" -> "\"map ASC NULLS FIRST\"",
"functionName" -> "`sortorder`",
"dataType" -> "\"MAP\""))
- errorClassTest(
+ errorConditionTest(
"sorting by attributes are not from grouping expressions",
testRelation2.groupBy($"a", $"c")($"a", $"c", count($"a").as("a3")).orderBy($"b".asc),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`b`", "proposal" -> "`a`, `c`, `a3`"))
- errorClassTest(
+ errorConditionTest(
"non-boolean filters",
testRelation.where(Literal(1)),
- errorClass = "DATATYPE_MISMATCH.FILTER_NOT_BOOLEAN",
+ condition = "DATATYPE_MISMATCH.FILTER_NOT_BOOLEAN",
messageParameters = Map("sqlExpr" -> "\"1\"", "filter" -> "\"1\"", "type" -> "\"INT\""))
- errorClassTest(
+ errorConditionTest(
"non-boolean join conditions",
testRelation.join(testRelation, condition = Some(Literal(1))),
- errorClass = "JOIN_CONDITION_IS_NOT_BOOLEAN_TYPE",
+ condition = "JOIN_CONDITION_IS_NOT_BOOLEAN_TYPE",
messageParameters = Map("joinCondition" -> "\"1\"", "conditionType" -> "\"INT\""))
- errorClassTest(
+ errorConditionTest(
"missing group by",
testRelation2.groupBy($"a")($"b"),
"MISSING_AGGREGATION",
@@ -477,27 +478,27 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"expressionAnyValue" -> "\"any_value(b)\"")
)
- errorClassTest(
+ errorConditionTest(
"ambiguous field",
nestedRelation.select($"top.duplicateField"),
- errorClass = "AMBIGUOUS_REFERENCE_TO_FIELDS",
+ condition = "AMBIGUOUS_REFERENCE_TO_FIELDS",
messageParameters = Map(
"field" -> "`duplicateField`",
"count" -> "2"),
caseSensitive = false
)
- errorClassTest(
+ errorConditionTest(
"ambiguous field due to case insensitivity",
nestedRelation.select($"top.differentCase"),
- errorClass = "AMBIGUOUS_REFERENCE_TO_FIELDS",
+ condition = "AMBIGUOUS_REFERENCE_TO_FIELDS",
messageParameters = Map(
"field" -> "`differentCase`",
"count" -> "2"),
caseSensitive = false
)
- errorClassTest(
+ errorConditionTest(
"missing field",
nestedRelation2.select($"top.c"),
"FIELD_NOT_FOUND",
@@ -509,7 +510,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
val analyzer = getAnalyzer
analyzer.checkAnalysis(analyzer.execute(UnresolvedTestPlan()))
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> "Found the unresolved operator: 'UnresolvedTestPlan"))
errorTest(
@@ -560,14 +561,14 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
testRelation3.except(testRelation4, isAll = false),
"except" :: "compatible column types" :: "map" :: "decimal" :: Nil)
- errorClassTest(
+ errorConditionTest(
"SPARK-9955: correct error message for aggregate",
// When parse SQL string, we will wrap aggregate expressions with UnresolvedAlias.
testRelation2.where($"bad_column" > 1).groupBy($"a")(UnresolvedAlias(max($"b"))),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`bad_column`", "proposal" -> "`a`, `c`, `d`, `b`, `e`"))
- errorClassTest(
+ errorConditionTest(
"slide duration greater than window in time window",
testRelation2.select(
TimeWindow(Literal("2016-01-01 01:01:01"), "1 second", "2 second", "0 second").as("window")),
@@ -582,7 +583,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"start time greater than slide duration in time window",
testRelation.select(
TimeWindow(Literal("2016-01-01 01:01:01"), "1 second", "1 second", "1 minute").as("window")),
@@ -597,7 +598,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"start time equal to slide duration in time window",
testRelation.select(
TimeWindow(Literal("2016-01-01 01:01:01"), "1 second", "1 second", "1 second").as("window")),
@@ -612,7 +613,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"SPARK-21590: absolute value of start time greater than slide duration in time window",
testRelation.select(
TimeWindow(Literal("2016-01-01 01:01:01"), "1 second", "1 second", "-1 minute").as("window")),
@@ -627,7 +628,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"SPARK-21590: absolute value of start time equal to slide duration in time window",
testRelation.select(
TimeWindow(Literal("2016-01-01 01:01:01"), "1 second", "1 second", "-1 second").as("window")),
@@ -642,7 +643,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"negative window duration in time window",
testRelation.select(
TimeWindow(Literal("2016-01-01 01:01:01"), "-1 second", "1 second", "0 second").as("window")),
@@ -655,7 +656,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"zero window duration in time window",
testRelation.select(
TimeWindow(Literal("2016-01-01 01:01:01"), "0 second", "1 second", "0 second").as("window")),
@@ -668,7 +669,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"negative slide duration in time window",
testRelation.select(
TimeWindow(Literal("2016-01-01 01:01:01"), "1 second", "-1 second", "0 second").as("window")),
@@ -681,7 +682,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"zero slide duration in time window",
testRelation.select(
TimeWindow(Literal("2016-01-01 01:01:01"), "1 second", "0 second", "0 second").as("window")),
@@ -733,7 +734,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"The generator is not supported: outside the SELECT clause, found: Sort" :: Nil
)
- errorClassTest(
+ errorConditionTest(
"an evaluated limit class must not be string",
testRelation.limit(Literal(UTF8String.fromString("abc"), StringType)),
"INVALID_LIMIT_LIKE_EXPRESSION.DATA_TYPE",
@@ -744,7 +745,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"an evaluated limit class must not be long",
testRelation.limit(Literal(10L, LongType)),
"INVALID_LIMIT_LIKE_EXPRESSION.DATA_TYPE",
@@ -755,7 +756,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"an evaluated limit class must not be null",
testRelation.limit(Literal(null, IntegerType)),
"INVALID_LIMIT_LIKE_EXPRESSION.IS_NULL",
@@ -765,7 +766,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"num_rows in limit clause must be equal to or greater than 0",
listRelation.limit(-1),
"INVALID_LIMIT_LIKE_EXPRESSION.IS_NEGATIVE",
@@ -776,7 +777,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"an evaluated offset class must not be string",
testRelation.offset(Literal(UTF8String.fromString("abc"), StringType)),
"INVALID_LIMIT_LIKE_EXPRESSION.DATA_TYPE",
@@ -787,7 +788,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"an evaluated offset class must not be long",
testRelation.offset(Literal(10L, LongType)),
"INVALID_LIMIT_LIKE_EXPRESSION.DATA_TYPE",
@@ -798,7 +799,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"an evaluated offset class must not be null",
testRelation.offset(Literal(null, IntegerType)),
"INVALID_LIMIT_LIKE_EXPRESSION.IS_NULL",
@@ -808,7 +809,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"num_rows in offset clause must be equal to or greater than 0",
testRelation.offset(-1),
"INVALID_LIMIT_LIKE_EXPRESSION.IS_NEGATIVE",
@@ -819,7 +820,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
)
)
- errorClassTest(
+ errorConditionTest(
"the sum of num_rows in limit clause and num_rows in offset clause less than Int.MaxValue",
testRelation.offset(Literal(2000000000, IntegerType)).limit(Literal(1000000000, IntegerType)),
"SUM_OF_LIMIT_AND_OFFSET_EXCEEDS_MAX_INT",
@@ -833,14 +834,14 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
""""explode(array(min(a)))", "explode(array(max(a)))"""" :: Nil
)
- errorClassTest(
+ errorConditionTest(
"EXEC IMMEDIATE - nested execute immediate not allowed",
CatalystSqlParser.parsePlan("EXECUTE IMMEDIATE 'EXECUTE IMMEDIATE \\\'SELECT 42\\\''"),
"NESTED_EXECUTE_IMMEDIATE",
Map(
"sqlString" -> "EXECUTE IMMEDIATE 'SELECT 42'"))
- errorClassTest(
+ errorConditionTest(
"EXEC IMMEDIATE - both positional and named used",
CatalystSqlParser.parsePlan("EXECUTE IMMEDIATE 'SELECT 42 where ? = :first'" +
" USING 1, 2 as first"),
@@ -853,9 +854,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
scala.util.Right(UnresolvedAttribute("testVarA")),
Seq(UnresolvedAttribute("testVarA")))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = execImmediatePlan,
- expectedErrorClass = "INVALID_VARIABLE_TYPE_FOR_QUERY_EXECUTE_IMMEDIATE",
+ expectedErrorCondition = "INVALID_VARIABLE_TYPE_FOR_QUERY_EXECUTE_IMMEDIATE",
expectedMessageParameters = Map(
"varType" -> "\"INT\""
))
@@ -867,9 +868,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
scala.util.Right(UnresolvedAttribute("testVarNull")),
Seq(UnresolvedAttribute("testVarNull")))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = execImmediatePlan,
- expectedErrorClass = "NULL_QUERY_STRING_EXECUTE_IMMEDIATE",
+ expectedErrorCondition = "NULL_QUERY_STRING_EXECUTE_IMMEDIATE",
expectedMessageParameters = Map("varName" -> "`testVarNull`"))
}
@@ -880,9 +881,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
scala.util.Left("SELECT ?"),
Seq.empty)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = execImmediatePlan,
- expectedErrorClass = "UNSUPPORTED_EXPR_FOR_PARAMETER",
+ expectedErrorCondition = "UNSUPPORTED_EXPR_FOR_PARAMETER",
expectedMessageParameters = Map(
"invalidExprSql" -> "\"nanvl(1, 1)\""
))
@@ -894,9 +895,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
scala.util.Left("SELECT :first"),
Seq.empty)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = execImmediateSetVariablePlan,
- expectedErrorClass = "ALL_PARAMETERS_MUST_BE_NAMED",
+ expectedErrorCondition = "ALL_PARAMETERS_MUST_BE_NAMED",
expectedMessageParameters = Map(
"exprs" -> "\"2\", \"3\""
))
@@ -908,9 +909,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
scala.util.Left("SET VAR testVarA = 1"),
Seq(UnresolvedAttribute("testVarA")))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = execImmediateSetVariablePlan,
- expectedErrorClass = "INVALID_STATEMENT_FOR_EXECUTE_INTO",
+ expectedErrorCondition = "INVALID_STATEMENT_FOR_EXECUTE_INTO",
expectedMessageParameters = Map(
"sqlString" -> "SET VAR TESTVARA = 1"
))
@@ -931,9 +932,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
assert(plan.resolved)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = plan,
- expectedErrorClass = "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
+ expectedErrorCondition = "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
expectedMessageParameters = Map(
"missingAttributes" -> "\"a\", \"c\"",
"input" -> "\"a\"",
@@ -949,7 +950,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
exception = intercept[SparkException] {
SimpleAnalyzer.checkAnalysis(join)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" ->
"""
|Failure when resolving conflicting references in Join:
@@ -966,7 +967,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
exception = intercept[SparkException] {
SimpleAnalyzer.checkAnalysis(intersect)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" ->
"""
|Failure when resolving conflicting references in Intersect All:
@@ -983,7 +984,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
exception = intercept[SparkException] {
SimpleAnalyzer.checkAnalysis(except)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" ->
"""
|Failure when resolving conflicting references in Except All:
@@ -1003,7 +1004,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
exception = intercept[SparkException] {
SimpleAnalyzer.checkAnalysis(asOfJoin)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" ->
"""
|Failure when resolving conflicting references in AsOfJoin:
@@ -1059,9 +1060,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
AttributeReference("a", IntegerType)(exprId = ExprId(2)),
AttributeReference("b", IntegerType)(exprId = ExprId(1))))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = plan,
- expectedErrorClass = "NESTED_AGGREGATE_FUNCTION",
+ expectedErrorCondition = "NESTED_AGGREGATE_FUNCTION",
expectedMessageParameters = Map.empty
)
}
@@ -1082,9 +1083,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
joinType = Cross,
condition = Some($"b" === $"d"))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = plan2,
- expectedErrorClass = "DATATYPE_MISMATCH.INVALID_ORDERING_TYPE",
+ expectedErrorCondition = "DATATYPE_MISMATCH.INVALID_ORDERING_TYPE",
expectedMessageParameters = Map(
"functionName" -> "`=`",
"dataType" -> "\"MAP\"",
@@ -1145,8 +1146,8 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
val a = AttributeReference("a", IntegerType)()
val b = AttributeReference("b", IntegerType)()
val plan = Filter($"a" === UnresolvedFunction("max", Seq(b), true), LocalRelation(a, b))
- assertAnalysisErrorClass(plan,
- expectedErrorClass = "INVALID_WHERE_CONDITION",
+ assertAnalysisErrorCondition(plan,
+ expectedErrorCondition = "INVALID_WHERE_CONDITION",
expectedMessageParameters = Map(
"condition" -> "\"(a = max(DISTINCT b))\"",
"expressionList" -> "max(DISTINCT b)"))
@@ -1160,8 +1161,8 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
Project(
Alias(Literal(1), "x")() :: Nil,
UnresolvedRelation(TableIdentifier("t", Option("nonexist")))))))
- assertAnalysisErrorClass(plan,
- expectedErrorClass = "TABLE_OR_VIEW_NOT_FOUND",
+ assertAnalysisErrorCondition(plan,
+ expectedErrorCondition = "TABLE_OR_VIEW_NOT_FOUND",
Map("relationName" -> "`nonexist`.`t`"))
}
@@ -1170,8 +1171,8 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
(Randn("a".attr), "\"randn(a)\"")).foreach {
case (r, expectedArg) =>
val plan = Project(Seq(r.as("r")), testRelation)
- assertAnalysisErrorClass(plan,
- expectedErrorClass = "SEED_EXPRESSION_IS_UNFOLDABLE",
+ assertAnalysisErrorCondition(plan,
+ expectedErrorCondition = "SEED_EXPRESSION_IS_UNFOLDABLE",
expectedMessageParameters = Map(
"seedExpr" -> "\"a\"",
"exprWithSeed" -> expectedArg),
@@ -1184,8 +1185,8 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
Randn("a") -> ("\"randn(a)\"", "\"a\"", "\"STRING\"")
).foreach { case (r, (sqlExpr, inputSql, inputType)) =>
val plan = Project(Seq(r.as("r")), testRelation)
- assertAnalysisErrorClass(plan,
- expectedErrorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ assertAnalysisErrorCondition(plan,
+ expectedErrorCondition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
expectedMessageParameters = Map(
"sqlExpr" -> sqlExpr,
"paramIndex" -> "first",
@@ -1208,9 +1209,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
t.as("t2")))
) :: Nil,
sum($"c2").as("sum") :: Nil, t.as("t1"))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
plan,
- expectedErrorClass =
+ expectedErrorCondition =
"UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.MUST_AGGREGATE_CORRELATED_SCALAR_SUBQUERY",
expectedMessageParameters = Map.empty)
}
@@ -1226,37 +1227,37 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
Filter($"t1.c1" === $"t2.c1",
t.as("t2")))
).as("sub") :: Nil, t.as("t1"))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
plan,
- expectedErrorClass =
+ expectedErrorCondition =
"SCALAR_SUBQUERY_IS_IN_GROUP_BY_OR_AGGREGATE_FUNCTION",
expectedMessageParameters = Map("sqlExpr" -> "\"scalarsubquery(c1)\""))
}
- errorClassTest(
+ errorConditionTest(
"SPARK-34920: error code to error message",
testRelation2.where($"bad_column" > 1).groupBy($"a")(UnresolvedAlias(max($"b"))),
- errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
+ condition = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
messageParameters = Map(
"objectName" -> "`bad_column`",
"proposal" -> "`a`, `c`, `d`, `b`, `e`"))
- errorClassTest(
+ errorConditionTest(
"SPARK-39783: backticks in error message for candidate column with dots",
// This selects a column that does not exist,
// the error message suggest the existing column with correct backticks
testRelation6.select($"`the`.`id`"),
- errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
+ condition = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
messageParameters = Map(
"objectName" -> "`the`.`id`",
"proposal" -> "`the.id`"))
- errorClassTest(
+ errorConditionTest(
"SPARK-39783: backticks in error message for candidate struct column",
// This selects a column that does not exist,
// the error message suggest the existing column with correct backticks
nestedRelation2.select($"`top.aField`"),
- errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
+ condition = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
messageParameters = Map(
"objectName" -> "`top.aField`",
"proposal" -> "`top`"))
@@ -1272,7 +1273,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
exception = intercept[SparkException] {
SimpleAnalyzer.checkAnalysis(plan)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> "Hint not found: `some_random_hint_that_does_not_exist`"))
// UnresolvedHint be removed by batch `Remove Unresolved Hints`
@@ -1291,9 +1292,9 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
"Scalar subquery must return only one column, but got 2" :: Nil)
// t2.* cannot be resolved and the error should be the initial analysis exception.
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Project(ScalarSubquery(t0.select(star("t2"))).as("sub") :: Nil, t1),
- expectedErrorClass = "CANNOT_RESOLVE_STAR_EXPAND",
+ expectedErrorCondition = "CANNOT_RESOLVE_STAR_EXPAND",
expectedMessageParameters = Map("targetString" -> "`t2`", "columns" -> "")
)
}
@@ -1306,70 +1307,70 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
val t2 = LocalRelation(b, c).as("t2")
// SELECT * FROM t1 WHERE a = (SELECT sum(c) FROM t2 WHERE t1.* = t2.b)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Filter(EqualTo(a, ScalarSubquery(t2.select(sum(c)).where(star("t1") === b))), t1),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map("elem" -> "'*'", "prettyName" -> "expression `equalto`")
)
// SELECT * FROM t1 JOIN t2 ON (EXISTS (SELECT 1 FROM t2 WHERE t1.* = b))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
t1.join(t2, condition = Some(Exists(t2.select(1).where(star("t1") === b)))),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map("elem" -> "'*'", "prettyName" -> "expression `equalto`")
)
}
test("SPARK-36488: Regular expression expansion should fail with a meaningful message") {
withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "true") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
testRelation.select(Divide(UnresolvedRegex(".?", None, false), "a")),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map(
"elem" -> "regular expression '.?'",
"prettyName" -> "expression `divide`")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
testRelation.select(
Divide(UnresolvedRegex(".?", None, false), UnresolvedRegex(".*", None, false))),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map(
"elem" -> "regular expressions '.?', '.*'",
"prettyName" -> "expression `divide`")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
testRelation.select(
Divide(UnresolvedRegex(".?", None, false), UnresolvedRegex(".?", None, false))),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map(
"elem" -> "regular expression '.?'",
"prettyName" -> "expression `divide`")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
testRelation.select(Divide(UnresolvedStar(None), "a")),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map(
"elem" -> "'*'",
"prettyName" -> "expression `divide`")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
testRelation.select(Divide(UnresolvedStar(None), UnresolvedStar(None))),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map(
"elem" -> "'*'",
"prettyName" -> "expression `divide`")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
testRelation.select(Divide(UnresolvedStar(None), UnresolvedRegex(".?", None, false))),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map(
"elem" -> "'*' and regular expression '.?'",
"prettyName" -> "expression `divide`")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
testRelation.select(Least(Seq(UnresolvedStar(None),
UnresolvedRegex(".*", None, false), UnresolvedRegex(".?", None, false)))),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map(
"elem" -> "'*' and regular expressions '.*', '.?'",
"prettyName" -> "expression `least`")
@@ -1377,7 +1378,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
}
}
- errorClassTest(
+ errorConditionTest(
"SPARK-47572: Enforce Window partitionSpec is orderable",
testRelation2.select(
WindowExpression(
@@ -1386,7 +1387,7 @@ class AnalysisErrorSuite extends AnalysisTest with DataTypeErrorsBase {
CreateMap(Literal("key") :: UnresolvedAttribute("a") :: Nil) :: Nil,
SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
UnspecifiedFrame)).as("window")),
- errorClass = "EXPRESSION_TYPE_IS_NOT_ORDERABLE",
+ condition = "EXPRESSION_TYPE_IS_NOT_ORDERABLE",
messageParameters = Map(
"expr" -> "\"_w0\"",
"exprType" -> "\"MAP\""))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExceptionPositionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExceptionPositionSuite.scala
index be256adbd8929..55f59f7a22574 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExceptionPositionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExceptionPositionSuite.scala
@@ -48,7 +48,7 @@ class AnalysisExceptionPositionSuite extends AnalysisTest {
verifyTableOrViewPosition("REFRESH TABLE unknown", "unknown")
verifyTableOrViewPosition("SHOW COLUMNS FROM unknown", "unknown")
// Special case where namespace is prepended to the table name.
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsePlan("SHOW COLUMNS FROM unknown IN db"),
"TABLE_OR_VIEW_NOT_FOUND",
Map("relationName" -> "`db`.`unknown`"),
@@ -94,7 +94,7 @@ class AnalysisExceptionPositionSuite extends AnalysisTest {
private def verifyPosition(sql: String, table: String): Unit = {
val startPos = sql.indexOf(table)
assert(startPos != -1)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsePlan(sql),
"TABLE_OR_VIEW_NOT_FOUND",
Map("relationName" -> s"`$table`"),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 1816c620414c9..e23a753dafe8c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -74,7 +74,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
None,
CaseInsensitiveStringMap.empty()).analyze
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" ->
"Logical plan should not have output of char/varchar type.*\n"),
matchPVals = true)
@@ -112,7 +112,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
SubqueryAlias("TbL", UnresolvedRelation(TableIdentifier("TaBlE")))),
Project(testRelation.output, testRelation))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Project(Seq(UnresolvedAttribute("tBl.a")),
SubqueryAlias("TbL", UnresolvedRelation(TableIdentifier("TaBlE")))),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
@@ -359,7 +359,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
val plan = Project(Alias(In(Literal(null), Seq(Literal(true), Literal(1))), "a")() :: Nil,
LocalRelation()
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
plan,
"DATATYPE_MISMATCH.DATA_DIFF_TYPES",
Map(
@@ -555,7 +555,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
assertAnalysisSuccess(rangeWithAliases(3 :: Nil, "a" :: Nil))
assertAnalysisSuccess(rangeWithAliases(1 :: 4 :: Nil, "b" :: Nil))
assertAnalysisSuccess(rangeWithAliases(2 :: 6 :: 2 :: Nil, "c" :: Nil))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
rangeWithAliases(3 :: Nil, "a" :: "b" :: Nil),
"NUM_TABLE_VALUE_ALIASES_MISMATCH",
Map("funcName" -> "`range`", "aliasesNum" -> "2", "outColsNum" -> "1"))
@@ -569,12 +569,12 @@ class AnalysisSuite extends AnalysisTest with Matchers {
).select(star())
}
assertAnalysisSuccess(tableColumnsWithAliases("col1" :: "col2" :: "col3" :: "col4" :: Nil))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
tableColumnsWithAliases("col1" :: Nil),
"ASSIGNMENT_ARITY_MISMATCH",
Map("numExpr" -> "1", "numTarget" -> "4")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
tableColumnsWithAliases("col1" :: "col2" :: "col3" :: "col4" :: "col5" :: Nil),
"ASSIGNMENT_ARITY_MISMATCH",
Map("numExpr" -> "5", "numTarget" -> "4")
@@ -591,12 +591,12 @@ class AnalysisSuite extends AnalysisTest with Matchers {
).select(star())
}
assertAnalysisSuccess(tableColumnsWithAliases("col1" :: "col2" :: "col3" :: "col4" :: Nil))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
tableColumnsWithAliases("col1" :: Nil),
"ASSIGNMENT_ARITY_MISMATCH",
Map("numExpr" -> "1", "numTarget" -> "4")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
tableColumnsWithAliases("col1" :: "col2" :: "col3" :: "col4" :: "col5" :: Nil),
"ASSIGNMENT_ARITY_MISMATCH",
Map("numExpr" -> "5", "numTarget" -> "4")
@@ -615,12 +615,12 @@ class AnalysisSuite extends AnalysisTest with Matchers {
).select(star())
}
assertAnalysisSuccess(joinRelationWithAliases("col1" :: "col2" :: "col3" :: "col4" :: Nil))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
joinRelationWithAliases("col1" :: Nil),
"ASSIGNMENT_ARITY_MISMATCH",
Map("numExpr" -> "1", "numTarget" -> "4")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
joinRelationWithAliases("col1" :: "col2" :: "col3" :: "col4" :: "col5" :: Nil),
"ASSIGNMENT_ARITY_MISMATCH",
Map("numExpr" -> "5", "numTarget" -> "4")
@@ -755,7 +755,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
test("SPARK-34741: Avoid ambiguous reference in MergeIntoTable") {
val cond = $"a" > 1
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
MergeIntoTable(
testRelation,
testRelation,
@@ -794,7 +794,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
}
test("CTE with non-existing column alias") {
- assertAnalysisErrorClass(parsePlan("WITH t(x) AS (SELECT 1) SELECT * FROM t WHERE y = 1"),
+ assertAnalysisErrorCondition(parsePlan("WITH t(x) AS (SELECT 1) SELECT * FROM t WHERE y = 1"),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`y`", "proposal" -> "`x`"),
Array(ExpectedContext("y", 46, 46))
@@ -802,7 +802,8 @@ class AnalysisSuite extends AnalysisTest with Matchers {
}
test("CTE with non-matching column alias") {
- assertAnalysisErrorClass(parsePlan("WITH t(x, y) AS (SELECT 1) SELECT * FROM t WHERE x = 1"),
+ assertAnalysisErrorCondition(
+ parsePlan("WITH t(x, y) AS (SELECT 1) SELECT * FROM t WHERE x = 1"),
"ASSIGNMENT_ARITY_MISMATCH",
Map("numExpr" -> "2", "numTarget" -> "1"),
Array(ExpectedContext("t(x, y) AS (SELECT 1)", 5, 25))
@@ -810,7 +811,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
}
test("SPARK-28251: Insert into non-existing table error message is user friendly") {
- assertAnalysisErrorClass(parsePlan("INSERT INTO test VALUES (1)"),
+ assertAnalysisErrorCondition(parsePlan("INSERT INTO test VALUES (1)"),
"TABLE_OR_VIEW_NOT_FOUND", Map("relationName" -> "`test`"),
Array(ExpectedContext("test", 12, 15)))
}
@@ -826,9 +827,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
// Bad name
assert(!CollectMetrics("", sum :: Nil, testRelation, 0).resolved)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CollectMetrics("", sum :: Nil, testRelation, 0),
- expectedErrorClass = "INVALID_OBSERVED_METRICS.MISSING_NAME",
+ expectedErrorCondition = "INVALID_OBSERVED_METRICS.MISSING_NAME",
expectedMessageParameters = Map(
"operator" ->
"'CollectMetrics , [sum(a#x) AS sum#xL], 0\n+- LocalRelation , [a#x]\n")
@@ -853,37 +854,38 @@ class AnalysisSuite extends AnalysisTest with Matchers {
)
// Unwrapped attribute
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CollectMetrics("event", a :: Nil, testRelation, 0),
- expectedErrorClass = "INVALID_OBSERVED_METRICS.NON_AGGREGATE_FUNC_ARG_IS_ATTRIBUTE",
+ expectedErrorCondition = "INVALID_OBSERVED_METRICS.NON_AGGREGATE_FUNC_ARG_IS_ATTRIBUTE",
expectedMessageParameters = Map("expr" -> "\"a\"")
)
// Unwrapped non-deterministic expression
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CollectMetrics("event", Rand(10).as("rnd") :: Nil, testRelation, 0),
- expectedErrorClass = "INVALID_OBSERVED_METRICS.NON_AGGREGATE_FUNC_ARG_IS_NON_DETERMINISTIC",
+ expectedErrorCondition =
+ "INVALID_OBSERVED_METRICS.NON_AGGREGATE_FUNC_ARG_IS_NON_DETERMINISTIC",
expectedMessageParameters = Map("expr" -> "\"rand(10) AS rnd\"")
)
// Distinct aggregate
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CollectMetrics(
"event",
Sum(a).toAggregateExpression(isDistinct = true).as("sum") :: Nil,
testRelation, 0),
- expectedErrorClass =
+ expectedErrorCondition =
"INVALID_OBSERVED_METRICS.AGGREGATE_EXPRESSION_WITH_DISTINCT_UNSUPPORTED",
expectedMessageParameters = Map("expr" -> "\"sum(DISTINCT a) AS sum\"")
)
// Nested aggregate
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CollectMetrics(
"event",
Sum(Sum(a).toAggregateExpression()).toAggregateExpression().as("sum") :: Nil,
testRelation, 0),
- expectedErrorClass = "INVALID_OBSERVED_METRICS.NESTED_AGGREGATES_UNSUPPORTED",
+ expectedErrorCondition = "INVALID_OBSERVED_METRICS.NESTED_AGGREGATES_UNSUPPORTED",
expectedMessageParameters = Map("expr" -> "\"sum(sum(a)) AS sum\"")
)
@@ -892,9 +894,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
RowNumber(),
WindowSpecDefinition(Nil, a.asc :: Nil,
SpecifiedWindowFrame(RowFrame, UnboundedPreceding, CurrentRow)))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CollectMetrics("event", windowExpr.as("rn") :: Nil, testRelation, 0),
- expectedErrorClass = "INVALID_OBSERVED_METRICS.WINDOW_EXPRESSIONS_UNSUPPORTED",
+ expectedErrorCondition = "INVALID_OBSERVED_METRICS.WINDOW_EXPRESSIONS_UNSUPPORTED",
expectedMessageParameters = Map(
"expr" ->
"""
@@ -915,22 +917,22 @@ class AnalysisSuite extends AnalysisTest with Matchers {
CollectMetrics("evt1", count :: Nil, testRelation, 0) :: Nil))
// Same children, structurally different metrics - fail
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Union(
CollectMetrics("evt1", count :: Nil, testRelation, 0) ::
CollectMetrics("evt1", sum :: Nil, testRelation, 1) :: Nil),
- expectedErrorClass = "DUPLICATED_METRICS_NAME",
+ expectedErrorCondition = "DUPLICATED_METRICS_NAME",
expectedMessageParameters = Map("metricName" -> "evt1")
)
// Different children, same metrics - fail
val b = $"b".string
val tblB = LocalRelation(b)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Union(
CollectMetrics("evt1", count :: Nil, testRelation, 0) ::
CollectMetrics("evt1", count :: Nil, tblB, 1) :: Nil),
- expectedErrorClass = "DUPLICATED_METRICS_NAME",
+ expectedErrorCondition = "DUPLICATED_METRICS_NAME",
expectedMessageParameters = Map("metricName" -> "evt1")
)
@@ -939,9 +941,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
val query = Project(
b :: ScalarSubquery(subquery, Nil).as("sum") :: Nil,
CollectMetrics("evt1", count :: Nil, tblB, 1))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
query,
- expectedErrorClass = "DUPLICATED_METRICS_NAME",
+ expectedErrorCondition = "DUPLICATED_METRICS_NAME",
expectedMessageParameters = Map("metricName" -> "evt1")
)
@@ -949,9 +951,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
val sumWithFilter = sum.transform {
case a: AggregateExpression => a.copy(filter = Some(true))
}.asInstanceOf[NamedExpression]
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
CollectMetrics("evt1", sumWithFilter :: Nil, testRelation, 0),
- expectedErrorClass =
+ expectedErrorCondition =
"INVALID_OBSERVED_METRICS.AGGREGATE_EXPRESSION_WITH_FILTER_UNSUPPORTED",
expectedMessageParameters = Map("expr" -> "\"sum(a) FILTER (WHERE true) AS sum\"")
)
@@ -1062,9 +1064,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
AttributeReference("c", IntegerType)(),
AttributeReference("d", TimestampType)())
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Union(firstTable, secondTable),
- expectedErrorClass = "INCOMPATIBLE_COLUMN_TYPE",
+ expectedErrorCondition = "INCOMPATIBLE_COLUMN_TYPE",
expectedMessageParameters = Map(
"tableOrdinalNumber" -> "second",
"columnOrdinalNumber" -> "second",
@@ -1074,9 +1076,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
"dataType1" -> "\"TIMESTAMP\"")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Union(firstTable, thirdTable),
- expectedErrorClass = "INCOMPATIBLE_COLUMN_TYPE",
+ expectedErrorCondition = "INCOMPATIBLE_COLUMN_TYPE",
expectedMessageParameters = Map(
"tableOrdinalNumber" -> "second",
"columnOrdinalNumber" -> "third",
@@ -1086,9 +1088,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
"dataType1" -> "\"TIMESTAMP\"")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Union(firstTable, fourthTable),
- expectedErrorClass = "INCOMPATIBLE_COLUMN_TYPE",
+ expectedErrorCondition = "INCOMPATIBLE_COLUMN_TYPE",
expectedMessageParameters = Map(
"tableOrdinalNumber" -> "second",
"columnOrdinalNumber" -> "4th",
@@ -1098,9 +1100,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
"dataType1" -> "\"TIMESTAMP\"")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Except(firstTable, secondTable, isAll = false),
- expectedErrorClass = "INCOMPATIBLE_COLUMN_TYPE",
+ expectedErrorCondition = "INCOMPATIBLE_COLUMN_TYPE",
expectedMessageParameters = Map(
"tableOrdinalNumber" -> "second",
"columnOrdinalNumber" -> "second",
@@ -1110,9 +1112,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
"dataType1" -> "\"TIMESTAMP\"")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
Intersect(firstTable, secondTable, isAll = false),
- expectedErrorClass = "INCOMPATIBLE_COLUMN_TYPE",
+ expectedErrorCondition = "INCOMPATIBLE_COLUMN_TYPE",
expectedMessageParameters = Map(
"tableOrdinalNumber" -> "second",
"columnOrdinalNumber" -> "second",
@@ -1124,21 +1126,21 @@ class AnalysisSuite extends AnalysisTest with Matchers {
}
test("SPARK-31975: Throw user facing error when use WindowFunction directly") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = testRelation2.select(RowNumber()),
- expectedErrorClass = "WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE",
+ expectedErrorCondition = "WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE",
expectedMessageParameters = Map("funcName" -> "\"row_number()\"")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = testRelation2.select(Sum(RowNumber())),
- expectedErrorClass = "WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE",
+ expectedErrorCondition = "WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE",
expectedMessageParameters = Map("funcName" -> "\"row_number()\"")
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = testRelation2.select(RowNumber() + 1),
- expectedErrorClass = "WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE",
+ expectedErrorCondition = "WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE",
expectedMessageParameters = Map("funcName" -> "\"row_number()\"")
)
}
@@ -1297,7 +1299,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
| ORDER BY grouping__id > 0
""".stripMargin), false)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsePlan(
"""
|SELECT grouping__id FROM (
@@ -1328,7 +1330,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
|ORDER BY c.x
|""".stripMargin))
- assertAnalysisErrorClass(parsePlan(
+ assertAnalysisErrorCondition(parsePlan(
"""
|SELECT c.x
|FROM VALUES NAMED_STRUCT('x', 'A', 'y', 1), NAMED_STRUCT('x', 'A', 'y', 2) AS t(c)
@@ -1342,7 +1344,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
}
test("SPARK-38118: Func(wrong_type) in the HAVING clause should throw data mismatch error") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = parsePlan(
s"""
|WITH t as (SELECT true c)
@@ -1350,7 +1352,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
|FROM t
|GROUP BY t.c
|HAVING mean(t.c) > 0d""".stripMargin),
- expectedErrorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ expectedErrorCondition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
expectedMessageParameters = Map(
"sqlExpr" -> "\"mean(c)\"",
"paramIndex" -> "first",
@@ -1361,7 +1363,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
caseSensitive = false
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = parsePlan(
s"""
|WITH t as (SELECT true c, false d)
@@ -1369,7 +1371,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
|FROM t
|GROUP BY t.c, t.d
|HAVING mean(c) > 0d""".stripMargin),
- expectedErrorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ expectedErrorCondition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
expectedMessageParameters = Map(
"sqlExpr" -> "\"mean(c)\"",
"paramIndex" -> "first",
@@ -1379,7 +1381,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
queryContext = Array(ExpectedContext("mean(c)", 91, 97)),
caseSensitive = false)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = parsePlan(
s"""
|WITH t as (SELECT true c)
@@ -1387,7 +1389,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
|FROM t
|GROUP BY t.c
|HAVING abs(t.c) > 0d""".stripMargin),
- expectedErrorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ expectedErrorCondition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
expectedMessageParameters = Map(
"sqlExpr" -> "\"abs(c)\"",
"paramIndex" -> "first",
@@ -1399,7 +1401,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
caseSensitive = false
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = parsePlan(
s"""
|WITH t as (SELECT true c, false d)
@@ -1407,7 +1409,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
|FROM t
|GROUP BY t.c, t.d
|HAVING abs(c) > 0d""".stripMargin),
- expectedErrorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ expectedErrorCondition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
expectedMessageParameters = Map(
"sqlExpr" -> "\"abs(c)\"",
"paramIndex" -> "first",
@@ -1421,7 +1423,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
}
test("SPARK-39354: should be [TABLE_OR_VIEW_NOT_FOUND]") {
- assertAnalysisErrorClass(parsePlan(
+ assertAnalysisErrorCondition(parsePlan(
s"""
|WITH t1 as (SELECT 1 user_id, CAST("2022-06-02" AS DATE) dt)
|SELECT *
@@ -1531,13 +1533,13 @@ class AnalysisSuite extends AnalysisTest with Matchers {
}
test("SPARK-41489: type of filter expression should be a bool") {
- assertAnalysisErrorClass(parsePlan(
+ assertAnalysisErrorCondition(parsePlan(
s"""
|WITH t1 as (SELECT 1 user_id)
|SELECT *
|FROM t1
|WHERE 'true'""".stripMargin),
- expectedErrorClass = "DATATYPE_MISMATCH.FILTER_NOT_BOOLEAN",
+ expectedErrorCondition = "DATATYPE_MISMATCH.FILTER_NOT_BOOLEAN",
expectedMessageParameters = Map(
"sqlExpr" -> "\"true\"", "filter" -> "\"true\"", "type" -> "\"STRING\"")
,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
index dc95198574fb4..33b9fb488c94f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
@@ -178,9 +178,9 @@ trait AnalysisTest extends PlanTest {
}
}
- protected def assertAnalysisErrorClass(
+ protected def assertAnalysisErrorCondition(
inputPlan: LogicalPlan,
- expectedErrorClass: String,
+ expectedErrorCondition: String,
expectedMessageParameters: Map[String, String],
queryContext: Array[ExpectedContext] = Array.empty,
caseSensitive: Boolean = true): Unit = {
@@ -191,7 +191,7 @@ trait AnalysisTest extends PlanTest {
}
checkError(
exception = e,
- errorClass = expectedErrorClass,
+ condition = expectedErrorCondition,
parameters = expectedMessageParameters,
queryContext = queryContext
)
@@ -199,14 +199,13 @@ trait AnalysisTest extends PlanTest {
}
protected def interceptParseException(parser: String => Any)(
- sqlCommand: String, messages: String*)(
- errorClass: Option[String] = None): Unit = {
+ sqlCommand: String, messages: String*)(condition: Option[String] = None): Unit = {
val e = parseException(parser)(sqlCommand)
messages.foreach { message =>
assert(e.message.contains(message))
}
- if (errorClass.isDefined) {
- assert(e.getErrorClass == errorClass.get)
+ if (condition.isDefined) {
+ assert(e.getErrorClass == condition.get)
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/CreateTablePartitioningValidationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/CreateTablePartitioningValidationSuite.scala
index c9e37e255ab44..6b034d3dbee09 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/CreateTablePartitioningValidationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/CreateTablePartitioningValidationSuite.scala
@@ -41,8 +41,8 @@ class CreateTablePartitioningValidationSuite extends AnalysisTest {
ignoreIfExists = false)
assert(!plan.resolved)
- assertAnalysisErrorClass(plan,
- expectedErrorClass = "UNSUPPORTED_FEATURE.PARTITION_WITH_NESTED_COLUMN_IS_UNSUPPORTED",
+ assertAnalysisErrorCondition(plan,
+ expectedErrorCondition = "UNSUPPORTED_FEATURE.PARTITION_WITH_NESTED_COLUMN_IS_UNSUPPORTED",
expectedMessageParameters = Map("cols" -> "`does_not_exist`"))
}
@@ -56,8 +56,8 @@ class CreateTablePartitioningValidationSuite extends AnalysisTest {
ignoreIfExists = false)
assert(!plan.resolved)
- assertAnalysisErrorClass(plan,
- expectedErrorClass = "UNSUPPORTED_FEATURE.PARTITION_WITH_NESTED_COLUMN_IS_UNSUPPORTED",
+ assertAnalysisErrorCondition(plan,
+ expectedErrorCondition = "UNSUPPORTED_FEATURE.PARTITION_WITH_NESTED_COLUMN_IS_UNSUPPORTED",
expectedMessageParameters = Map("cols" -> "`does_not_exist`.`z`"))
}
@@ -71,8 +71,8 @@ class CreateTablePartitioningValidationSuite extends AnalysisTest {
ignoreIfExists = false)
assert(!plan.resolved)
- assertAnalysisErrorClass(plan,
- expectedErrorClass = "UNSUPPORTED_FEATURE.PARTITION_WITH_NESTED_COLUMN_IS_UNSUPPORTED",
+ assertAnalysisErrorCondition(plan,
+ expectedErrorCondition = "UNSUPPORTED_FEATURE.PARTITION_WITH_NESTED_COLUMN_IS_UNSUPPORTED",
expectedMessageParameters = Map("cols" -> "`point`.`z`"))
}
@@ -86,8 +86,8 @@ class CreateTablePartitioningValidationSuite extends AnalysisTest {
ignoreIfExists = false)
assert(!plan.resolved)
- assertAnalysisErrorClass(plan,
- expectedErrorClass = "UNSUPPORTED_FEATURE.PARTITION_WITH_NESTED_COLUMN_IS_UNSUPPORTED",
+ assertAnalysisErrorCondition(plan,
+ expectedErrorCondition = "UNSUPPORTED_FEATURE.PARTITION_WITH_NESTED_COLUMN_IS_UNSUPPORTED",
expectedMessageParameters = Map("cols" -> "`does_not_exist`, `point`.`z`"))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
index 4367cbbd24a89..95e118a30771c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
@@ -53,7 +53,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
expr: Expression, messageParameters: Map[String, String]): Unit = {
checkError(
exception = analysisException(expr),
- errorClass = "DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES",
+ condition = "DATATYPE_MISMATCH.BINARY_OP_DIFF_TYPES",
parameters = messageParameters)
}
@@ -61,7 +61,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
expr: Expression, messageParameters: Map[String, String]): Unit = {
checkError(
exception = analysisException(expr),
- errorClass = "DATATYPE_MISMATCH.INVALID_ORDERING_TYPE",
+ condition = "DATATYPE_MISMATCH.INVALID_ORDERING_TYPE",
parameters = messageParameters)
}
@@ -69,7 +69,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
expr: Expression, messageParameters: Map[String, String]): Unit = {
checkError(
exception = analysisException(expr),
- errorClass = "DATATYPE_MISMATCH.DATA_DIFF_TYPES",
+ condition = "DATATYPE_MISMATCH.DATA_DIFF_TYPES",
parameters = messageParameters)
}
@@ -77,14 +77,14 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
expr: Expression, messageParameters: Map[String, String]): Unit = {
checkError(
exception = analysisException(expr),
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = messageParameters)
}
private def assertForWrongType(expr: Expression, messageParameters: Map[String, String]): Unit = {
checkError(
exception = analysisException(expr),
- errorClass = "DATATYPE_MISMATCH.BINARY_OP_WRONG_TYPE",
+ condition = "DATATYPE_MISMATCH.BINARY_OP_WRONG_TYPE",
parameters = messageParameters)
}
@@ -93,7 +93,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(BitwiseNot($"stringField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"~stringField\"",
"paramIndex" -> ordinalNumber(0),
@@ -426,7 +426,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(Sum($"booleanField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"sum(booleanField)\"",
"paramIndex" -> ordinalNumber(0),
@@ -437,7 +437,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(Average($"booleanField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"avg(booleanField)\"",
"paramIndex" -> ordinalNumber(0),
@@ -469,7 +469,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(coalesce)
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> toSQLId(coalesce.prettyName),
"expectedNum" -> "> 0",
@@ -481,7 +481,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(murmur3Hash)
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> toSQLId(murmur3Hash.prettyName),
"expectedNum" -> "> 0",
@@ -493,7 +493,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(xxHash64)
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> toSQLId(xxHash64.prettyName),
"expectedNum" -> "> 0",
@@ -504,7 +504,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(Explode($"intField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"explode(intField)\"",
"paramIndex" -> ordinalNumber(0),
@@ -516,7 +516,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(PosExplode($"intField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"posexplode(intField)\"",
"paramIndex" -> ordinalNumber(0),
@@ -529,7 +529,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
test("check types for CreateNamedStruct") {
checkError(
exception = analysisException(CreateNamedStruct(Seq("a", "b", 2.0))),
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`named_struct`",
"expectedNum" -> "2n (n > 0)",
@@ -538,21 +538,21 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
)
checkError(
exception = analysisException(CreateNamedStruct(Seq(1, "a", "b", 2.0))),
- errorClass = "DATATYPE_MISMATCH.CREATE_NAMED_STRUCT_WITHOUT_FOLDABLE_STRING",
+ condition = "DATATYPE_MISMATCH.CREATE_NAMED_STRUCT_WITHOUT_FOLDABLE_STRING",
parameters = Map(
"sqlExpr" -> "\"named_struct(1, a, b, 2.0)\"",
"inputExprs" -> "[\"1\"]")
)
checkError(
exception = analysisException(CreateNamedStruct(Seq($"a".string.at(0), "a", "b", 2.0))),
- errorClass = "DATATYPE_MISMATCH.CREATE_NAMED_STRUCT_WITHOUT_FOLDABLE_STRING",
+ condition = "DATATYPE_MISMATCH.CREATE_NAMED_STRUCT_WITHOUT_FOLDABLE_STRING",
parameters = Map(
"sqlExpr" -> "\"named_struct(boundreference(), a, b, 2.0)\"",
"inputExprs" -> "[\"boundreference()\"]")
)
checkError(
exception = analysisException(CreateNamedStruct(Seq(Literal.create(null, StringType), "a"))),
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_NULL",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_NULL",
parameters = Map(
"sqlExpr" -> "\"named_struct(NULL, a)\"",
"exprName" -> "[\"NULL\"]")
@@ -562,7 +562,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
test("check types for CreateMap") {
checkError(
exception = analysisException(CreateMap(Seq("a", "b", 2.0))),
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`map`",
"expectedNum" -> "2n (n > 0)",
@@ -572,7 +572,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
checkError(
exception = analysisException(CreateMap(Seq(Literal(1),
Literal("a"), Literal(true), Literal("b")))),
- errorClass = "DATATYPE_MISMATCH.CREATE_MAP_KEY_DIFF_TYPES",
+ condition = "DATATYPE_MISMATCH.CREATE_MAP_KEY_DIFF_TYPES",
parameters = Map(
"sqlExpr" -> "\"map(1, a, true, b)\"",
"functionName" -> "`map`",
@@ -582,7 +582,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
checkError(
exception = analysisException(CreateMap(Seq(Literal("a"),
Literal(1), Literal("b"), Literal(true)))),
- errorClass = "DATATYPE_MISMATCH.CREATE_MAP_VALUE_DIFF_TYPES",
+ condition = "DATATYPE_MISMATCH.CREATE_MAP_VALUE_DIFF_TYPES",
parameters = Map(
"sqlExpr" -> "\"map(a, 1, b, true)\"",
"functionName" -> "`map`",
@@ -599,7 +599,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(Round($"intField", $"intField"))
},
- errorClass = "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
+ condition = "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
parameters = Map(
"sqlExpr" -> "\"round(intField, intField)\"",
"inputName" -> "`scale`",
@@ -610,7 +610,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(Round($"intField", $"booleanField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"round(intField, booleanField)\"",
"paramIndex" -> ordinalNumber(1),
@@ -621,7 +621,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(Round($"intField", $"mapField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"round(intField, mapField)\"",
"paramIndex" -> ordinalNumber(1),
@@ -632,7 +632,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(Round($"booleanField", $"intField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"round(booleanField, intField)\"",
"paramIndex" -> ordinalNumber(0),
@@ -646,7 +646,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(BRound($"intField", $"intField"))
},
- errorClass = "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
+ condition = "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
parameters = Map(
"sqlExpr" -> "\"bround(intField, intField)\"",
"inputName" -> "`scale`",
@@ -656,7 +656,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(BRound($"intField", $"booleanField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"bround(intField, booleanField)\"",
"paramIndex" -> ordinalNumber(1),
@@ -667,7 +667,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(BRound($"intField", $"mapField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"bround(intField, mapField)\"",
"paramIndex" -> ordinalNumber(1),
@@ -678,7 +678,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[AnalysisException] {
assertSuccess(BRound($"booleanField", $"intField"))
},
- errorClass = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
+ condition = "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE",
parameters = Map(
"sqlExpr" -> "\"bround(booleanField, intField)\"",
"paramIndex" -> ordinalNumber(0),
@@ -806,7 +806,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
exception = intercept[SparkException] {
wsd.checkInputDataTypes()
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> ("Cannot use an UnspecifiedFrame. " +
"This should have been converted during analysis."))
)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/LookupFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/LookupFunctionsSuite.scala
index 1fd81349ac720..1ae3e3fa68603 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/LookupFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/LookupFunctionsSuite.scala
@@ -62,7 +62,7 @@ class LookupFunctionsSuite extends PlanTest {
}
checkError(
exception = cause,
- errorClass = "UNRESOLVED_ROUTINE",
+ condition = "UNRESOLVED_ROUTINE",
parameters = Map(
"routineName" -> "`undefined_fn`",
"searchPath" -> "[`system`.`builtin`, `system`.`session`, `spark_catalog`.`db1`]"))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/NamedParameterFunctionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/NamedParameterFunctionSuite.scala
index 6bc0350a5785d..02543c9fba539 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/NamedParameterFunctionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/NamedParameterFunctionSuite.scala
@@ -98,18 +98,18 @@ class NamedParameterFunctionSuite extends AnalysisTest {
}
test("DUPLICATE_ROUTINE_PARAMETER_ASSIGNMENT") {
- val errorClass =
+ val condition =
"DUPLICATE_ROUTINE_PARAMETER_ASSIGNMENT.BOTH_POSITIONAL_AND_NAMED"
checkError(
exception = parseRearrangeException(
signature, Seq(k1Arg, k2Arg, k3Arg, k4Arg, namedK1Arg), "foo"),
- errorClass = errorClass,
+ condition = condition,
parameters = Map("routineName" -> toSQLId("foo"), "parameterName" -> toSQLId("k1"))
)
checkError(
exception = parseRearrangeException(
signature, Seq(k1Arg, k2Arg, k3Arg, k4Arg, k4Arg), "foo"),
- errorClass = "DUPLICATE_ROUTINE_PARAMETER_ASSIGNMENT.DOUBLE_NAMED_ARGUMENT_REFERENCE",
+ condition = "DUPLICATE_ROUTINE_PARAMETER_ASSIGNMENT.DOUBLE_NAMED_ARGUMENT_REFERENCE",
parameters = Map("routineName" -> toSQLId("foo"), "parameterName" -> toSQLId("k4"))
)
}
@@ -117,7 +117,7 @@ class NamedParameterFunctionSuite extends AnalysisTest {
test("REQUIRED_PARAMETER_NOT_FOUND") {
checkError(
exception = parseRearrangeException(signature, Seq(k1Arg, k2Arg, k3Arg), "foo"),
- errorClass = "REQUIRED_PARAMETER_NOT_FOUND",
+ condition = "REQUIRED_PARAMETER_NOT_FOUND",
parameters = Map(
"routineName" -> toSQLId("foo"), "parameterName" -> toSQLId("k4"), "index" -> "2"))
}
@@ -126,7 +126,7 @@ class NamedParameterFunctionSuite extends AnalysisTest {
checkError(
exception = parseRearrangeException(signature,
Seq(k1Arg, k2Arg, k3Arg, k4Arg, NamedArgumentExpression("k5", Literal("k5"))), "foo"),
- errorClass = "UNRECOGNIZED_PARAMETER_NAME",
+ condition = "UNRECOGNIZED_PARAMETER_NAME",
parameters = Map("routineName" -> toSQLId("foo"), "argumentName" -> toSQLId("k5"),
"proposal" -> (toSQLId("k1") + " " + toSQLId("k2") + " " + toSQLId("k3")))
)
@@ -136,7 +136,7 @@ class NamedParameterFunctionSuite extends AnalysisTest {
checkError(
exception = parseRearrangeException(signature,
Seq(k2Arg, k3Arg, k1Arg, k4Arg), "foo"),
- errorClass = "UNEXPECTED_POSITIONAL_ARGUMENT",
+ condition = "UNEXPECTED_POSITIONAL_ARGUMENT",
parameters = Map("routineName" -> toSQLId("foo"), "parameterName" -> toSQLId("k3"))
)
}
@@ -147,7 +147,7 @@ class NamedParameterFunctionSuite extends AnalysisTest {
s" All required arguments should come before optional arguments."
checkError(
exception = parseRearrangeException(illegalSignature, args, "foo"),
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> errorMessage)
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala
index 5809d1e04b9cf..6e911324e0759 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveLambdaVariablesSuite.scala
@@ -82,7 +82,7 @@ class ResolveLambdaVariablesSuite extends PlanTest {
checkError(
exception = intercept[AnalysisException](Analyzer.execute(p)),
- errorClass = "INVALID_LAMBDA_FUNCTION_CALL.DUPLICATE_ARG_NAMES",
+ condition = "INVALID_LAMBDA_FUNCTION_CALL.DUPLICATE_ARG_NAMES",
parameters = Map(
"args" -> "`x`, `x`",
"caseSensitiveConfig" -> "\"spark.sql.caseSensitive\"")
@@ -96,7 +96,7 @@ class ResolveLambdaVariablesSuite extends PlanTest {
checkError(
exception = intercept[AnalysisException](Analyzer.execute(p)),
- errorClass = "INVALID_LAMBDA_FUNCTION_CALL.NUM_ARGS_MISMATCH",
+ condition = "INVALID_LAMBDA_FUNCTION_CALL.NUM_ARGS_MISMATCH",
parameters = Map("expectedNumArgs" -> "3", "actualNumArgs" -> "1")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
index 5c843d62d6d7c..b7afc803410cf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
@@ -108,14 +108,14 @@ class ResolveNaturalJoinSuite extends AnalysisTest {
}
test("using unresolved attribute") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
r1.join(r2, UsingJoin(Inner, Seq("d"))),
- expectedErrorClass = "UNRESOLVED_USING_COLUMN_FOR_JOIN",
+ expectedErrorCondition = "UNRESOLVED_USING_COLUMN_FOR_JOIN",
expectedMessageParameters = Map(
"colName" -> "`d`", "side" -> "left", "suggestion" -> "`a`, `b`"))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
r1.join(r2, UsingJoin(Inner, Seq("b"))),
- expectedErrorClass = "UNRESOLVED_USING_COLUMN_FOR_JOIN",
+ expectedErrorCondition = "UNRESOLVED_USING_COLUMN_FOR_JOIN",
expectedMessageParameters = Map(
"colName" -> "`b`", "side" -> "right", "suggestion" -> "`a`, `c`"))
}
@@ -126,17 +126,17 @@ class ResolveNaturalJoinSuite extends AnalysisTest {
val usingPlan = r1.join(r2, UsingJoin(Inner, Seq("a")), None)
checkAnalysis(usingPlan, expected, caseSensitive = true)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
r1.join(r2, UsingJoin(Inner, Seq("A"))),
- expectedErrorClass = "UNRESOLVED_USING_COLUMN_FOR_JOIN",
+ expectedErrorCondition = "UNRESOLVED_USING_COLUMN_FOR_JOIN",
expectedMessageParameters = Map(
"colName" -> "`A`", "side" -> "left", "suggestion" -> "`a`, `b`"))
}
test("using join on nested fields") {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
r5.join(r6, UsingJoin(Inner, Seq("d.f1"))),
- expectedErrorClass = "UNRESOLVED_USING_COLUMN_FOR_JOIN",
+ expectedErrorCondition = "UNRESOLVED_USING_COLUMN_FOR_JOIN",
expectedMessageParameters = Map(
"colName" -> "`d`.`f1`", "side" -> "left", "suggestion" -> "`d`"))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveSubquerySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveSubquerySuite.scala
index 4e17f4624f7e0..86718ee434311 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveSubquerySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveSubquerySuite.scala
@@ -71,7 +71,7 @@ class ResolveSubquerySuite extends AnalysisTest {
test("lateral join with ambiguous join conditions") {
val plan = lateralJoin(t1, t0.select($"b"), condition = Some($"b" === 1))
- assertAnalysisErrorClass(plan,
+ assertAnalysisErrorCondition(plan,
"AMBIGUOUS_REFERENCE", Map("name" -> "`b`", "referenceNames" -> "[`b`, `b`]")
)
}
@@ -123,7 +123,7 @@ class ResolveSubquerySuite extends AnalysisTest {
// SELECT * FROM t1, LATERAL (SELECT * FROM t2, LATERAL (SELECT a, b, c))
// TODO: support accessing columns from outer outer query.
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
lateralJoin(t1, lateralJoin(t2, t0.select($"a", $"b", $"c"))),
"UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
Map("objectName" -> "`a`")
@@ -132,25 +132,25 @@ class ResolveSubquerySuite extends AnalysisTest {
test("lateral subquery with unresolvable attributes") {
// SELECT * FROM t1, LATERAL (SELECT a, c)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
lateralJoin(t1, t0.select($"a", $"c")),
"UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
Map("objectName" -> "`c`")
)
// SELECT * FROM t1, LATERAL (SELECT a, b, c, d FROM t2)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
lateralJoin(t1, t2.select($"a", $"b", $"c", $"d")),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`d`", "proposal" -> "`b`, `c`")
)
// SELECT * FROM t1, LATERAL (SELECT * FROM t2, LATERAL (SELECT t1.a))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
lateralJoin(t1, lateralJoin(t2, t0.select($"t1.a"))),
"UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
Map("objectName" -> "`t1`.`a`")
)
// SELECT * FROM t1, LATERAL (SELECT * FROM t2, LATERAL (SELECT a, b))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
lateralJoin(t1, lateralJoin(t2, t0.select($"a", $"b"))),
"UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
Map("objectName" -> "`a`")
@@ -165,7 +165,7 @@ class ResolveSubquerySuite extends AnalysisTest {
LateralJoin(t4, LateralSubquery(Project(Seq(xa, ya), t0), Seq(x, y)), Inner, None)
)
// Analyzer will try to resolve struct first before subquery alias.
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
lateralJoin(t1.as("x"), t4.select($"x.a", $"x.b")),
"FIELD_NOT_FOUND",
Map("fieldName" -> "`b`", "fields" -> "`a`"))
@@ -174,9 +174,9 @@ class ResolveSubquerySuite extends AnalysisTest {
test("lateral join with unsupported expressions") {
val plan = lateralJoin(t1, t0.select(($"a" + $"b").as("c")),
condition = Some(sum($"a") === sum($"c")))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
plan,
- expectedErrorClass = "UNSUPPORTED_EXPR_FOR_OPERATOR",
+ expectedErrorCondition = "UNSUPPORTED_EXPR_FOR_OPERATOR",
expectedMessageParameters = Map("invalidExprSqls" -> "\"sum(a)\", \"sum(c)\"")
)
}
@@ -206,17 +206,17 @@ class ResolveSubquerySuite extends AnalysisTest {
LateralSubquery(Project(Seq(outerA, outerB, b, c), t2.as("t2")), Seq(a, b)), Inner, None)
)
// SELECT * FROM t1, LATERAL (SELECT t2.*)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
lateralJoin(t1.as("t1"), t0.select(star("t2"))),
- expectedErrorClass = "CANNOT_RESOLVE_STAR_EXPAND",
+ expectedErrorCondition = "CANNOT_RESOLVE_STAR_EXPAND",
expectedMessageParameters = Map("targetString" -> "`t2`", "columns" -> "")
)
// Check case sensitivities.
// SELECT * FROM t1, LATERAL (SELECT T1.*)
val plan = lateralJoin(t1.as("t1"), t0.select(star("T1")))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
plan,
- expectedErrorClass = "CANNOT_RESOLVE_STAR_EXPAND",
+ expectedErrorCondition = "CANNOT_RESOLVE_STAR_EXPAND",
expectedMessageParameters = Map("targetString" -> "`T1`", "columns" -> "")
)
assertAnalysisSuccess(plan, caseSensitive = false)
@@ -232,9 +232,9 @@ class ResolveSubquerySuite extends AnalysisTest {
LateralJoin(t1,
LateralSubquery(t0.select(newArray.as(newArray.sql)), Seq(a, b)), Inner, None)
)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
lateralJoin(t1.as("t1"), t0.select(Count(star("t1")))),
- expectedErrorClass = "INVALID_USAGE_OF_STAR_OR_REGEX",
+ expectedErrorCondition = "INVALID_USAGE_OF_STAR_OR_REGEX",
expectedMessageParameters = Map("elem" -> "'*'", "prettyName" -> "expression `count`"))
}
@@ -293,9 +293,9 @@ class ResolveSubquerySuite extends AnalysisTest {
:: lv(Symbol("X"))
:: Nil))
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = lambdaPlanScanFromTable,
- expectedErrorClass =
+ expectedErrorCondition =
"UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.HIGHER_ORDER_FUNCTION",
expectedMessageParameters = Map.empty[String, String])
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
index ff9c0a1b34f7f..3e9a93dc743df 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
@@ -740,7 +740,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite with SQLHelper {
testUnaryOperatorInStreamingPlan(
"window",
Window(Nil, Nil, Nil, _),
- errorClass = "NON_TIME_WINDOW_NOT_SUPPORTED_IN_STREAMING")
+ condition = "NON_TIME_WINDOW_NOT_SUPPORTED_IN_STREAMING")
// Output modes with aggregation and non-aggregation plans
testOutputMode(Append, shouldSupportAggregation = false, shouldSupportNonAggregation = true)
@@ -869,11 +869,11 @@ class UnsupportedOperationsSuite extends SparkFunSuite with SQLHelper {
* supports having a batch child plan, forming a batch subplan inside a streaming plan.
*/
def testUnaryOperatorInStreamingPlan(
- operationName: String,
- logicalPlanGenerator: LogicalPlan => LogicalPlan,
- outputMode: OutputMode = Append,
- expectedMsg: String = "",
- errorClass: String = ""): Unit = {
+ operationName: String,
+ logicalPlanGenerator: LogicalPlan => LogicalPlan,
+ outputMode: OutputMode = Append,
+ expectedMsg: String = "",
+ condition: String = ""): Unit = {
val expectedMsgs = if (expectedMsg.isEmpty) Seq(operationName) else Seq(expectedMsg)
@@ -882,7 +882,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite with SQLHelper {
wrapInStreaming(logicalPlanGenerator(streamRelation)),
outputMode,
expectedMsgs,
- errorClass)
+ condition)
assertSupportedInStreamingPlan(
s"$operationName with batch relation",
@@ -1030,11 +1030,11 @@ class UnsupportedOperationsSuite extends SparkFunSuite with SQLHelper {
plan: LogicalPlan,
outputMode: OutputMode,
expectedMsgs: Seq[String],
- errorClass: String = ""): Unit = {
+ condition: String = ""): Unit = {
testError(
s"streaming plan - $name: not supported",
expectedMsgs :+ "streaming" :+ "DataFrame" :+ "Dataset" :+ "not supported",
- errorClass) {
+ condition) {
UnsupportedOperationChecker.checkForStreaming(wrapInStreaming(plan), outputMode)
}
}
@@ -1052,7 +1052,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[AnalysisException] {
UnsupportedOperationChecker.checkForStreaming(wrapInStreaming(plan), outputMode)
},
- errorClass = "STREAMING_OUTPUT_MODE.UNSUPPORTED_OPERATION",
+ condition = "STREAMING_OUTPUT_MODE.UNSUPPORTED_OPERATION",
sqlState = "42KDE",
parameters = Map(
"outputMode" -> outputMode.toString.toLowerCase(Locale.ROOT),
@@ -1120,7 +1120,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite with SQLHelper {
def testError(
testName: String,
expectedMsgs: Seq[String],
- errorClass: String = "")(testBody: => Unit): Unit = {
+ condition: String = "")(testBody: => Unit): Unit = {
test(testName) {
val e = intercept[AnalysisException] {
@@ -1132,8 +1132,8 @@ class UnsupportedOperationsSuite extends SparkFunSuite with SQLHelper {
s"actual exception message:\n\t'${e.getMessage}'")
}
}
- if (!errorClass.isEmpty) {
- assert(e.getErrorClass == errorClass)
+ if (!condition.isEmpty) {
+ assert(e.getErrorClass == condition)
}
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/V2WriteAnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/V2WriteAnalysisSuite.scala
index 2280463c2f244..29c6c63ecfeab 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/V2WriteAnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/V2WriteAnalysisSuite.scala
@@ -154,16 +154,16 @@ abstract class V2ANSIWriteAnalysisSuiteBase extends V2WriteAnalysisSuiteBase {
}
}
- override def assertAnalysisErrorClass(
+ override def assertAnalysisErrorCondition(
inputPlan: LogicalPlan,
- expectedErrorClass: String,
+ expectedErrorCondition: String,
expectedMessageParameters: Map[String, String],
queryContext: Array[ExpectedContext] = Array.empty,
caseSensitive: Boolean = true): Unit = {
withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.ANSI.toString) {
- super.assertAnalysisErrorClass(
+ super.assertAnalysisErrorCondition(
inputPlan,
- expectedErrorClass,
+ expectedErrorCondition,
expectedMessageParameters,
queryContext,
caseSensitive
@@ -191,16 +191,16 @@ abstract class V2StrictWriteAnalysisSuiteBase extends V2WriteAnalysisSuiteBase {
}
}
- override def assertAnalysisErrorClass(
+ override def assertAnalysisErrorCondition(
inputPlan: LogicalPlan,
- expectedErrorClass: String,
+ expectedErrorCondition: String,
expectedMessageParameters: Map[String, String],
queryContext: Array[ExpectedContext] = Array.empty,
caseSensitive: Boolean = true): Unit = {
withSQLConf(SQLConf.STORE_ASSIGNMENT_POLICY.key -> StoreAssignmentPolicy.STRICT.toString) {
- super.assertAnalysisErrorClass(
+ super.assertAnalysisErrorCondition(
inputPlan,
- expectedErrorClass,
+ expectedErrorCondition,
expectedMessageParameters,
queryContext,
caseSensitive
@@ -212,9 +212,9 @@ abstract class V2StrictWriteAnalysisSuiteBase extends V2WriteAnalysisSuiteBase {
val parsedPlan = byName(table, widerTable)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`x`",
@@ -235,9 +235,9 @@ abstract class V2StrictWriteAnalysisSuiteBase extends V2WriteAnalysisSuiteBase {
val parsedPlan = byName(xRequiredTable, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`x`",
@@ -254,9 +254,9 @@ abstract class V2StrictWriteAnalysisSuiteBase extends V2WriteAnalysisSuiteBase {
val parsedPlan = byPosition(table, widerTable)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`x`",
@@ -277,9 +277,9 @@ abstract class V2StrictWriteAnalysisSuiteBase extends V2WriteAnalysisSuiteBase {
val parsedPlan = byPosition(xRequiredTable, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`x`",
@@ -421,9 +421,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = byName(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
expectedMessageParameters = Map("tableName" -> "`table-name`", "colName" -> "`x`")
)
}
@@ -436,9 +436,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = byName(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
expectedMessageParameters = Map("tableName" -> "`table-name`", "colName" -> "`x`")
)
}
@@ -499,9 +499,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = byName(requiredTable, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
expectedMessageParameters = Map("tableName" -> "`table-name`", "colName" -> "`x`")
)
}
@@ -514,9 +514,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = byName(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
expectedMessageParameters = Map("tableName" -> "`table-name`", "colName" -> "`x`")
)
}
@@ -546,9 +546,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = byName(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = parsedPlan,
- expectedErrorClass = "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS",
+ expectedErrorCondition = "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"tableColumns" -> "`x`, `y`",
@@ -561,9 +561,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val query = TestRelation(Seq($"b".struct($"y".int, $"x".int, $"z".int), $"a".int))
val writePlan = byName(table, query)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
writePlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`b`",
@@ -636,9 +636,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = byPosition(requiredTable, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = parsedPlan,
- expectedErrorClass = "INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS",
+ expectedErrorCondition = "INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"tableColumns" -> "`x`, `y`",
@@ -654,9 +654,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = byPosition(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = parsedPlan,
- expectedErrorClass = "INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS",
+ expectedErrorCondition = "INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"tableColumns" -> "`x`, `y`",
@@ -693,9 +693,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = byName(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
inputPlan = parsedPlan,
- expectedErrorClass = "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS",
+ expectedErrorCondition = "INSERT_COLUMN_ARITY_MISMATCH.TOO_MANY_DATA_COLUMNS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"tableColumns" -> "`x`, `y`",
@@ -740,9 +740,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
withClue("byName") {
val parsedPlan = byName(tableWithStructCol, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
expectedMessageParameters = Map("tableName" -> "`table-name`", "colName" -> "`col`.`a`")
)
}
@@ -792,9 +792,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = if (byNameResolution) byName(table, query) else byPosition(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`b`.`n2`",
@@ -821,9 +821,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = if (byNameResolution) byName(table, query) else byPosition(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`arr`.`element`",
@@ -854,9 +854,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = if (byNameResolution) byName(table, query) else byPosition(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`m`.`key`",
@@ -887,9 +887,9 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan = if (byNameResolution) byName(table, query) else byPosition(table, query)
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`m`.`value`",
@@ -921,17 +921,17 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
assertNotResolved(parsedPlan)
if (byNameResolution) {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`b`.`n2`.`dn3`")
)
} else {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`b`.`n2`",
@@ -964,17 +964,17 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
assertNotResolved(parsedPlan)
if (byNameResolution) {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`arr`.`element`.`y`")
)
} else {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`arr`.`element`",
@@ -1011,17 +1011,17 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
assertNotResolved(parsedPlan)
if (byNameResolution) {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`m`.`key`.`y`")
)
} else {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`m`.`key`",
@@ -1058,17 +1058,17 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
assertNotResolved(parsedPlan)
if (byNameResolution) {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_FIND_DATA",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`m`.`value`.`y`")
)
} else {
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
- expectedErrorClass = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
+ expectedErrorCondition = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
expectedMessageParameters = Map(
"tableName" -> "`table-name`",
"colName" -> "`m`.`value`",
@@ -1363,7 +1363,7 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
LessThanOrEqual(UnresolvedAttribute(Seq("a")), Literal(15.0d)))
assertNotResolved(parsedPlan)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan,
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`a`", "proposal" -> "`x`, `y`")
@@ -1376,7 +1376,7 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
val parsedPlan2 = OverwriteByExpression.byPosition(tableAcceptAnySchema, query,
LessThanOrEqual(UnresolvedAttribute(Seq("a")), Literal(15.0d)))
assertNotResolved(parsedPlan2)
- assertAnalysisErrorClass(
+ assertAnalysisErrorCondition(
parsedPlan2,
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`a`", "proposal" -> "`x`, `y`")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
index 7e2bad484b3a6..fbe63f71ae029 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
@@ -121,7 +121,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
func(name)
},
- errorClass = "INVALID_SCHEMA_OR_RELATION_NAME",
+ condition = "INVALID_SCHEMA_OR_RELATION_NAME",
parameters = Map("name" -> toSQLId(name))
)
}
@@ -171,7 +171,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
ResolveDefaultColumns.analyze(columnC, statementType)
},
- errorClass = "INVALID_DEFAULT_VALUE.UNRESOLVED_EXPRESSION",
+ condition = "INVALID_DEFAULT_VALUE.UNRESOLVED_EXPRESSION",
parameters = Map(
"statement" -> "CREATE TABLE",
"colName" -> "`c`",
@@ -180,7 +180,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
ResolveDefaultColumns.analyze(columnD, statementType)
},
- errorClass = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ condition = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
parameters = Map(
"statement" -> "CREATE TABLE",
"colName" -> "`d`",
@@ -189,7 +189,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
ResolveDefaultColumns.analyze(columnE, statementType)
},
- errorClass = "INVALID_DEFAULT_VALUE.DATA_TYPE",
+ condition = "INVALID_DEFAULT_VALUE.DATA_TYPE",
parameters = Map(
"statement" -> "CREATE TABLE",
"colName" -> "`e`",
@@ -589,7 +589,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
sessionCatalog.alterTableDataSchema(
TableIdentifier("t1", Some("default")), StructType(oldTab.dataSchema.drop(1)))
},
- errorClass = "_LEGACY_ERROR_TEMP_1071",
+ condition = "_LEGACY_ERROR_TEMP_1071",
parameters = Map("nonExistentColumnNames" -> "[col1]"))
}
}
@@ -817,14 +817,14 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[NoSuchTableException] {
catalog.getTempViewOrPermanentTableMetadata(TableIdentifier("view1"))
},
- errorClass = "TABLE_OR_VIEW_NOT_FOUND",
+ condition = "TABLE_OR_VIEW_NOT_FOUND",
parameters = Map("relationName" -> "`default`.`view1`")
)
checkError(
exception = intercept[NoSuchTableException] {
catalog.getTempViewOrPermanentTableMetadata(TableIdentifier("view1", Some("default")))
},
- errorClass = "TABLE_OR_VIEW_NOT_FOUND",
+ condition = "TABLE_OR_VIEW_NOT_FOUND",
parameters = Map("relationName" -> "`default`.`view1`")
)
@@ -838,7 +838,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[NoSuchTableException] {
catalog.getTempViewOrPermanentTableMetadata(TableIdentifier("view1", Some("default")))
},
- errorClass = "TABLE_OR_VIEW_NOT_FOUND",
+ condition = "TABLE_OR_VIEW_NOT_FOUND",
parameters = Map("relationName" -> "`default`.`view1`")
)
}
@@ -1000,7 +1000,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
TableIdentifier("tbl2", Some("db2")),
Seq(part1, partWithLessColumns), ignoreIfExists = false)
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a",
"partitionColumnNames" -> "a, b",
@@ -1011,7 +1011,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
TableIdentifier("tbl2", Some("db2")),
Seq(part1, partWithMoreColumns), ignoreIfExists = true)
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a, b, c",
"partitionColumnNames" -> "a, b",
@@ -1022,7 +1022,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
TableIdentifier("tbl2", Some("db2")),
Seq(partWithUnknownColumns, part1), ignoreIfExists = true)
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a, unknown",
"partitionColumnNames" -> "a, b",
@@ -1033,7 +1033,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
TableIdentifier("tbl2", Some("db2")),
Seq(partWithEmptyValue, part1), ignoreIfExists = true)
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> "The spec ([a=3, b=]) contains an empty partition column value"))
}
@@ -1126,7 +1126,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
purge = false,
retainData = false)
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> ("The spec (a, b, c) must be contained within the partition " +
s"spec (a, b) defined in table '`$SESSION_CATALOG_NAME`.`db2`.`tbl2`'")))
@@ -1139,7 +1139,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
purge = false,
retainData = false)
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> ("The spec (a, unknown) must be contained within the partition " +
s"spec (a, b) defined in table '`$SESSION_CATALOG_NAME`.`db2`.`tbl2`'")))
@@ -1152,7 +1152,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
purge = false,
retainData = false)
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> "The spec ([a=3, b=]) contains an empty partition column value"))
}
@@ -1192,7 +1192,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
catalog.getPartition(TableIdentifier("tbl1", Some("db2")), partWithLessColumns.spec)
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a",
"partitionColumnNames" -> "a, b",
@@ -1201,7 +1201,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
catalog.getPartition(TableIdentifier("tbl1", Some("db2")), partWithMoreColumns.spec)
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a, b, c",
"partitionColumnNames" -> "a, b",
@@ -1210,7 +1210,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
catalog.getPartition(TableIdentifier("tbl1", Some("db2")), partWithUnknownColumns.spec)
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a, unknown",
"partitionColumnNames" -> "a, b",
@@ -1219,7 +1219,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
catalog.getPartition(TableIdentifier("tbl1", Some("db2")), partWithEmptyValue.spec)
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> "The spec ([a=3, b=]) contains an empty partition column value"))
}
@@ -1277,7 +1277,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
TableIdentifier("tbl1", Some("db2")),
Seq(part1.spec), Seq(partWithLessColumns.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a",
"partitionColumnNames" -> "a, b",
@@ -1288,7 +1288,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
TableIdentifier("tbl1", Some("db2")),
Seq(part1.spec), Seq(partWithMoreColumns.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a, b, c",
"partitionColumnNames" -> "a, b",
@@ -1299,7 +1299,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
TableIdentifier("tbl1", Some("db2")),
Seq(part1.spec), Seq(partWithUnknownColumns.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a, unknown",
"partitionColumnNames" -> "a, b",
@@ -1310,7 +1310,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
TableIdentifier("tbl1", Some("db2")),
Seq(part1.spec), Seq(partWithEmptyValue.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> "The spec ([a=3, b=]) contains an empty partition column value"))
}
@@ -1364,7 +1364,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
catalog.alterPartitions(TableIdentifier("tbl1", Some("db2")), Seq(partWithLessColumns))
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a",
"partitionColumnNames" -> "a, b",
@@ -1373,7 +1373,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
catalog.alterPartitions(TableIdentifier("tbl1", Some("db2")), Seq(partWithMoreColumns))
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a, b, c",
"partitionColumnNames" -> "a, b",
@@ -1382,7 +1382,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
catalog.alterPartitions(TableIdentifier("tbl1", Some("db2")), Seq(partWithUnknownColumns))
},
- errorClass = "_LEGACY_ERROR_TEMP_1232",
+ condition = "_LEGACY_ERROR_TEMP_1232",
parameters = Map(
"specKeys" -> "a, unknown",
"partitionColumnNames" -> "a, b",
@@ -1391,7 +1391,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[AnalysisException] {
catalog.alterPartitions(TableIdentifier("tbl1", Some("db2")), Seq(partWithEmptyValue))
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> "The spec ([a=3, b=]) contains an empty partition column value"))
}
@@ -1423,7 +1423,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
catalog.listPartitionNames(TableIdentifier("tbl2", Some("db2")),
Some(partWithMoreColumns.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> ("The spec (a, b, c) must be contained within the partition spec (a, b) " +
s"defined in table '`$SESSION_CATALOG_NAME`.`db2`.`tbl2`'")))
@@ -1432,7 +1432,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
catalog.listPartitionNames(TableIdentifier("tbl2", Some("db2")),
Some(partWithUnknownColumns.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> ("The spec (a, unknown) must be contained within the partition " +
s"spec (a, b) defined in table '`$SESSION_CATALOG_NAME`.`db2`.`tbl2`'")))
@@ -1441,7 +1441,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
catalog.listPartitionNames(TableIdentifier("tbl2", Some("db2")),
Some(partWithEmptyValue.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> "The spec ([a=3, b=]) contains an empty partition column value"))
}
@@ -1471,7 +1471,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
catalog.listPartitions(TableIdentifier("tbl2", Some("db2")),
Some(partWithMoreColumns.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> ("The spec (a, b, c) must be contained within the partition spec (a, b) " +
s"defined in table '`$SESSION_CATALOG_NAME`.`db2`.`tbl2`'")))
@@ -1480,7 +1480,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
catalog.listPartitions(TableIdentifier("tbl2", Some("db2")),
Some(partWithUnknownColumns.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> ("The spec (a, unknown) must be contained within the partition " +
s"spec (a, b) defined in table '`$SESSION_CATALOG_NAME`.`db2`.`tbl2`'")))
@@ -1489,7 +1489,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
catalog.listPartitions(TableIdentifier("tbl2", Some("db2")),
Some(partWithEmptyValue.spec))
},
- errorClass = "_LEGACY_ERROR_TEMP_1076",
+ condition = "_LEGACY_ERROR_TEMP_1076",
parameters = Map(
"details" -> "The spec ([a=3, b=]) contains an empty partition column value"))
}
@@ -1582,7 +1582,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
newFunc("temp1", None), overrideIfExists = false, functionBuilder = Some(tempFunc3))
}
checkError(e,
- errorClass = "ROUTINE_ALREADY_EXISTS",
+ condition = "ROUTINE_ALREADY_EXISTS",
parameters = Map("routineName" -> "`temp1`",
"newRoutineType" -> "routine",
"existingRoutineType" -> "routine"))
@@ -1601,7 +1601,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
overrideIfExists = false,
None)
},
- errorClass = "CANNOT_LOAD_FUNCTION_CLASS",
+ condition = "CANNOT_LOAD_FUNCTION_CLASS",
parameters = Map(
"className" -> "function_class_cannot_load",
"functionName" -> "`temp2`"
@@ -1712,14 +1712,14 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[NoSuchFunctionException] {
catalog.lookupFunction(FunctionIdentifier("func1"), arguments)
},
- errorClass = "ROUTINE_NOT_FOUND",
+ condition = "ROUTINE_NOT_FOUND",
parameters = Map("routineName" -> "`default`.`func1`")
)
checkError(
exception = intercept[NoSuchTempFunctionException] {
catalog.dropTempFunction("func1", ignoreIfNotExists = false)
},
- errorClass = "ROUTINE_NOT_FOUND",
+ condition = "ROUTINE_NOT_FOUND",
parameters = Map("routineName" -> "`func1`")
)
catalog.dropTempFunction("func1", ignoreIfNotExists = true)
@@ -1728,7 +1728,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually {
exception = intercept[NoSuchTempFunctionException] {
catalog.dropTempFunction("func2", ignoreIfNotExists = false)
},
- errorClass = "ROUTINE_NOT_FOUND",
+ condition = "ROUTINE_NOT_FOUND",
parameters = Map("routineName" -> "`func2`")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtilsSuite.scala
index 7d9015e566a8c..e8239c7523948 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtilsSuite.scala
@@ -38,7 +38,7 @@ class CSVExprUtilsSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException]{
CSVExprUtils.toDelimiterStr(null)
},
- errorClass = "INVALID_DELIMITER_VALUE.NULL_VALUE",
+ condition = "INVALID_DELIMITER_VALUE.NULL_VALUE",
parameters = Map.empty)
}
@@ -47,7 +47,7 @@ class CSVExprUtilsSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException]{
CSVExprUtils.toChar("ab")
},
- errorClass = "INVALID_DELIMITER_VALUE.DELIMITER_LONGER_THAN_EXPECTED",
+ condition = "INVALID_DELIMITER_VALUE.DELIMITER_LONGER_THAN_EXPECTED",
parameters = Map("str" -> "ab"))
}
@@ -56,7 +56,7 @@ class CSVExprUtilsSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException]{
CSVExprUtils.toChar("""\1""")
},
- errorClass = "INVALID_DELIMITER_VALUE.UNSUPPORTED_SPECIAL_CHARACTER",
+ condition = "INVALID_DELIMITER_VALUE.UNSUPPORTED_SPECIAL_CHARACTER",
parameters = Map("str" -> """\1"""))
}
@@ -65,7 +65,7 @@ class CSVExprUtilsSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException]{
CSVExprUtils.toChar("""\""")
},
- errorClass = "INVALID_DELIMITER_VALUE.SINGLE_BACKSLASH",
+ condition = "INVALID_DELIMITER_VALUE.SINGLE_BACKSLASH",
parameters = Map.empty)
}
@@ -74,7 +74,7 @@ class CSVExprUtilsSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException]{
CSVExprUtils.toChar("")
},
- errorClass = "INVALID_DELIMITER_VALUE.EMPTY_STRING",
+ condition = "INVALID_DELIMITER_VALUE.EMPTY_STRING",
parameters = Map.empty)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
index cbc98d2f23020..514b529ea8cc0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
@@ -308,7 +308,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
check(filters = Seq(EqualTo("invalid attr", 1)), expected = None)
},
- errorClass = "FIELD_NOT_FOUND",
+ condition = "FIELD_NOT_FOUND",
parameters = Map("fieldName" -> "`invalid attr`", "fields" -> "`i`"))
checkError(
@@ -319,7 +319,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
filters = Seq(EqualTo("i", 1)),
expected = Some(InternalRow.empty))
},
- errorClass = "FIELD_NOT_FOUND",
+ condition = "FIELD_NOT_FOUND",
parameters = Map("fieldName" -> "`i`", "fields" -> ""))
}
@@ -374,7 +374,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
check(new UnivocityParser(StructType(Seq.empty), optionsWithPattern(false)))
},
- errorClass = "INVALID_DATETIME_PATTERN.ILLEGAL_CHARACTER",
+ condition = "INVALID_DATETIME_PATTERN.ILLEGAL_CHARACTER",
parameters = Map(
"c" -> "n",
"pattern" -> "invalid"))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderErrorMessageSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderErrorMessageSuite.scala
index e852b474aa18c..b7309923ac206 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderErrorMessageSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderErrorMessageSuite.scala
@@ -55,7 +55,7 @@ class EncoderErrorMessageSuite extends SparkFunSuite {
checkError(
exception = intercept[
SparkUnsupportedOperationException](ExpressionEncoder[ComplexNonEncodable1]()),
- errorClass = "ENCODER_NOT_FOUND",
+ condition = "ENCODER_NOT_FOUND",
parameters = Map(
"typeName" -> "org.apache.spark.sql.catalyst.encoders.NonEncodable",
"docroot" -> SPARK_DOC_ROOT)
@@ -64,7 +64,7 @@ class EncoderErrorMessageSuite extends SparkFunSuite {
checkError(
exception = intercept[
SparkUnsupportedOperationException](ExpressionEncoder[ComplexNonEncodable2]()),
- errorClass = "ENCODER_NOT_FOUND",
+ condition = "ENCODER_NOT_FOUND",
parameters = Map(
"typeName" -> "org.apache.spark.sql.catalyst.encoders.NonEncodable",
"docroot" -> SPARK_DOC_ROOT)
@@ -73,7 +73,7 @@ class EncoderErrorMessageSuite extends SparkFunSuite {
checkError(
exception = intercept[
SparkUnsupportedOperationException](ExpressionEncoder[ComplexNonEncodable3]()),
- errorClass = "ENCODER_NOT_FOUND",
+ condition = "ENCODER_NOT_FOUND",
parameters = Map(
"typeName" -> "org.apache.spark.sql.catalyst.encoders.NonEncodable",
"docroot" -> SPARK_DOC_ROOT)
@@ -82,7 +82,7 @@ class EncoderErrorMessageSuite extends SparkFunSuite {
checkError(
exception = intercept[
SparkUnsupportedOperationException](ExpressionEncoder[ComplexNonEncodable4]()),
- errorClass = "ENCODER_NOT_FOUND",
+ condition = "ENCODER_NOT_FOUND",
parameters = Map(
"typeName" -> "org.apache.spark.sql.catalyst.encoders.NonEncodable",
"docroot" -> SPARK_DOC_ROOT)
@@ -91,7 +91,7 @@ class EncoderErrorMessageSuite extends SparkFunSuite {
checkError(
exception = intercept[
SparkUnsupportedOperationException](ExpressionEncoder[ComplexNonEncodable5]()),
- errorClass = "ENCODER_NOT_FOUND",
+ condition = "ENCODER_NOT_FOUND",
parameters = Map(
"typeName" -> "org.apache.spark.sql.catalyst.encoders.NonEncodable",
"docroot" -> SPARK_DOC_ROOT)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
index e29609c741633..35a27f41da80a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.encoders
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.SparkRuntimeException
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.{AnalysisException, Encoders}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
@@ -71,9 +71,9 @@ class EncoderResolutionSuite extends PlanTest {
}
test("real type doesn't match encoder schema but they are compatible: tupled encoder") {
- val encoder = ExpressionEncoder.tuple(
- ExpressionEncoder[StringLongClass](),
- ExpressionEncoder[Long]())
+ val encoder = encoderFor(Encoders.tuple(
+ Encoders.product[StringLongClass],
+ Encoders.scalaLong))
val attrs = Seq($"a".struct($"a".string, $"b".byte), $"b".int)
testFromRow(encoder, attrs, InternalRow(InternalRow(str, 1.toByte), 2))
}
@@ -90,7 +90,7 @@ class EncoderResolutionSuite extends PlanTest {
val attrs = Seq($"arr".array(StringType))
checkError(
exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "CANNOT_UP_CAST_DATATYPE",
+ condition = "CANNOT_UP_CAST_DATATYPE",
parameters = Map("expression" -> "array element",
"sourceType" -> "\"STRING\"", "targetType" -> "\"BIGINT\"",
"details" -> (
@@ -125,7 +125,7 @@ class EncoderResolutionSuite extends PlanTest {
val attrs = Seq($"arr".int)
checkError(
exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "UNSUPPORTED_DESERIALIZER.DATA_TYPE_MISMATCH",
+ condition = "UNSUPPORTED_DESERIALIZER.DATA_TYPE_MISMATCH",
parameters = Map("desiredType" -> "\"ARRAY\"", "dataType" -> "\"INT\""))
}
@@ -134,7 +134,7 @@ class EncoderResolutionSuite extends PlanTest {
val attrs = Seq($"arr".array(new StructType().add("c", "int")))
checkError(
exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "FIELD_NOT_FOUND",
+ condition = "FIELD_NOT_FOUND",
parameters = Map("fieldName" -> "`a`", "fields" -> "`c`"))
}
@@ -145,7 +145,7 @@ class EncoderResolutionSuite extends PlanTest {
val attrs = Seq($"nestedArr".array(new StructType().add("arr", "int")))
checkError(
exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "UNSUPPORTED_DESERIALIZER.DATA_TYPE_MISMATCH",
+ condition = "UNSUPPORTED_DESERIALIZER.DATA_TYPE_MISMATCH",
parameters = Map("desiredType" -> "\"ARRAY\"", "dataType" -> "\"INT\""))
}
@@ -154,7 +154,7 @@ class EncoderResolutionSuite extends PlanTest {
.add("arr", ArrayType(new StructType().add("c", "int")))))
checkError(
exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "FIELD_NOT_FOUND",
+ condition = "FIELD_NOT_FOUND",
parameters = Map("fieldName" -> "`a`", "fields" -> "`c`"))
}
}
@@ -183,7 +183,7 @@ class EncoderResolutionSuite extends PlanTest {
val attrs = Seq($"a".string, $"b".long, $"c".int)
checkError(
exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
+ condition = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
parameters = Map("schema" -> "\"STRUCT\"",
"ordinal" -> "2"))
}
@@ -192,7 +192,7 @@ class EncoderResolutionSuite extends PlanTest {
val attrs = Seq($"a".string)
checkError(
exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
+ condition = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
parameters = Map("schema" -> "\"STRUCT\"",
"ordinal" -> "2"))
}
@@ -205,7 +205,7 @@ class EncoderResolutionSuite extends PlanTest {
val attrs = Seq($"a".string, $"b".struct($"x".long, $"y".string, $"z".int))
checkError(
exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
+ condition = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
parameters = Map("schema" -> "\"STRUCT\"",
"ordinal" -> "2"))
}
@@ -214,7 +214,7 @@ class EncoderResolutionSuite extends PlanTest {
val attrs = Seq($"a".string, $"b".struct($"x".long))
checkError(
exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
+ condition = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
parameters = Map("schema" -> "\"STRUCT\"",
"ordinal" -> "2"))
}
@@ -233,7 +233,7 @@ class EncoderResolutionSuite extends PlanTest {
.foreach { attr =>
val attrs = Seq(attr)
checkError(exception = intercept[AnalysisException](encoder.resolveAndBind(attrs)),
- errorClass = "CANNOT_UP_CAST_DATATYPE",
+ condition = "CANNOT_UP_CAST_DATATYPE",
parameters = Map("expression" -> "a",
"sourceType" -> ("\"" + attr.dataType.sql + "\""), "targetType" -> "\"STRING\"",
"details" -> (
@@ -250,7 +250,7 @@ class EncoderResolutionSuite extends PlanTest {
ExpressionEncoder[StringIntClass]().resolveAndBind(Seq($"a".string, $"b".long))
}
checkError(exception = e1,
- errorClass = "CANNOT_UP_CAST_DATATYPE",
+ condition = "CANNOT_UP_CAST_DATATYPE",
parameters = Map("expression" -> "b",
"sourceType" -> ("\"BIGINT\""), "targetType" -> "\"INT\"",
"details" -> (
@@ -267,7 +267,7 @@ class EncoderResolutionSuite extends PlanTest {
}
checkError(exception = e2,
- errorClass = "CANNOT_UP_CAST_DATATYPE",
+ condition = "CANNOT_UP_CAST_DATATYPE",
parameters = Map("expression" -> "b.`b`",
"sourceType" -> ("\"DECIMAL(38,18)\""), "targetType" -> "\"BIGINT\"",
"details" -> (
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index 7b8d8be6bbeeb..3b5cbed2cc527 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -321,29 +321,29 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
encodeDecodeTest(
1 -> 10L,
"tuple with 2 flat encoders")(
- ExpressionEncoder.tuple(ExpressionEncoder[Int](), ExpressionEncoder[Long]()))
+ encoderFor(Encoders.tuple(Encoders.scalaInt, Encoders.scalaLong)))
encodeDecodeTest(
(PrimitiveData(1, 1, 1, 1, 1, 1, true), (3, 30L)),
"tuple with 2 product encoders")(
- ExpressionEncoder.tuple(ExpressionEncoder[PrimitiveData](), ExpressionEncoder[(Int, Long)]()))
+ encoderFor(Encoders.tuple(Encoders.product[PrimitiveData], Encoders.product[(Int, Long)])))
encodeDecodeTest(
(PrimitiveData(1, 1, 1, 1, 1, 1, true), 3),
"tuple with flat encoder and product encoder")(
- ExpressionEncoder.tuple(ExpressionEncoder[PrimitiveData](), ExpressionEncoder[Int]()))
+ encoderFor(Encoders.tuple(Encoders.product[PrimitiveData], Encoders.scalaInt)))
encodeDecodeTest(
(3, PrimitiveData(1, 1, 1, 1, 1, 1, true)),
"tuple with product encoder and flat encoder")(
- ExpressionEncoder.tuple(ExpressionEncoder[Int](), ExpressionEncoder[PrimitiveData]()))
+ encoderFor(Encoders.tuple(Encoders.scalaInt, Encoders.product[PrimitiveData])))
encodeDecodeTest(
(1, (10, 100L)),
"nested tuple encoder") {
- val intEnc = ExpressionEncoder[Int]()
- val longEnc = ExpressionEncoder[Long]()
- ExpressionEncoder.tuple(intEnc, ExpressionEncoder.tuple(intEnc, longEnc))
+ val intEnc = Encoders.scalaInt
+ val longEnc = Encoders.scalaLong
+ encoderFor(Encoders.tuple(intEnc, Encoders.tuple(intEnc, longEnc)))
}
// test for value classes
@@ -435,7 +435,7 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
implicitly[ExpressionEncoder[Foo]])
checkError(
exception = exception,
- errorClass = "ENCODER_NOT_FOUND",
+ condition = "ENCODER_NOT_FOUND",
parameters = Map(
"typeName" -> "Any",
"docroot" -> SPARK_DOC_ROOT)
@@ -468,9 +468,8 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
// test for tupled encoders
{
- val schema = ExpressionEncoder.tuple(
- ExpressionEncoder[Int](),
- ExpressionEncoder[(String, Int)]()).schema
+ val encoder = encoderFor(Encoders.tuple(Encoders.scalaInt, Encoders.product[(String, Int)]))
+ val schema = encoder.schema
assert(schema(0).nullable === false)
assert(schema(1).nullable)
assert(schema(1).dataType.asInstanceOf[StructType](0).nullable)
@@ -496,7 +495,7 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
assert(e.getCause.isInstanceOf[SparkRuntimeException])
checkError(
exception = e.getCause.asInstanceOf[SparkRuntimeException],
- errorClass = "NULL_MAP_KEY",
+ condition = "NULL_MAP_KEY",
parameters = Map.empty
)
}
@@ -507,19 +506,19 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
assert(e.getCause.isInstanceOf[SparkRuntimeException])
checkError(
exception = e.getCause.asInstanceOf[SparkRuntimeException],
- errorClass = "NULL_MAP_KEY",
+ condition = "NULL_MAP_KEY",
parameters = Map.empty
)
}
test("throw exception for tuples with more than 22 elements") {
- val encoders = (0 to 22).map(_ => Encoders.scalaInt.asInstanceOf[ExpressionEncoder[_]])
+ val encoders = (0 to 22).map(_ => Encoders.scalaInt)
checkError(
exception = intercept[SparkUnsupportedOperationException] {
- ExpressionEncoder.tuple(encoders)
+ Encoders.tupleEncoder(encoders: _*)
},
- errorClass = "_LEGACY_ERROR_TEMP_2150",
+ condition = "_LEGACY_ERROR_TEMP_2150",
parameters = Map.empty)
}
@@ -531,11 +530,11 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
val encoder = ExpressionEncoder(schema, lenient = true)
val unexpectedSerializer = NaNvl(encoder.objSerializer, encoder.objSerializer)
val exception = intercept[org.apache.spark.SparkRuntimeException] {
- new ExpressionEncoder[Row](unexpectedSerializer, encoder.objDeserializer, encoder.clsTag)
+ new ExpressionEncoder[Row](encoder.encoder, unexpectedSerializer, encoder.objDeserializer)
}
checkError(
exception = exception,
- errorClass = "UNEXPECTED_SERIALIZER_FOR_CLASS",
+ condition = "UNEXPECTED_SERIALIZER_FOR_CLASS",
parameters = Map(
"className" -> Utils.getSimpleName(encoder.clsTag.runtimeClass),
"expr" -> toSQLExpr(unexpectedSerializer))
@@ -552,11 +551,14 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
encodeDecodeTest(FooClassWithEnum(1, FooEnum.E1), "case class with Int and scala Enum")
encodeDecodeTest(FooEnum.E1, "scala Enum")
- test("transforming encoder") {
+
+ private def testTransformingEncoder(
+ name: String,
+ provider: () => Codec[Any, Array[Byte]]): Unit = test(name) {
val encoder = ExpressionEncoder(TransformingEncoder(
classTag[(Long, Long)],
BinaryEncoder,
- JavaSerializationCodec))
+ provider))
.resolveAndBind()
assert(encoder.schema == new StructType().add("value", BinaryType))
val toRow = encoder.createSerializer()
@@ -564,6 +566,9 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
assert(fromRow(toRow((11, 14))) == (11, 14))
}
+ testTransformingEncoder("transforming java serialization encoder", JavaSerializationCodec)
+ testTransformingEncoder("transforming kryo encoder", KryoSerializationCodec)
+
// Scala / Java big decimals ----------------------------------------------------------
encodeDecodeTest(BigDecimal(("9" * 20) + "." + "9" * 18),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala
index 71fa60b0c0345..29c5bf3b8d2db 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala
@@ -69,7 +69,7 @@ class AttributeResolutionSuite extends SparkFunSuite {
exception = intercept[AnalysisException] {
attrs.resolve(Seq("a"), resolver)
},
- errorClass = "AMBIGUOUS_REFERENCE",
+ condition = "AMBIGUOUS_REFERENCE",
parameters = Map(
"name" -> "`a`",
"referenceNames" -> "[`ns1`.`ns2`.`t2`.`a`, `ns1`.`t1`.`a`]"
@@ -86,7 +86,7 @@ class AttributeResolutionSuite extends SparkFunSuite {
exception = intercept[AnalysisException] {
attrs.resolve(Seq("ns1", "t", "a"), resolver)
},
- errorClass = "AMBIGUOUS_REFERENCE",
+ condition = "AMBIGUOUS_REFERENCE",
parameters = Map(
"name" -> "`ns1`.`t`.`a`",
"referenceNames" -> "[`ns1`.`t`.`a`, `ns2`.`ns1`.`t`.`a`]"
@@ -108,7 +108,7 @@ class AttributeResolutionSuite extends SparkFunSuite {
exception = intercept[AnalysisException] {
attrs.resolve(Seq("ns1", "t", "a", "cc"), resolver)
},
- errorClass = "FIELD_NOT_FOUND",
+ condition = "FIELD_NOT_FOUND",
parameters = Map("fieldName" -> "`cc`", "fields" -> "`aa`, `bb`"))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
index 77fdb33e515fc..995b519bd05d7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
@@ -103,7 +103,7 @@ class CallMethodViaReflectionSuite extends SparkFunSuite with ExpressionEvalHelp
exception = intercept[AnalysisException] {
CallMethodViaReflection(Seq.empty).checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`reflect`",
"expectedNum" -> "> 1",
@@ -114,7 +114,7 @@ class CallMethodViaReflectionSuite extends SparkFunSuite with ExpressionEvalHelp
exception = intercept[AnalysisException] {
CallMethodViaReflection(Seq(Literal(staticClassName))).checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`reflect`",
"expectedNum" -> "> 1",
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
index 67a68fc92a300..e87b54339821f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
@@ -1107,8 +1107,10 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
.foreach { interval =>
checkErrorInExpression[SparkIllegalArgumentException](
cast(Literal.create(interval), YearMonthIntervalType()),
- "_LEGACY_ERROR_TEMP_3213",
- Map("interval" -> "year-month", "msg" -> "integer overflow"))
+ "INVALID_INTERVAL_FORMAT.INTERVAL_PARSING",
+ Map(
+ "interval" -> "year-month",
+ "input" -> interval))
}
Seq(Byte.MaxValue, Short.MaxValue, Int.MaxValue, Int.MinValue + 1, Int.MinValue)
@@ -1176,9 +1178,8 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
val dataType = YearMonthIntervalType()
checkErrorInExpression[SparkIllegalArgumentException](
cast(Literal.create(interval), dataType),
- "_LEGACY_ERROR_TEMP_3214",
+ "INVALID_INTERVAL_FORMAT.UNMATCHED_FORMAT_STRING",
Map(
- "fallBackNotice" -> "",
"typeName" -> "interval year to month",
"intervalStr" -> "year-month",
"supportedFormat" -> "`[+|-]y-m`, `INTERVAL [+|-]'[+|-]y-m' YEAR TO MONTH`",
@@ -1198,9 +1199,8 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
.foreach { case (interval, dataType) =>
checkErrorInExpression[SparkIllegalArgumentException](
cast(Literal.create(interval), dataType),
- "_LEGACY_ERROR_TEMP_3214",
+ "INVALID_INTERVAL_FORMAT.UNMATCHED_FORMAT_STRING",
Map(
- "fallBackNotice" -> "",
"typeName" -> dataType.typeName,
"intervalStr" -> "year-month",
"supportedFormat" ->
@@ -1322,10 +1322,8 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
.foreach { case (interval, dataType) =>
checkErrorInExpression[SparkIllegalArgumentException](
cast(Literal.create(interval), dataType),
- "_LEGACY_ERROR_TEMP_3214",
- Map("fallBackNotice" -> (", set spark.sql.legacy.fromDayTimeString.enabled" +
- " to true to restore the behavior before Spark 3.0."),
- "intervalStr" -> "day-time",
+ "INVALID_INTERVAL_FORMAT.UNMATCHED_FORMAT_STRING_WITH_NOTICE",
+ Map("intervalStr" -> "day-time",
"typeName" -> dataType.typeName,
"input" -> interval,
"supportedFormat" ->
@@ -1348,10 +1346,8 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
.foreach { case (interval, dataType) =>
checkErrorInExpression[SparkIllegalArgumentException](
cast(Literal.create(interval), dataType),
- "_LEGACY_ERROR_TEMP_3214",
- Map("fallBackNotice" -> (", set spark.sql.legacy.fromDayTimeString.enabled" +
- " to true to restore the behavior before Spark 3.0."),
- "intervalStr" -> "day-time",
+ "INVALID_INTERVAL_FORMAT.UNMATCHED_FORMAT_STRING_WITH_NOTICE",
+ Map("intervalStr" -> "day-time",
"typeName" -> dataType.typeName,
"input" -> interval,
"supportedFormat" ->
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollationExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollationExpressionSuite.scala
index 175dd05d5911e..e34b54c7086cd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollationExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollationExpressionSuite.scala
@@ -61,7 +61,7 @@ class CollationExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
test("collate on non existing collation") {
checkError(
exception = intercept[SparkException] { Collate(Literal("abc"), "UTF8_BS") },
- errorClass = "COLLATION_INVALID_NAME",
+ condition = "COLLATION_INVALID_NAME",
sqlState = "42704",
parameters = Map("collationName" -> "UTF8_BS", "proposals" -> "UTF8_LCASE"))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index c7e995feb5ed8..55148978fa005 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -95,7 +95,7 @@ class CollectionExpressionsSuite
}
checkError(
exception = exception,
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map(
"message" -> ("The size function doesn't support the operand type " +
toSQLType(StringType))
@@ -266,7 +266,7 @@ class CollectionExpressionsSuite
checkErrorInExpression[SparkRuntimeException](
MapConcat(Seq(m0, m1)),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "a",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -426,7 +426,7 @@ class CollectionExpressionsSuite
checkErrorInExpression[SparkRuntimeException](
MapFromEntries(ai4),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "1",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -458,7 +458,7 @@ class CollectionExpressionsSuite
checkErrorInExpression[SparkRuntimeException](
MapFromEntries(as4),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "a",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -720,7 +720,7 @@ class CollectionExpressionsSuite
checkEvaluation(Slice(a0, Literal(-1), Literal(2)), Seq(6))
checkErrorInExpression[SparkRuntimeException](
expression = Slice(a0, Literal(1), Literal(-1)),
- errorClass = "INVALID_PARAMETER_VALUE.LENGTH",
+ condition = "INVALID_PARAMETER_VALUE.LENGTH",
parameters = Map(
"parameter" -> toSQLId("length"),
"length" -> (-1).toString,
@@ -728,7 +728,7 @@ class CollectionExpressionsSuite
))
checkErrorInExpression[SparkRuntimeException](
expression = Slice(a0, Literal(0), Literal(1)),
- errorClass = "INVALID_PARAMETER_VALUE.START",
+ condition = "INVALID_PARAMETER_VALUE.START",
parameters = Map(
"parameter" -> toSQLId("start"),
"functionName" -> toSQLId("slice")
@@ -910,7 +910,7 @@ class CollectionExpressionsSuite
// SPARK-43393: test Sequence overflow checking
checkErrorInExpression[SparkRuntimeException](
new Sequence(Literal(Int.MinValue), Literal(Int.MaxValue), Literal(1)),
- errorClass = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
+ condition = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
parameters = Map(
"numberOfElements" -> (BigInt(Int.MaxValue) - BigInt { Int.MinValue } + 1).toString,
"functionName" -> toSQLId("sequence"),
@@ -918,7 +918,7 @@ class CollectionExpressionsSuite
"parameter" -> toSQLId("count")))
checkErrorInExpression[SparkRuntimeException](
new Sequence(Literal(0L), Literal(Long.MaxValue), Literal(1L)),
- errorClass = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
+ condition = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
parameters = Map(
"numberOfElements" -> (BigInt(Long.MaxValue) + 1).toString,
"functionName" -> toSQLId("sequence"),
@@ -926,7 +926,7 @@ class CollectionExpressionsSuite
"parameter" -> toSQLId("count")))
checkErrorInExpression[SparkRuntimeException](
new Sequence(Literal(0L), Literal(Long.MinValue), Literal(-1L)),
- errorClass = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
+ condition = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
parameters = Map(
"numberOfElements" -> ((0 - BigInt(Long.MinValue)) + 1).toString(),
"functionName" -> toSQLId("sequence"),
@@ -934,7 +934,7 @@ class CollectionExpressionsSuite
"parameter" -> toSQLId("count")))
checkErrorInExpression[SparkRuntimeException](
new Sequence(Literal(Long.MinValue), Literal(Long.MaxValue), Literal(1L)),
- errorClass = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
+ condition = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
parameters = Map(
"numberOfElements" -> (BigInt(Long.MaxValue) - BigInt { Long.MinValue } + 1).toString,
"functionName" -> toSQLId("sequence"),
@@ -942,7 +942,7 @@ class CollectionExpressionsSuite
"parameter" -> toSQLId("count")))
checkErrorInExpression[SparkRuntimeException](
new Sequence(Literal(Long.MaxValue), Literal(Long.MinValue), Literal(-1L)),
- errorClass = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
+ condition = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
parameters = Map(
"numberOfElements" -> (BigInt(Long.MaxValue) - BigInt { Long.MinValue } + 1).toString,
"functionName" -> toSQLId("sequence"),
@@ -950,7 +950,7 @@ class CollectionExpressionsSuite
"parameter" -> toSQLId("count")))
checkErrorInExpression[SparkRuntimeException](
new Sequence(Literal(Long.MaxValue), Literal(-1L), Literal(-1L)),
- errorClass = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
+ condition = "COLLECTION_SIZE_LIMIT_EXCEEDED.PARAMETER",
parameters = Map(
"numberOfElements" -> (BigInt(Long.MaxValue) - BigInt { -1L } + 1).toString,
"functionName" -> toSQLId("sequence"),
@@ -2293,6 +2293,14 @@ class CollectionExpressionsSuite
evaluateWithMutableProjection(Shuffle(ai0, seed2)))
assert(evaluateWithUnsafeProjection(Shuffle(ai0, seed1)) !==
evaluateWithUnsafeProjection(Shuffle(ai0, seed2)))
+
+ val seed3 = Literal.create(r.nextInt())
+ assert(evaluateWithoutCodegen(new Shuffle(ai0, seed3)) ===
+ evaluateWithoutCodegen(new Shuffle(ai0, seed3)))
+ assert(evaluateWithMutableProjection(new Shuffle(ai0, seed3)) ===
+ evaluateWithMutableProjection(new Shuffle(ai0, seed3)))
+ assert(evaluateWithUnsafeProjection(new Shuffle(ai0, seed3)) ===
+ evaluateWithUnsafeProjection(new Shuffle(ai0, seed3)))
}
test("Array Except") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
index 497b335289b11..7baad5ea92a00 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
@@ -285,7 +285,7 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
checkErrorInExpression[SparkRuntimeException](
CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "1",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -328,7 +328,7 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
map3.checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`map`",
"expectedNum" -> "2n (n > 0)",
@@ -430,7 +430,7 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
MapFromArrays(
Literal.create(Seq(1, 1), ArrayType(IntegerType)),
Literal.create(Seq(2, 3), ArrayType(IntegerType))),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "1",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -483,7 +483,7 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
namedStruct1.checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`named_struct`",
"expectedNum" -> "2n (n > 0)",
@@ -556,7 +556,7 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
checkErrorInExpression[SparkRuntimeException](
new StringToMap(Literal("a:1,b:2,a:3")),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "a",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
index 7ffb321217024..f4c71a1056939 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
import org.apache.spark.sql.types._
@@ -277,4 +278,27 @@ class ConditionalExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
assert(!caseWhenObj1.semanticEquals(caseWhenObj2))
assert(!caseWhenObj2.semanticEquals(caseWhenObj1))
}
+
+ test("SPARK-49396 accurate nullability check") {
+ val trueBranch = (TrueLiteral, Literal(5))
+ val normalBranch = (NonFoldableLiteral(true), Literal(10))
+
+ val nullLiteral = Literal.create(null, BooleanType)
+ val noElseValue = CaseWhen(normalBranch :: trueBranch :: Nil, None)
+ assert(!noElseValue.nullable)
+ val withElseValue = CaseWhen(normalBranch :: trueBranch :: Nil, Some(Literal(1)))
+ assert(!withElseValue.nullable)
+ val withNullableElseValue = CaseWhen(normalBranch :: trueBranch :: Nil, Some(nullLiteral))
+ assert(!withNullableElseValue.nullable)
+ val firstTrueNonNullableSecondTrueNullable = CaseWhen(trueBranch ::
+ (TrueLiteral, nullLiteral) :: Nil, None)
+ assert(!firstTrueNonNullableSecondTrueNullable.nullable)
+ val firstTrueNullableSecondTrueNonNullable = CaseWhen((TrueLiteral, nullLiteral) ::
+ trueBranch :: Nil, None)
+ assert(firstTrueNullableSecondTrueNonNullable.nullable)
+ val hasNullInNotTrueBranch = CaseWhen(trueBranch :: (FalseLiteral, nullLiteral) :: Nil, None)
+ assert(!hasNullInNotTrueBranch.nullable)
+ val noTrueBranch = CaseWhen(normalBranch :: Nil, Literal(1))
+ assert(!noTrueBranch.nullable)
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index f1c04c7e33821..21ae35146282b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -2033,12 +2033,12 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkErrorInExpression[SparkArithmeticException](TimestampAdd("DAY",
Literal(106751992),
Literal(0L, TimestampType)),
- errorClass = "DATETIME_OVERFLOW",
+ condition = "DATETIME_OVERFLOW",
parameters = Map("operation" -> "add 106751992 DAY to TIMESTAMP '1970-01-01 00:00:00'"))
checkErrorInExpression[SparkArithmeticException](TimestampAdd("QUARTER",
Literal(1431655764),
Literal(0L, TimestampType)),
- errorClass = "DATETIME_OVERFLOW",
+ condition = "DATETIME_OVERFLOW",
parameters = Map("operation" ->
"add 1431655764 QUARTER to TIMESTAMP '1970-01-01 00:00:00'"))
}
@@ -2103,11 +2103,11 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
test("datetime function CurrentDate and localtimestamp are Unevaluable") {
checkError(exception = intercept[SparkException] { CurrentDate(UTC_OPT).eval(EmptyRow) },
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> "Cannot evaluate expression: current_date(Some(UTC))"))
checkError(exception = intercept[SparkException] { LocalTimestamp(UTC_OPT).eval(EmptyRow) },
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> "Cannot evaluate expression: localtimestamp(Some(UTC))"))
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
index a063e53486ad8..02c7ed727a530 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
@@ -154,22 +154,22 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
protected def checkErrorInExpression[T <: SparkThrowable : ClassTag](
expression: => Expression,
- errorClass: String,
+ condition: String,
parameters: Map[String, String] = Map.empty): Unit = {
- checkErrorInExpression[T](expression, InternalRow.empty, errorClass, parameters)
+ checkErrorInExpression[T](expression, InternalRow.empty, condition, parameters)
}
protected def checkErrorInExpression[T <: SparkThrowable : ClassTag](
expression: => Expression,
inputRow: InternalRow,
- errorClass: String): Unit = {
- checkErrorInExpression[T](expression, inputRow, errorClass, Map.empty[String, String])
+ condition: String): Unit = {
+ checkErrorInExpression[T](expression, inputRow, condition, Map.empty[String, String])
}
protected def checkErrorInExpression[T <: SparkThrowable : ClassTag](
expression: => Expression,
inputRow: InternalRow,
- errorClass: String,
+ condition: String,
parameters: Map[String, String]): Unit = {
def checkException(eval: => Unit, testMode: String): Unit = {
@@ -179,7 +179,7 @@ trait ExpressionEvalHelper extends ScalaCheckDrivenPropertyChecks with PlanTestB
withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> fallbackMode.toString) {
checkError(
exception = intercept[T](eval),
- errorClass = errorClass,
+ condition = condition,
parameters = parameters
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/GeneratorExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/GeneratorExpressionSuite.scala
index dd512b0d83e5c..b6a3d61cb13a6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/GeneratorExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/GeneratorExpressionSuite.scala
@@ -82,7 +82,7 @@ class GeneratorExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
Stack(Seq(Literal(1))).checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`stack`",
"expectedNum" -> "> 1",
@@ -93,7 +93,7 @@ class GeneratorExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
Stack(Seq(Literal(1.0))).checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`stack`",
"expectedNum" -> "> 1",
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
index c06705606567d..cc36cd73d6d77 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
@@ -471,7 +471,7 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
checkErrorInExpression[SparkRuntimeException](
transformKeys(ai0, modKey),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "1",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -858,7 +858,7 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
SQLConf.LEGACY_ALLOW_NULL_COMPARISON_RESULT_IN_ARRAY_SORT.key -> "false") {
checkErrorInExpression[SparkException](
expression = arraySort(Literal.create(Seq(3, 1, 1, 2)), comparator),
- errorClass = "COMPARATOR_RETURNS_NULL",
+ condition = "COMPARATOR_RETURNS_NULL",
parameters = Map("firstValue" -> "1", "secondValue" -> "1")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
index ff5ffe4e869a0..7caf23490a0ce 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
@@ -351,7 +351,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
Seq(
(Period.ofMonths(2), Int.MaxValue) -> "overflow",
- (Period.ofMonths(Int.MinValue), 10d) -> "not in range",
+ (Period.ofMonths(Int.MinValue), 10d) -> "out of range",
(Period.ofMonths(-100), Float.NaN) -> "input is infinite or NaN",
(Period.ofMonths(200), Double.PositiveInfinity) -> "input is infinite or NaN",
(Period.ofMonths(-200), Float.NegativeInfinity) -> "input is infinite or NaN"
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index a23e7f44a48d1..adb39fcd568c9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -448,7 +448,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
}.getCause
checkError(
exception = exception.asInstanceOf[SparkException],
- errorClass = "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION",
+ condition = "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION",
parameters = Map("badRecord" -> "[null]", "failFastMode" -> "FAILFAST")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index d42e0b7d681db..b351d69d3a0bb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -97,7 +97,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[SparkException] {
Literal.default(errType)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> s"No default value for type: ${toSQLType(errType)}.")
)
})
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
index 1f37886f44258..40e6fe1a90a63 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
@@ -71,6 +71,13 @@ class MiscExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
evaluateWithMutableProjection(Uuid(seed2)))
assert(evaluateWithUnsafeProjection(Uuid(seed1)) !==
evaluateWithUnsafeProjection(Uuid(seed2)))
+
+ val seed3 = Literal.create(r.nextInt())
+ assert(evaluateWithoutCodegen(new Uuid(seed3)) === evaluateWithoutCodegen(new Uuid(seed3)))
+ assert(evaluateWithMutableProjection(new Uuid(seed3)) ===
+ evaluateWithMutableProjection(new Uuid(seed3)))
+ assert(evaluateWithUnsafeProjection(new Uuid(seed3)) ===
+ evaluateWithUnsafeProjection(new Uuid(seed3)))
}
test("PrintToStderr") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullExpressionsSuite.scala
index ace017b1cddc3..c74a9e35833d1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NullExpressionsSuite.scala
@@ -57,7 +57,7 @@ class NullExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[SparkRuntimeException] {
evaluateWithoutCodegen(AssertNotNull(Literal(null)))
},
- errorClass = "NOT_NULL_ASSERT_VIOLATION",
+ condition = "NOT_NULL_ASSERT_VIOLATION",
sqlState = "42000",
parameters = Map("walkedTypePath" -> "\n\n"))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
index af38fc1f12f7f..762a4e9166d51 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
@@ -82,7 +82,7 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[SparkException] {
Invoke(inputObject, "zeroArgNotExistMethod", IntegerType).eval(inputRow)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" ->
("Couldn't find method zeroArgNotExistMethod with arguments " +
"() on class java.lang.Object.")
@@ -98,7 +98,7 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
Seq(Literal.fromObject(UTF8String.fromString("dummyInputString"))),
Seq(StringType)).eval(inputRow)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" ->
("Couldn't find method oneArgNotExistMethod with arguments " +
"(class org.apache.spark.unsafe.types.UTF8String) on class java.lang.Object.")
@@ -417,7 +417,7 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[SparkRuntimeException] {
testMapObjects(collection, classOf[scala.collection.Map[Int, Int]], inputType)
},
- errorClass = "CLASS_UNSUPPORTED_BY_MAP_OBJECTS",
+ condition = "CLASS_UNSUPPORTED_BY_MAP_OBJECTS",
parameters = Map("cls" -> "scala.collection.Map"))
}
}
@@ -588,7 +588,7 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
DoubleType,
DoubleType),
inputRow = InternalRow.fromSeq(Seq(Row(1))),
- errorClass = "INVALID_EXTERNAL_TYPE",
+ condition = "INVALID_EXTERNAL_TYPE",
parameters = Map[String, String](
"externalType" -> "java.lang.Integer",
"type" -> "\"DOUBLE\"",
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RandomSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RandomSuite.scala
index 2aa53f581555f..2d58d9d3136aa 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RandomSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RandomSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.CatalystTypeConverters
import org.apache.spark.sql.types.{IntegerType, LongType}
class RandomSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -41,4 +42,27 @@ class RandomSuite extends SparkFunSuite with ExpressionEvalHelper {
assert(Rand(Literal(1L), false).sql === "rand(1L)")
assert(Randn(Literal(1L), false).sql === "randn(1L)")
}
+
+ test("SPARK-49505: Test the RANDSTR and UNIFORM SQL functions without codegen") {
+ // Note that we use a seed of zero in these tests to keep the results deterministic.
+ def testRandStr(first: Any, result: Any): Unit = {
+ checkEvaluationWithoutCodegen(
+ RandStr(Literal(first), Literal(0)), CatalystTypeConverters.convertToCatalyst(result))
+ }
+ testRandStr(1, "c")
+ testRandStr(5, "ceV0P")
+ testRandStr(10, "ceV0PXaR2I")
+ testRandStr(10L, "ceV0PXaR2I")
+
+ def testUniform(first: Any, second: Any, result: Any): Unit = {
+ checkEvaluationWithoutCodegen(
+ Uniform(Literal(first), Literal(second), Literal(0)).replacement,
+ CatalystTypeConverters.convertToCatalyst(result))
+ }
+ testUniform(0, 1, 0)
+ testUniform(0, 10, 7)
+ testUniform(0L, 10L, 7L)
+ testUniform(10.0F, 20.0F, 17.604954F)
+ testUniform(10L, 20.0F, 17.604954F)
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
index 690db18bbfa69..12aeb7d6685bf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
@@ -158,13 +158,13 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
evaluateWithoutCodegen("""a""" like """\a""")
},
- errorClass = "INVALID_FORMAT.ESC_IN_THE_MIDDLE",
+ condition = "INVALID_FORMAT.ESC_IN_THE_MIDDLE",
parameters = Map("format" -> """'\\a'""", "char" -> "'a'"))
checkError(
exception = intercept[AnalysisException] {
evaluateWithoutCodegen("""a""" like """a\""")
},
- errorClass = "INVALID_FORMAT.ESC_AT_THE_END",
+ condition = "INVALID_FORMAT.ESC_AT_THE_END",
parameters = Map("format" -> """'a\\'"""))
// case
@@ -238,7 +238,7 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
evaluateWithoutCodegen("""a""" like(s"""${escapeChar}a""", escapeChar))
},
- errorClass = "INVALID_FORMAT.ESC_IN_THE_MIDDLE",
+ condition = "INVALID_FORMAT.ESC_IN_THE_MIDDLE",
parameters = Map("format" -> s"'${escapeChar}a'", "char" -> "'a'"))
// case
@@ -283,7 +283,7 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[SparkRuntimeException] {
evaluateWithoutCodegen("abbbbc" rlike "**")
},
- errorClass = "INVALID_PARAMETER_VALUE.PATTERN",
+ condition = "INVALID_PARAMETER_VALUE.PATTERN",
parameters = Map(
"parameter" -> toSQLId("regexp"),
"functionName" -> toSQLId("rlike"),
@@ -294,7 +294,7 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val regex = $"a".string.at(0)
evaluateWithoutCodegen("abbbbc" rlike regex, create_row("**"))
},
- errorClass = "INVALID_PARAMETER_VALUE.PATTERN",
+ condition = "INVALID_PARAMETER_VALUE.PATTERN",
parameters = Map(
"parameter" -> toSQLId("regexp"),
"functionName" -> toSQLId("rlike"),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index b39820f0d317d..29b878230472d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -155,7 +155,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
Elt(Seq.empty).checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`elt`",
"expectedNum" -> "> 1",
@@ -166,7 +166,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
Elt(Seq(Literal(1))).checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`elt`",
"expectedNum" -> "> 1",
@@ -1505,7 +1505,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkErrorInExpression[SparkIllegalArgumentException](
toNumberExpr,
- errorClass = "INVALID_FORMAT.MISMATCH_INPUT",
+ condition = "INVALID_FORMAT.MISMATCH_INPUT",
parameters = Map(
"inputType" -> "\"STRING\"",
"input" -> str,
@@ -1910,7 +1910,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
ParseUrl(Seq(Literal("1"))).checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`parse_url`",
"expectedNum" -> "[2, 3]",
@@ -1922,7 +1922,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
ParseUrl(Seq(Literal("1"), Literal("2"), Literal("3"),
Literal("4"))).checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`parse_url`",
"expectedNum" -> "[2, 3]",
@@ -1987,7 +1987,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
// Test escaping of arguments
GenerateUnsafeProjection.generate(
- Sentences(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil)
+ Sentences(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")).replacement :: Nil)
}
test("SPARK-33386: elt ArrayIndexOutOfBoundsException") {
@@ -2037,7 +2037,7 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
exception = intercept[AnalysisException] {
expr1.checkInputDataTypes()
},
- errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ condition = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
parameters = Map(
"functionName" -> "`elt`",
"expectedNum" -> "> 1",
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryCastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryCastSuite.scala
index f138d9642d1e1..446514de91d69 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryCastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TryCastSuite.scala
@@ -47,7 +47,7 @@ class TryCastSuite extends CastWithAnsiOnSuite {
override def checkErrorInExpression[T <: SparkThrowable : ClassTag](
expression: => Expression,
inputRow: InternalRow,
- errorClass: String,
+ condition: String,
parameters: Map[String, String]): Unit = {
checkEvaluation(expression, null, inputRow)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XmlExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XmlExpressionsSuite.scala
index 9a10985153044..66baf6b1430fa 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XmlExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/XmlExpressionsSuite.scala
@@ -64,7 +64,7 @@ class XmlExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
}.getCause
checkError(
exception = exception.asInstanceOf[SparkException],
- errorClass = "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION",
+ condition = "MALFORMED_RECORD_IN_PARSING.WITHOUT_SUGGESTION",
parameters = Map("badRecord" -> "[null]", "failFastMode" -> "FAILFAST")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala
index 389c757eefb63..f1d9fd6a36584 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/PercentileSuite.scala
@@ -410,7 +410,7 @@ class PercentileSuite extends SparkFunSuite {
agg.update(buffer, InternalRow(1, -5))
agg.eval(buffer)
},
- errorClass = "_LEGACY_ERROR_TEMP_2013",
+ condition = "_LEGACY_ERROR_TEMP_2013",
parameters = Map("frequencyExpression" -> "CAST(boundreference() AS INT)"))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSuite.scala
index 79f03f23eb245..ca2eaf7be0c21 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSuite.scala
@@ -27,7 +27,7 @@ class BufferHolderSuite extends SparkFunSuite {
exception = intercept[SparkUnsupportedOperationException] {
new BufferHolder(new UnsafeRow(Int.MaxValue / 8))
},
- errorClass = "_LEGACY_ERROR_TEMP_3130",
+ condition = "_LEGACY_ERROR_TEMP_3130",
parameters = Map("numFields" -> "268435455"))
val holder = new BufferHolder(new UnsafeRow(1000))
@@ -38,7 +38,7 @@ class BufferHolderSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
holder.grow(Integer.MAX_VALUE)
},
- errorClass = "_LEGACY_ERROR_TEMP_3199",
+ condition = "_LEGACY_ERROR_TEMP_3199",
parameters = Map("neededSize" -> "2147483647", "arrayMax" -> "2147483632")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
index d51647ee96df9..4f81ef49e5736 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
@@ -121,7 +121,7 @@ class CodeBlockSuite extends SparkFunSuite {
exception = intercept[SparkException] {
code"$obj"
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> s"Can not interpolate ${obj.getClass.getName} into code block.")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriterSuite.scala
index a968b1fe53506..8a8f0afeb1224 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/UnsafeArrayWriterSuite.scala
@@ -30,7 +30,7 @@ class UnsafeArrayWriterSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
arrayWriter.initialize(numElements)
},
- errorClass = "COLLECTION_SIZE_LIMIT_EXCEEDED.INITIALIZE",
+ condition = "COLLECTION_SIZE_LIMIT_EXCEEDED.INITIALIZE",
parameters = Map(
"numberOfElements" -> (numElements * elementSize).toString,
"maxRoundedArrayLength" -> Int.MaxValue.toString
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtilsSuite.scala
index 2b8c64a1af679..f599fead45015 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtilsSuite.scala
@@ -116,7 +116,7 @@ class VariantExpressionEvalUtilsSuite extends SparkFunSuite {
}
test("parseJson negative") {
- def checkException(json: String, errorClass: String, parameters: Map[String, String]): Unit = {
+ def checkException(json: String, condition: String, parameters: Map[String, String]): Unit = {
val try_parse_json_output = VariantExpressionEvalUtils.parseJson(UTF8String.fromString(json),
allowDuplicateKeys = false, failOnError = false)
checkError(
@@ -124,7 +124,7 @@ class VariantExpressionEvalUtilsSuite extends SparkFunSuite {
VariantExpressionEvalUtils.parseJson(UTF8String.fromString(json),
allowDuplicateKeys = false)
},
- errorClass = errorClass,
+ condition = condition,
parameters = parameters
)
assert(try_parse_json_output === null)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index fb0bf63c01123..59392548a6ae2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -45,12 +45,14 @@ class VariantExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
}
test("to_json malformed") {
- def check(value: Array[Byte], metadata: Array[Byte],
- errorClass: String = "MALFORMED_VARIANT"): Unit = {
+ def check(
+ value: Array[Byte],
+ metadata: Array[Byte],
+ condition: String = "MALFORMED_VARIANT"): Unit = {
checkErrorInExpression[SparkRuntimeException](
ResolveTimeZone.resolveTimeZones(
StructsToJson(Map.empty, Literal(new VariantVal(value, metadata)))),
- errorClass
+ condition
)
}
@@ -949,12 +951,24 @@ class VariantExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
)
}
- test("cast to variant") {
- def check[T : TypeTag](input: T, expectedJson: String): Unit = {
- val cast = Cast(Literal.create(input), VariantType, evalMode = EvalMode.ANSI)
- checkEvaluation(StructsToJson(Map.empty, cast), expectedJson)
+ test("cast to variant/to_variant_object") {
+ def check[T : TypeTag](input: T, expectedJson: String,
+ toVariantObject: Boolean = false): Unit = {
+ val expr =
+ if (toVariantObject) ToVariantObject(Literal.create(input))
+ else Cast(Literal.create(input), VariantType, evalMode = EvalMode.ANSI)
+ checkEvaluation(StructsToJson(Map.empty, expr), expectedJson)
}
+ def checkFailure[T: TypeTag](input: T, toVariantObject: Boolean = false): Unit = {
+ val expr =
+ if (toVariantObject) ToVariantObject(Literal.create(input))
+ else Cast(Literal.create(input), VariantType, evalMode = EvalMode.ANSI)
+ val resolvedExpr = ResolveTimeZone.resolveTimeZones(expr)
+ assert(!resolvedExpr.resolved)
+ }
+
+ // cast to variant - success cases
check(null.asInstanceOf[String], null)
// The following tests cover all allowed scalar types.
for (input <- Seq[Any](false, true, 0.toByte, 1.toShort, 2, 3L, 4.0F, 5.0D)) {
@@ -1023,17 +1037,52 @@ class VariantExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
}
check(Array(null, "a", "b", "c"), """[null,"a","b","c"]""")
- check(Map("z" -> 1, "y" -> 2, "x" -> 3), """{"x":3,"y":2,"z":1}""")
check(Array(parseJson("""{"a": 1,"b": [1, 2, 3]}"""),
parseJson("""{"c": true,"d": {"e": "str"}}""")),
"""[{"a":1,"b":[1,2,3]},{"c":true,"d":{"e":"str"}}]""")
- val struct = Literal.create(
+
+ // cast to variant - failure cases - struct and map types
+ val mp = Map("z" -> 1, "y" -> 2, "x" -> 3)
+ val arrayMp = Array(Map("z" -> 1, "y" -> 2, "x" -> 3))
+ val arrayArrayMp = Array(Array(Map("z" -> 1, "y" -> 2, "x" -> 3)))
+ checkFailure(mp)
+ checkFailure(arrayMp)
+ checkFailure(arrayArrayMp)
+ val struct = Literal.create(create_row(1),
+ StructType(Array(StructField("a", IntegerType))))
+ checkFailure(struct)
+ val arrayStruct = Literal.create(
+ Array(create_row(1)),
+ ArrayType(StructType(Array(StructField("a", IntegerType)))))
+ checkFailure(arrayStruct)
+
+ // to_variant_object - success cases - nested types
+ check(Array(1, 2, 3), "[1,2,3]", toVariantObject = true)
+ check(mp, """{"x":3,"y":2,"z":1}""", toVariantObject = true)
+ check(arrayMp, """[{"x":3,"y":2,"z":1}]""", toVariantObject = true)
+ check(arrayArrayMp, """[[{"x":3,"y":2,"z":1}]]""", toVariantObject = true)
+ check(struct, """{"a":1}""", toVariantObject = true)
+ check(arrayStruct, """[{"a":1}]""", toVariantObject = true)
+ val complexStruct = Literal.create(
Row(
Seq("123", "true", "f"),
Map("a" -> "123", "b" -> "true", "c" -> "f"),
+ Map("a" -> Row(132)),
Row(0)),
- StructType.fromDDL("c ARRAY,b MAP,a STRUCT"))
- check(struct, """{"a":{"i":0},"b":{"a":"123","b":"true","c":"f"},"c":["123","true","f"]}""")
+ StructType.fromDDL("c ARRAY,b MAP,d MAP>," +
+ "a STRUCT"))
+ check(complexStruct,
+ """{"a":{"i":0},"b":{"a":"123","b":"true","c":"f"},"c":["123","true","f"],""" +
+ """"d":{"a":{"i":132}}}""",
+ toVariantObject = true)
+ check(ymArrLit, """["INTERVAL '0' MONTH","INTERVAL""" +
+ """ '2147483647' MONTH","INTERVAL '-2147483647' MONTH"]""", toVariantObject = true)
+
+ // to_variant_object - failure cases - non-nested types or map with non-string key
+ checkFailure(1, toVariantObject = true)
+ checkFailure(true, toVariantObject = true)
+ checkFailure(Literal.create(Literal.create(Period.ofMonths(0))), toVariantObject = true)
+ checkFailure(Map(1 -> 1), toVariantObject = true)
}
test("schema_of_variant - unknown type") {
@@ -1092,7 +1141,7 @@ class VariantExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
val results = mutable.HashMap.empty[(Literal, Literal), String]
for (i <- inputs) {
- val inputType = if (i.value == null) "VOID" else i.dataType.sql
+ val inputType = if (i.value == null) "VOID" else SchemaOfVariant.printSchema(i.dataType)
results.put((nul, i), inputType)
results.put((i, i), inputType)
}
@@ -1106,14 +1155,24 @@ class VariantExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
results.put((timestamp, timestampNtz), "TIMESTAMP")
results.put((float, decimal), "DOUBLE")
results.put((array1, array2), "ARRAY")
- results.put((struct1, struct2), "STRUCT")
+ results.put((struct1, struct2), "OBJECT")
results.put((dtInterval1, dtInterval2), "INTERVAL DAY TO SECOND")
results.put((ymInterval1, ymInterval2), "INTERVAL YEAR TO MONTH")
for (i1 <- inputs) {
for (i2 <- inputs) {
val expected = results.getOrElse((i1, i2), results.getOrElse((i2, i1), "VARIANT"))
- val array = CreateArray(Seq(Cast(i1, VariantType), Cast(i2, VariantType)))
+ val elem1 =
+ if (i1.dataType.isInstanceOf[ArrayType] || i1.dataType.isInstanceOf[MapType] ||
+ i1.dataType.isInstanceOf[StructType]) {
+ ToVariantObject(i1)
+ } else Cast(i1, VariantType)
+ val elem2 =
+ if (i2.dataType.isInstanceOf[ArrayType] || i2.dataType.isInstanceOf[MapType] ||
+ i2.dataType.isInstanceOf[StructType]) {
+ ToVariantObject(i2)
+ } else Cast(i2, VariantType)
+ val array = CreateArray(Seq(elem1, elem2))
checkEvaluation(SchemaOfVariant(Cast(array, VariantType)).replacement, s"ARRAY<$expected>")
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ExtractPythonUDFFromJoinConditionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ExtractPythonUDFFromJoinConditionSuite.scala
index 854a3e8f7a74d..776600bbdcf5d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ExtractPythonUDFFromJoinConditionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ExtractPythonUDFFromJoinConditionSuite.scala
@@ -189,7 +189,7 @@ class ExtractPythonUDFFromJoinConditionSuite extends PlanTest {
}
checkError(
exception = e,
- errorClass = "UNSUPPORTED_FEATURE.PYTHON_UDF_IN_ON_CLAUSE",
+ condition = "UNSUPPORTED_FEATURE.PYTHON_UDF_IN_ON_CLAUSE",
parameters = Map("joinType" -> joinType.sql)
)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
index 5aeb27f7ee6b4..451236162343b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
@@ -27,12 +27,13 @@ import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.logical.{Expand, Filter, LocalRelation, LogicalPlan, Project}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.catalyst.types.DataTypeUtils
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{IntegerType, MetadataBuilder}
-class PropagateEmptyRelationSuite extends PlanTest {
+class PropagateEmptyRelationSuite extends PlanTest {
object Optimize extends RuleExecutor[LogicalPlan] {
val batches =
- Batch("PropagateEmptyRelation", Once,
+ Batch("PropagateEmptyRelation", FixedPoint(1),
CombineUnions,
ReplaceDistinctWithAggregate,
ReplaceExceptWithAntiJoin,
@@ -45,7 +46,7 @@ class PropagateEmptyRelationSuite extends PlanTest {
object OptimizeWithoutPropagateEmptyRelation extends RuleExecutor[LogicalPlan] {
val batches =
- Batch("OptimizeWithoutPropagateEmptyRelation", Once,
+ Batch("OptimizeWithoutPropagateEmptyRelation", FixedPoint(1),
CombineUnions,
ReplaceDistinctWithAggregate,
ReplaceExceptWithAntiJoin,
@@ -216,10 +217,24 @@ class PropagateEmptyRelationSuite extends PlanTest {
.where($"a" =!= 200)
.orderBy($"a".asc)
- val optimized = Optimize.execute(query.analyze)
- val correctAnswer = LocalRelation(output, isStreaming = true)
+ withSQLConf(
+ SQLConf.PRUNE_FILTERS_CAN_PRUNE_STREAMING_SUBPLAN.key -> "true") {
+ val optimized = Optimize.execute(query.analyze)
+ val correctAnswer = LocalRelation(output, isStreaming = true)
+ comparePlans(optimized, correctAnswer)
+ }
- comparePlans(optimized, correctAnswer)
+ withSQLConf(
+ SQLConf.PRUNE_FILTERS_CAN_PRUNE_STREAMING_SUBPLAN.key -> "false") {
+ val optimized = Optimize.execute(query.analyze)
+ val correctAnswer = relation
+ .where(false)
+ .where($"a" > 1)
+ .select($"a")
+ .where($"a" =!= 200)
+ .orderBy($"a".asc).analyze
+ comparePlans(optimized, correctAnswer)
+ }
}
test("SPARK-47305 correctly tag isStreaming when propagating empty relation " +
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PruneFiltersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PruneFiltersSuite.scala
index b81a57f4f8cd5..66ded338340f3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PruneFiltersSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PruneFiltersSuite.scala
@@ -174,4 +174,38 @@ class PruneFiltersSuite extends PlanTest {
testRelation.where(!$"a".attr.in(1, 3, 5) && $"a".attr === 7 && $"b".attr === 1)
.where(Rand(10) > 0.1 && Rand(10) < 1.1).analyze)
}
+
+ test("Streaming relation is not lost under true filter") {
+ Seq("true", "false").foreach(x => withSQLConf(
+ SQLConf.PRUNE_FILTERS_CAN_PRUNE_STREAMING_SUBPLAN.key -> x) {
+ val streamingRelation =
+ LocalRelation(Seq($"a".int, $"b".int, $"c".int), Nil, isStreaming = true)
+ val originalQuery = streamingRelation.where(10 > 5).select($"a").analyze
+ val optimized = Optimize.execute(originalQuery)
+ val correctAnswer = streamingRelation.select($"a").analyze
+ comparePlans(optimized, correctAnswer)
+ })
+ }
+
+ test("Streaming relation is not lost under false filter") {
+ withSQLConf(
+ SQLConf.PRUNE_FILTERS_CAN_PRUNE_STREAMING_SUBPLAN.key -> "true") {
+ val streamingRelation =
+ LocalRelation(Seq($"a".int, $"b".int, $"c".int), Nil, isStreaming = true)
+ val originalQuery = streamingRelation.where(10 < 5).select($"a").analyze
+ val optimized = Optimize.execute(originalQuery)
+ val correctAnswer = streamingRelation.select($"a").analyze
+ comparePlans(optimized, correctAnswer)
+ }
+
+ withSQLConf(
+ SQLConf.PRUNE_FILTERS_CAN_PRUNE_STREAMING_SUBPLAN.key -> "false") {
+ val streamingRelation =
+ LocalRelation(Seq($"a".int, $"b".int, $"c".int), Nil, isStreaming = true)
+ val originalQuery = streamingRelation.where(10 < 5).select($"a").analyze
+ val optimized = Optimize.execute(originalQuery)
+ val correctAnswer = streamingRelation.where(10 < 5).select($"a").analyze
+ comparePlans(optimized, correctAnswer)
+ }
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReassignLambdaVariableIDSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReassignLambdaVariableIDSuite.scala
index bf9f922978f6d..677a5d7928fc1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReassignLambdaVariableIDSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReassignLambdaVariableIDSuite.scala
@@ -58,7 +58,7 @@ class ReassignLambdaVariableIDSuite extends PlanTest {
val query = testRelation.where(var1 && var2)
checkError(
exception = intercept[SparkException](Optimize.execute(query)),
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map(
"message" -> "LambdaVariable IDs in a query should be all positive or negative."))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
index a50842a26b2ce..eaa651e62455e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
@@ -59,7 +59,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
exception = intercept[AnalysisException] {
testFilter(originalCond = Literal(null, IntegerType), expectedCond = FalseLiteral)
},
- errorClass = "DATATYPE_MISMATCH.FILTER_NOT_BOOLEAN",
+ condition = "DATATYPE_MISMATCH.FILTER_NOT_BOOLEAN",
parameters = Map("sqlExpr" -> "\"NULL\"", "filter" -> "\"NULL\"", "type" -> "\"INT\"")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregatesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregatesSuite.scala
index ac136dfb898ef..4d31999ded655 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregatesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregatesSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.expressions.{Literal, Round}
import org.apache.spark.sql.catalyst.expressions.aggregate.CollectSet
import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Expand, LocalRelation, LogicalPlan}
@@ -109,4 +109,20 @@ class RewriteDistinctAggregatesSuite extends PlanTest {
case _ => fail(s"Plan is not rewritten:\n$rewrite")
}
}
+
+ test("SPARK-49261: Literals in grouping expressions shouldn't result in unresolved aggregation") {
+ val relation = testRelation2
+ .select(Literal(6).as("gb"), $"a", $"b", $"c", $"d")
+ val input = relation
+ .groupBy($"a", $"gb")(
+ countDistinct($"b").as("agg1"),
+ countDistinct($"d").as("agg2"),
+ Round(sum($"c").as("sum1"), 6)).analyze
+ val rewriteFold = FoldablePropagation(input)
+ // without the fix, the below produces an unresolved plan
+ val rewrite = RewriteDistinctAggregates(rewriteFold)
+ if (!rewrite.resolved) {
+ fail(s"Plan is not as expected:\n$rewrite")
+ }
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index d514f777e5544..b7e2490b552cc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -20,14 +20,16 @@ package org.apache.spark.sql.catalyst.parser
import java.util.Locale
import org.apache.spark.SparkThrowable
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis._
import org.apache.spark.sql.catalyst.expressions.{EqualTo, Hex, Literal}
import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.connector.catalog.IdentityColumnSpec
import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition.{after, first}
import org.apache.spark.sql.connector.expressions.{ApplyTransform, BucketTransform, ClusterByTransform, DaysTransform, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, Transform, YearsTransform}
import org.apache.spark.sql.connector.expressions.LogicalExpressions.bucket
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{Decimal, IntegerType, LongType, StringType, StructType, TimestampType}
+import org.apache.spark.sql.types.{DataType, Decimal, IntegerType, LongType, StringType, StructType, TimestampType}
import org.apache.spark.storage.StorageLevelMapper
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -79,7 +81,7 @@ class DDLParserSuite extends AnalysisTest {
val sql = "CREATE TABLE my_tab(a: INT COMMENT 'test', b: STRING) USING parquet"
checkError(
exception = parseException(sql),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "':'", "hint" -> ""))
}
@@ -380,7 +382,7 @@ class DDLParserSuite extends AnalysisTest {
|Columns: p2 string""".stripMargin
checkError(
exception = parseException(createSql),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> value1),
context = ExpectedContext(
fragment = createSql,
@@ -390,7 +392,7 @@ class DDLParserSuite extends AnalysisTest {
val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
checkError(
exception = parseException(replaceSql),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> value1),
context = ExpectedContext(
fragment = replaceSql,
@@ -405,7 +407,7 @@ class DDLParserSuite extends AnalysisTest {
|Columns: p2 string""".stripMargin
checkError(
exception = parseException(createSqlWithExpr),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> value2),
context = ExpectedContext(
fragment = createSqlWithExpr,
@@ -415,7 +417,7 @@ class DDLParserSuite extends AnalysisTest {
val replaceSqlWithExpr = createSqlWithExpr.replaceFirst("CREATE", "REPLACE")
checkError(
exception = parseException(replaceSqlWithExpr),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> value2),
context = ExpectedContext(
fragment = replaceSqlWithExpr,
@@ -482,7 +484,7 @@ class DDLParserSuite extends AnalysisTest {
"which also specifies a serde"
checkError(
exception = parseException(createSql),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> value),
context = ExpectedContext(
fragment = createSql,
@@ -492,7 +494,7 @@ class DDLParserSuite extends AnalysisTest {
val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
checkError(
exception = parseException(replaceSql),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> value),
context = ExpectedContext(
fragment = replaceSql,
@@ -538,7 +540,7 @@ class DDLParserSuite extends AnalysisTest {
val value = "ROW FORMAT DELIMITED is only compatible with 'textfile', not 'otherformat'"
checkError(
exception = parseException(createFailSql),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> value),
context = ExpectedContext(
fragment = createFailSql,
@@ -548,7 +550,7 @@ class DDLParserSuite extends AnalysisTest {
val replaceFailSql = createFailSql.replaceFirst("CREATE", "REPLACE")
checkError(
exception = parseException(replaceFailSql),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> value),
context = ExpectedContext(
fragment = replaceFailSql,
@@ -610,7 +612,7 @@ class DDLParserSuite extends AnalysisTest {
|STORED AS parquet""".stripMargin
checkError(
exception = parseException(createSql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "CREATE TABLE ... USING ... STORED AS PARQUET "),
context = ExpectedContext(
fragment = createSql,
@@ -620,7 +622,7 @@ class DDLParserSuite extends AnalysisTest {
val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
checkError(
exception = parseException(replaceSql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "REPLACE TABLE ... USING ... STORED AS PARQUET "),
context = ExpectedContext(
fragment = replaceSql,
@@ -635,7 +637,7 @@ class DDLParserSuite extends AnalysisTest {
|ROW FORMAT SERDE 'customSerde'""".stripMargin
checkError(
exception = parseException(createSql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "CREATE TABLE ... USING ... ROW FORMAT SERDE CUSTOMSERDE"),
context = ExpectedContext(
fragment = createSql,
@@ -645,7 +647,7 @@ class DDLParserSuite extends AnalysisTest {
val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
checkError(
exception = parseException(replaceSql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "REPLACE TABLE ... USING ... ROW FORMAT SERDE CUSTOMSERDE"),
context = ExpectedContext(
fragment = replaceSql,
@@ -660,7 +662,7 @@ class DDLParserSuite extends AnalysisTest {
|ROW FORMAT DELIMITED FIELDS TERMINATED BY ','""".stripMargin
checkError(
exception = parseException(createSql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "CREATE TABLE ... USING ... ROW FORMAT DELIMITED"),
context = ExpectedContext(
fragment = createSql,
@@ -670,7 +672,7 @@ class DDLParserSuite extends AnalysisTest {
val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
checkError(
exception = parseException(replaceSql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "REPLACE TABLE ... USING ... ROW FORMAT DELIMITED"),
context = ExpectedContext(
fragment = replaceSql,
@@ -685,7 +687,7 @@ class DDLParserSuite extends AnalysisTest {
val fragment = "STORED BY 'handler'"
checkError(
exception = parseException(createSql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "STORED BY"),
context = ExpectedContext(
fragment = fragment,
@@ -695,7 +697,7 @@ class DDLParserSuite extends AnalysisTest {
val replaceSql = createSql.replaceFirst("CREATE", "REPLACE")
checkError(
exception = parseException(replaceSql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "STORED BY"),
context = ExpectedContext(
fragment = fragment,
@@ -707,7 +709,7 @@ class DDLParserSuite extends AnalysisTest {
val sql1 = "CREATE TABLE my_tab (id bigint) SKEWED BY (id) ON (1,2,3)"
checkError(
exception = parseException(sql1),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "CREATE TABLE ... SKEWED BY"),
context = ExpectedContext(
fragment = sql1,
@@ -717,7 +719,7 @@ class DDLParserSuite extends AnalysisTest {
val sql2 = "REPLACE TABLE my_tab (id bigint) SKEWED BY (id) ON (1,2,3)"
checkError(
exception = parseException(sql2),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "CREATE TABLE ... SKEWED BY"),
context = ExpectedContext(
fragment = sql2,
@@ -737,7 +739,7 @@ class DDLParserSuite extends AnalysisTest {
val sql1 = createTableHeader("TBLPROPERTIES('test' = 'test2')")
checkError(
exception = parseException(sql1),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "TBLPROPERTIES"),
context = ExpectedContext(
fragment = sql1,
@@ -747,7 +749,7 @@ class DDLParserSuite extends AnalysisTest {
val sql2 = createTableHeader("LOCATION '/tmp/file'")
checkError(
exception = parseException(sql2),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "LOCATION"),
context = ExpectedContext(
fragment = sql2,
@@ -757,7 +759,7 @@ class DDLParserSuite extends AnalysisTest {
val sql3 = createTableHeader("COMMENT 'a table'")
checkError(
exception = parseException(sql3),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "COMMENT"),
context = ExpectedContext(
fragment = sql3,
@@ -767,7 +769,7 @@ class DDLParserSuite extends AnalysisTest {
val sql4 = createTableHeader("CLUSTERED BY(b) INTO 256 BUCKETS")
checkError(
exception = parseException(sql4),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "CLUSTERED BY"),
context = ExpectedContext(
fragment = sql4,
@@ -777,7 +779,7 @@ class DDLParserSuite extends AnalysisTest {
val sql5 = createTableHeader("PARTITIONED BY (b)")
checkError(
exception = parseException(sql5),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "PARTITIONED BY"),
context = ExpectedContext(
fragment = sql5,
@@ -787,7 +789,7 @@ class DDLParserSuite extends AnalysisTest {
val sql6 = createTableHeader("PARTITIONED BY (c int)")
checkError(
exception = parseException(sql6),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "PARTITIONED BY"),
context = ExpectedContext(
fragment = sql6,
@@ -797,7 +799,7 @@ class DDLParserSuite extends AnalysisTest {
val sql7 = createTableHeader("STORED AS parquet")
checkError(
exception = parseException(sql7),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "STORED AS/BY"),
context = ExpectedContext(
fragment = sql7,
@@ -807,7 +809,7 @@ class DDLParserSuite extends AnalysisTest {
val sql8 = createTableHeader("STORED AS INPUTFORMAT 'in' OUTPUTFORMAT 'out'")
checkError(
exception = parseException(sql8),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "STORED AS/BY"),
context = ExpectedContext(
fragment = sql8,
@@ -817,7 +819,7 @@ class DDLParserSuite extends AnalysisTest {
val sql9 = createTableHeader("ROW FORMAT SERDE 'serde'")
checkError(
exception = parseException(sql9),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "ROW FORMAT"),
context = ExpectedContext(
fragment = sql9,
@@ -827,7 +829,7 @@ class DDLParserSuite extends AnalysisTest {
val sql10 = replaceTableHeader("TBLPROPERTIES('test' = 'test2')")
checkError(
exception = parseException(sql10),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "TBLPROPERTIES"),
context = ExpectedContext(
fragment = sql10,
@@ -837,7 +839,7 @@ class DDLParserSuite extends AnalysisTest {
val sql11 = replaceTableHeader("LOCATION '/tmp/file'")
checkError(
exception = parseException(sql11),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "LOCATION"),
context = ExpectedContext(
fragment = sql11,
@@ -847,7 +849,7 @@ class DDLParserSuite extends AnalysisTest {
val sql12 = replaceTableHeader("COMMENT 'a table'")
checkError(
exception = parseException(sql12),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "COMMENT"),
context = ExpectedContext(
fragment = sql12,
@@ -857,7 +859,7 @@ class DDLParserSuite extends AnalysisTest {
val sql13 = replaceTableHeader("CLUSTERED BY(b) INTO 256 BUCKETS")
checkError(
exception = parseException(sql13),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "CLUSTERED BY"),
context = ExpectedContext(
fragment = sql13,
@@ -867,7 +869,7 @@ class DDLParserSuite extends AnalysisTest {
val sql14 = replaceTableHeader("PARTITIONED BY (b)")
checkError(
exception = parseException(sql14),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "PARTITIONED BY"),
context = ExpectedContext(
fragment = sql14,
@@ -877,7 +879,7 @@ class DDLParserSuite extends AnalysisTest {
val sql15 = replaceTableHeader("PARTITIONED BY (c int)")
checkError(
exception = parseException(sql15),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "PARTITIONED BY"),
context = ExpectedContext(
fragment = sql15,
@@ -887,7 +889,7 @@ class DDLParserSuite extends AnalysisTest {
val sql16 = replaceTableHeader("STORED AS parquet")
checkError(
exception = parseException(sql16),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "STORED AS/BY"),
context = ExpectedContext(
fragment = sql16,
@@ -897,7 +899,7 @@ class DDLParserSuite extends AnalysisTest {
val sql17 = replaceTableHeader("STORED AS INPUTFORMAT 'in' OUTPUTFORMAT 'out'")
checkError(
exception = parseException(sql17),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "STORED AS/BY"),
context = ExpectedContext(
fragment = sql17,
@@ -907,7 +909,7 @@ class DDLParserSuite extends AnalysisTest {
val sql18 = replaceTableHeader("ROW FORMAT SERDE 'serde'")
checkError(
exception = parseException(sql18),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "ROW FORMAT"),
context = ExpectedContext(
fragment = sql18,
@@ -917,7 +919,7 @@ class DDLParserSuite extends AnalysisTest {
val sql19 = createTableHeader("CLUSTER BY (a)")
checkError(
exception = parseException(sql19),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "CLUSTER BY"),
context = ExpectedContext(
fragment = sql19,
@@ -927,7 +929,7 @@ class DDLParserSuite extends AnalysisTest {
val sql20 = replaceTableHeader("CLUSTER BY (a)")
checkError(
exception = parseException(sql20),
- errorClass = "DUPLICATE_CLAUSES",
+ condition = "DUPLICATE_CLAUSES",
parameters = Map("clauseName" -> "CLUSTER BY"),
context = ExpectedContext(
fragment = sql20,
@@ -1231,7 +1233,7 @@ class DDLParserSuite extends AnalysisTest {
val fragment = "bad_type"
checkError(
exception = parseException(sql),
- errorClass = "UNSUPPORTED_DATATYPE",
+ condition = "UNSUPPORTED_DATATYPE",
parameters = Map("typeName" -> "\"BAD_TYPE\""),
context = ExpectedContext(
fragment = fragment,
@@ -1282,19 +1284,19 @@ class DDLParserSuite extends AnalysisTest {
val sql1 = "ALTER TABLE table_name ALTER COLUMN a.b.c TYPE bigint COMMENT 'new comment'"
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'COMMENT'", "hint" -> ""))
val sql2 = "ALTER TABLE table_name ALTER COLUMN a.b.c TYPE bigint COMMENT AFTER d"
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'COMMENT'", "hint" -> ""))
val sql3 = "ALTER TABLE table_name ALTER COLUMN a.b.c TYPE bigint COMMENT 'new comment' AFTER d"
checkError(
exception = parseException(sql3),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'COMMENT'", "hint" -> ""))
}
@@ -1364,7 +1366,7 @@ class DDLParserSuite extends AnalysisTest {
val sql4 = "ALTER TABLE table_name CHANGE COLUMN a.b.c new_name INT"
checkError(
exception = parseException(sql4),
- errorClass = "_LEGACY_ERROR_TEMP_0034",
+ condition = "_LEGACY_ERROR_TEMP_0034",
parameters = Map(
"operation" -> "Renaming column",
"command" -> "ALTER COLUMN",
@@ -1378,7 +1380,7 @@ class DDLParserSuite extends AnalysisTest {
val sql5 = "ALTER TABLE table_name PARTITION (a='1') CHANGE COLUMN a.b.c c INT"
checkError(
exception = parseException(sql5),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "ALTER TABLE ... PARTITION ... CHANGE COLUMN"),
context = ExpectedContext(
fragment = sql5,
@@ -1425,7 +1427,7 @@ class DDLParserSuite extends AnalysisTest {
val sql5 = "ALTER TABLE table_name PARTITION (a='1') REPLACE COLUMNS (x string)"
checkError(
exception = parseException(sql5),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "ALTER TABLE ... PARTITION ... REPLACE COLUMNS"),
context = ExpectedContext(
fragment = sql5,
@@ -1435,7 +1437,7 @@ class DDLParserSuite extends AnalysisTest {
val sql6 = "ALTER TABLE table_name REPLACE COLUMNS (x string NOT NULL)"
checkError(
exception = parseException(sql6),
- errorClass = "_LEGACY_ERROR_TEMP_0034",
+ condition = "_LEGACY_ERROR_TEMP_0034",
parameters = Map("operation" -> "NOT NULL", "command" -> "REPLACE COLUMNS", "msg" -> ""),
context = ExpectedContext(
fragment = sql6,
@@ -1445,7 +1447,7 @@ class DDLParserSuite extends AnalysisTest {
val sql7 = "ALTER TABLE table_name REPLACE COLUMNS (x string FIRST)"
checkError(
exception = parseException(sql7),
- errorClass = "_LEGACY_ERROR_TEMP_0034",
+ condition = "_LEGACY_ERROR_TEMP_0034",
parameters = Map(
"operation" -> "Column position",
"command" -> "REPLACE COLUMNS",
@@ -1458,7 +1460,7 @@ class DDLParserSuite extends AnalysisTest {
val sql8 = "ALTER TABLE table_name REPLACE COLUMNS (a.b.c string)"
checkError(
exception = parseException(sql8),
- errorClass = "_LEGACY_ERROR_TEMP_0034",
+ condition = "_LEGACY_ERROR_TEMP_0034",
parameters = Map(
"operation" -> "Replacing with a nested column",
"command" -> "REPLACE COLUMNS",
@@ -1471,7 +1473,7 @@ class DDLParserSuite extends AnalysisTest {
val sql9 = "ALTER TABLE table_name REPLACE COLUMNS (a STRING COMMENT 'x' COMMENT 'y')"
checkError(
exception = parseException(sql9),
- errorClass = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
+ condition = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
parameters = Map(
"type" -> "REPLACE",
"columnName" -> "a",
@@ -1646,7 +1648,7 @@ class DDLParserSuite extends AnalysisTest {
|PARTITION (p1 = 3, p2) IF NOT EXISTS""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> "IF NOT EXISTS with dynamic partitions: p2"),
context = ExpectedContext(
fragment = fragment,
@@ -1664,7 +1666,7 @@ class DDLParserSuite extends AnalysisTest {
|PARTITION (p1 = 3) IF NOT EXISTS""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "INSERT INTO ... IF NOT EXISTS"),
context = ExpectedContext(
fragment = fragment,
@@ -1704,7 +1706,7 @@ class DDLParserSuite extends AnalysisTest {
checkError(
exception = parseException(
"INSERT INTO TABLE t1 BY NAME (c1,c2) SELECT * FROM tmp_view"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map(
"error" -> "'c1'",
"hint" -> "")
@@ -1729,7 +1731,7 @@ class DDLParserSuite extends AnalysisTest {
val sql = "DELETE FROM testcat.ns1.ns2.tbl AS t(a,b,c,d) WHERE d = 2"
checkError(
exception = parseException(sql),
- errorClass = "COLUMN_ALIASES_NOT_ALLOWED",
+ condition = "COLUMN_ALIASES_NOT_ALLOWED",
parameters = Map("op" -> "DELETE"),
context = ExpectedContext(
fragment = sql,
@@ -1771,7 +1773,7 @@ class DDLParserSuite extends AnalysisTest {
|WHERE d=2""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "COLUMN_ALIASES_NOT_ALLOWED",
+ condition = "COLUMN_ALIASES_NOT_ALLOWED",
parameters = Map("op" -> "UPDATE"),
context = ExpectedContext(
fragment = sql,
@@ -1931,7 +1933,7 @@ class DDLParserSuite extends AnalysisTest {
""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'*'", "hint" -> ""))
}
@@ -1977,7 +1979,7 @@ class DDLParserSuite extends AnalysisTest {
.stripMargin
checkError(
exception = parseException(sql),
- errorClass = "COLUMN_ALIASES_NOT_ALLOWED",
+ condition = "COLUMN_ALIASES_NOT_ALLOWED",
parameters = Map("op" -> "MERGE"),
context = ExpectedContext(
fragment = sql,
@@ -2056,7 +2058,7 @@ class DDLParserSuite extends AnalysisTest {
|THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION",
+ condition = "NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION",
parameters = Map.empty,
context = ExpectedContext(
fragment = sql,
@@ -2079,7 +2081,7 @@ class DDLParserSuite extends AnalysisTest {
|THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "NON_LAST_NOT_MATCHED_BY_TARGET_CLAUSE_OMIT_CONDITION",
+ condition = "NON_LAST_NOT_MATCHED_BY_TARGET_CLAUSE_OMIT_CONDITION",
parameters = Map.empty,
context = ExpectedContext(
fragment = sql,
@@ -2103,7 +2105,7 @@ class DDLParserSuite extends AnalysisTest {
|WHEN NOT MATCHED BY SOURCE THEN DELETE""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "NON_LAST_NOT_MATCHED_BY_SOURCE_CLAUSE_OMIT_CONDITION",
+ condition = "NON_LAST_NOT_MATCHED_BY_SOURCE_CLAUSE_OMIT_CONDITION",
parameters = Map.empty,
context = ExpectedContext(
fragment = sql,
@@ -2118,7 +2120,7 @@ class DDLParserSuite extends AnalysisTest {
|ON target.col1 = source.col1""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "_LEGACY_ERROR_TEMP_0008",
+ condition = "_LEGACY_ERROR_TEMP_0008",
parameters = Map.empty,
context = ExpectedContext(
fragment = sql,
@@ -2209,7 +2211,7 @@ class DDLParserSuite extends AnalysisTest {
val sql1 = "analyze table a.b.c compute statistics xxxx"
checkError(
exception = parseException(sql1),
- errorClass = "INVALID_SQL_SYNTAX.ANALYZE_TABLE_UNEXPECTED_NOSCAN",
+ condition = "INVALID_SQL_SYNTAX.ANALYZE_TABLE_UNEXPECTED_NOSCAN",
parameters = Map("ctx" -> "XXXX"),
context = ExpectedContext(
fragment = sql1,
@@ -2219,7 +2221,7 @@ class DDLParserSuite extends AnalysisTest {
val sql2 = "analyze table a.b.c partition (a) compute statistics xxxx"
checkError(
exception = parseException(sql2),
- errorClass = "INVALID_SQL_SYNTAX.ANALYZE_TABLE_UNEXPECTED_NOSCAN",
+ condition = "INVALID_SQL_SYNTAX.ANALYZE_TABLE_UNEXPECTED_NOSCAN",
parameters = Map("ctx" -> "XXXX"),
context = ExpectedContext(
fragment = sql2,
@@ -2238,7 +2240,7 @@ class DDLParserSuite extends AnalysisTest {
val sql = "ANALYZE TABLES IN a.b.c COMPUTE STATISTICS xxxx"
checkError(
exception = parseException(sql),
- errorClass = "INVALID_SQL_SYNTAX.ANALYZE_TABLE_UNEXPECTED_NOSCAN",
+ condition = "INVALID_SQL_SYNTAX.ANALYZE_TABLE_UNEXPECTED_NOSCAN",
parameters = Map("ctx" -> "XXXX"),
context = ExpectedContext(
fragment = sql,
@@ -2250,7 +2252,7 @@ class DDLParserSuite extends AnalysisTest {
val sql1 = "ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR COLUMNS"
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "end of input", "hint" -> ""))
comparePlans(
@@ -2287,13 +2289,13 @@ class DDLParserSuite extends AnalysisTest {
val sql2 = "ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR ALL COLUMNS key, value"
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'key'", "hint" -> "")) // expecting {, ';'}
val sql3 = "ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR ALL"
checkError(
exception = parseException(sql3),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "end of input", "hint" -> ": missing 'COLUMNS'"))
}
@@ -2370,7 +2372,7 @@ class DDLParserSuite extends AnalysisTest {
val sql = "CACHE TABLE a.b.c AS SELECT * FROM testData"
checkError(
exception = parseException(sql),
- errorClass = "_LEGACY_ERROR_TEMP_0037",
+ condition = "_LEGACY_ERROR_TEMP_0037",
parameters = Map("quoted" -> "a.b"),
context = ExpectedContext(
fragment = sql,
@@ -2382,7 +2384,7 @@ class DDLParserSuite extends AnalysisTest {
val createTableSql = "create table test_table using my_data_source options (password)"
checkError(
exception = parseException(createTableSql),
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> "A value must be specified for the key: password."),
context = ExpectedContext(
fragment = createTableSql,
@@ -2413,7 +2415,7 @@ class DDLParserSuite extends AnalysisTest {
|(dt='2009-09-09', country='uk')""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "INVALID_STATEMENT_OR_CLAUSE",
+ condition = "INVALID_STATEMENT_OR_CLAUSE",
parameters = Map("operation" -> "ALTER VIEW ... ADD PARTITION"),
context = ExpectedContext(
fragment = sql,
@@ -2698,14 +2700,14 @@ class DDLParserSuite extends AnalysisTest {
val sql1 = "ALTER TABLE t1 ALTER COLUMN a.b.c SET DEFAULT "
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "end of input", "hint" -> ""))
// It is not possible to both SET DEFAULT and DROP DEFAULT at the same time.
// This results in a parsing error.
val sql2 = "ALTER TABLE t1 ALTER COLUMN a.b.c DROP DEFAULT SET DEFAULT 42"
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'SET'", "hint" -> ""))
comparePlans(
@@ -2724,7 +2726,7 @@ class DDLParserSuite extends AnalysisTest {
val fragment = "b STRING NOT NULL DEFAULT \"abc\""
checkError(
exception = parseException(sql),
- errorClass = "UNSUPPORTED_DEFAULT_VALUE.WITH_SUGGESTION",
+ condition = "UNSUPPORTED_DEFAULT_VALUE.WITH_SUGGESTION",
parameters = Map.empty,
context = ExpectedContext(
fragment = fragment,
@@ -2784,7 +2786,7 @@ class DDLParserSuite extends AnalysisTest {
exception = intercept[ParseException](
parsePlan(
"CREATE TABLE my_tab(a INT, b STRING NOT NULL DEFAULT \"abc\" NOT NULL)")),
- errorClass = "CREATE_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
+ condition = "CREATE_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
parameters = Map(
"columnName" -> "b",
"optionName" -> "NOT NULL"),
@@ -2794,7 +2796,7 @@ class DDLParserSuite extends AnalysisTest {
exception = intercept[ParseException](
parsePlan(
"CREATE TABLE my_tab(a INT, b STRING DEFAULT \"123\" NOT NULL DEFAULT \"abc\")")),
- errorClass = "CREATE_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
+ condition = "CREATE_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
parameters = Map(
"columnName" -> "b",
"optionName" -> "DEFAULT"),
@@ -2804,7 +2806,7 @@ class DDLParserSuite extends AnalysisTest {
exception = intercept[ParseException](
parsePlan(
"CREATE TABLE my_tab(a INT, b STRING COMMENT \"abc\" NOT NULL COMMENT \"abc\")")),
- errorClass = "CREATE_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
+ condition = "CREATE_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
parameters = Map(
"columnName" -> "b",
"optionName" -> "COMMENT"),
@@ -2836,7 +2838,7 @@ class DDLParserSuite extends AnalysisTest {
checkError(
exception = parseException("CREATE TABLE my_tab(a INT, " +
"b INT GENERATED ALWAYS AS (a + 1) GENERATED ALWAYS AS (a + 2)) USING PARQUET"),
- errorClass = "CREATE_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
+ condition = "CREATE_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
parameters = Map("columnName" -> "b", "optionName" -> "GENERATED ALWAYS AS"),
context = ExpectedContext(
fragment = "b INT GENERATED ALWAYS AS (a + 1) GENERATED ALWAYS AS (a + 2)",
@@ -2848,18 +2850,225 @@ class DDLParserSuite extends AnalysisTest {
checkError(
exception = parseException(
"CREATE TABLE my_tab(a INT, b INT GENERATED ALWAYS AS ()) USING PARQUET"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "')'", "hint" -> "")
)
// No parenthesis
checkError(
exception = parseException(
"CREATE TABLE my_tab(a INT, b INT GENERATED ALWAYS AS a + 1) USING PARQUET"),
- errorClass = "PARSE_SYNTAX_ERROR",
- parameters = Map("error" -> "'a'", "hint" -> ": missing '('")
+ condition = "PARSE_SYNTAX_ERROR",
+ parameters = Map("error" -> "'a'", "hint" -> "")
)
}
+ test("SPARK-48824: implement parser support for " +
+ "GENERATED ALWAYS/BY DEFAULT AS IDENTITY columns in tables ") {
+ def parseAndCompareIdentityColumnPlan(
+ identityColumnDataTypeStr: String,
+ identityColumnDefStr: String,
+ identityColumnSpecStr: String,
+ expectedDataType: DataType,
+ expectedStart: Long,
+ expectedStep: Long,
+ expectedAllowExplicitInsert: Boolean): Unit = {
+ val columnsWithIdentitySpec = Seq(
+ ColumnDefinition(
+ name = "id",
+ dataType = expectedDataType,
+ nullable = true,
+ identityColumnSpec = Some(
+ new IdentityColumnSpec(
+ expectedStart,
+ expectedStep,
+ expectedAllowExplicitInsert
+ )
+ )
+ ),
+ ColumnDefinition("val", IntegerType)
+ )
+ comparePlans(
+ parsePlan(
+ s"CREATE TABLE my_tab(id $identityColumnDataTypeStr GENERATED $identityColumnDefStr" +
+ s" AS IDENTITY $identityColumnSpecStr, val INT) USING parquet"
+ ),
+ CreateTable(
+ UnresolvedIdentifier(Seq("my_tab")),
+ columnsWithIdentitySpec,
+ Seq.empty[Transform],
+ UnresolvedTableSpec(
+ Map.empty[String, String],
+ Some("parquet"),
+ OptionList(Seq.empty),
+ None,
+ None,
+ None,
+ false
+ ),
+ false
+ )
+ )
+
+ comparePlans(
+ parsePlan(
+ s"REPLACE TABLE my_tab(id $identityColumnDataTypeStr GENERATED $identityColumnDefStr" +
+ s" AS IDENTITY $identityColumnSpecStr, val INT) USING parquet"
+ ),
+ ReplaceTable(
+ UnresolvedIdentifier(Seq("my_tab")),
+ columnsWithIdentitySpec,
+ Seq.empty[Transform],
+ UnresolvedTableSpec(
+ Map.empty[String, String],
+ Some("parquet"),
+ OptionList(Seq.empty),
+ None,
+ None,
+ None,
+ false
+ ),
+ false
+ )
+ )
+ }
+ for {
+ identityColumnDefStr <- Seq("BY DEFAULT", "ALWAYS")
+ identityColumnDataTypeStr <- Seq("BIGINT", "INT")
+ } {
+ val expectedAllowExplicitInsert = identityColumnDefStr == "BY DEFAULT"
+ val expectedDataType = identityColumnDataTypeStr match {
+ case "BIGINT" => LongType
+ case "INT" => IntegerType
+ }
+ parseAndCompareIdentityColumnPlan(
+ identityColumnDataTypeStr,
+ identityColumnDefStr,
+ "(START WITH 2 INCREMENT BY 2)",
+ expectedDataType,
+ expectedStart = 2,
+ expectedStep = 2,
+ expectedAllowExplicitInsert = expectedAllowExplicitInsert)
+ parseAndCompareIdentityColumnPlan(
+ identityColumnDataTypeStr,
+ identityColumnDefStr,
+ "(START WITH -2 INCREMENT BY -2)",
+ expectedDataType,
+ expectedStart = -2,
+ expectedStep = -2,
+ expectedAllowExplicitInsert = expectedAllowExplicitInsert)
+ parseAndCompareIdentityColumnPlan(
+ identityColumnDataTypeStr,
+ identityColumnDefStr,
+ "(START WITH 2)",
+ expectedDataType,
+ expectedStart = 2,
+ expectedStep = 1,
+ expectedAllowExplicitInsert = expectedAllowExplicitInsert)
+ parseAndCompareIdentityColumnPlan(
+ identityColumnDataTypeStr,
+ identityColumnDefStr,
+ "(START WITH -2)",
+ expectedDataType,
+ expectedStart = -2,
+ expectedStep = 1,
+ expectedAllowExplicitInsert = expectedAllowExplicitInsert)
+ parseAndCompareIdentityColumnPlan(
+ identityColumnDataTypeStr,
+ identityColumnDefStr,
+ "(INCREMENT BY 2)",
+ expectedDataType,
+ expectedStart = 1,
+ expectedStep = 2,
+ expectedAllowExplicitInsert = expectedAllowExplicitInsert)
+ parseAndCompareIdentityColumnPlan(
+ identityColumnDataTypeStr,
+ identityColumnDefStr,
+ "(INCREMENT BY -2)",
+ expectedDataType,
+ expectedStart = 1,
+ expectedStep = -2,
+ expectedAllowExplicitInsert = expectedAllowExplicitInsert)
+ parseAndCompareIdentityColumnPlan(
+ identityColumnDataTypeStr,
+ identityColumnDefStr,
+ "()",
+ expectedDataType,
+ expectedStart = 1,
+ expectedStep = 1,
+ expectedAllowExplicitInsert = expectedAllowExplicitInsert)
+ parseAndCompareIdentityColumnPlan(
+ identityColumnDataTypeStr,
+ identityColumnDefStr,
+ "",
+ expectedDataType,
+ expectedStart = 1,
+ expectedStep = 1,
+ expectedAllowExplicitInsert = expectedAllowExplicitInsert)
+ }
+ }
+
+ test("SPARK-48824: Column cannot have both a generation expression and an identity column spec") {
+ checkError(
+ exception = intercept[AnalysisException] {
+ parsePlan(s"CREATE TABLE testcat.my_tab(id BIGINT GENERATED ALWAYS AS 1" +
+ s" GENERATED ALWAYS AS IDENTITY, val INT) USING foo")
+ },
+ condition = "PARSE_SYNTAX_ERROR",
+ parameters = Map("error" -> "'1'", "hint" -> "")
+ )
+ }
+
+ test("SPARK-48824: Identity column step must not be zero") {
+ checkError(
+ exception = intercept[ParseException] {
+ parsePlan(
+ s"CREATE TABLE testcat.my_tab" +
+ s"(id BIGINT GENERATED ALWAYS AS IDENTITY(INCREMENT BY 0), val INT) USING foo"
+ )
+ },
+ condition = "IDENTITY_COLUMNS_ILLEGAL_STEP",
+ parameters = Map.empty,
+ context = ExpectedContext(
+ fragment = "id BIGINT GENERATED ALWAYS AS IDENTITY(INCREMENT BY 0)",
+ start = 28,
+ stop = 81)
+ )
+ }
+
+ test("SPARK-48824: Identity column datatype must be long or integer") {
+ checkError(
+ exception = intercept[ParseException] {
+ parsePlan(
+ s"CREATE TABLE testcat.my_tab(id FLOAT GENERATED ALWAYS AS IDENTITY(), val INT) USING foo"
+ )
+ },
+ condition = "IDENTITY_COLUMNS_UNSUPPORTED_DATA_TYPE",
+ parameters = Map("dataType" -> "FloatType"),
+ context =
+ ExpectedContext(fragment = "id FLOAT GENERATED ALWAYS AS IDENTITY()", start = 28, stop = 66)
+ )
+ }
+
+ test("SPARK-48824: Identity column sequence generator option cannot be duplicated") {
+ val identityColumnSpecStrs = Seq(
+ "(START WITH 0 START WITH 1)",
+ "(INCREMENT BY 1 INCREMENT BY 2)",
+ "(START WITH 0 INCREMENT BY 1 START WITH 1)",
+ "(INCREMENT BY 1 START WITH 0 INCREMENT BY 2)"
+ )
+ for {
+ identitySpecStr <- identityColumnSpecStrs
+ } {
+ val exception = intercept[ParseException] {
+ parsePlan(
+ s"CREATE TABLE testcat.my_tab" +
+ s"(id BIGINT GENERATED ALWAYS AS IDENTITY $identitySpecStr, val INT) USING foo"
+ )
+ }
+ assert(exception.getErrorClass === "IDENTITY_COLUMNS_DUPLICATED_SEQUENCE_GENERATOR_OPTION")
+ }
+ }
+
test("SPARK-42681: Relax ordering constraint for ALTER TABLE ADD COLUMN options") {
// Positive test cases to verify that column definition options could be applied in any order.
val expectedPlan = AddColumns(
@@ -2887,7 +3096,7 @@ class DDLParserSuite extends AnalysisTest {
exception = intercept[ParseException](
parsePlan("ALTER TABLE my_tab ADD COLUMN b STRING NOT NULL DEFAULT \"abc\" NOT NULL")
),
- errorClass = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
+ condition = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
parameters = Map("type" -> "ADD", "columnName" -> "b", "optionName" -> "NOT NULL"),
context = ExpectedContext(
fragment = "b STRING NOT NULL DEFAULT \"abc\" NOT NULL",
@@ -2899,7 +3108,7 @@ class DDLParserSuite extends AnalysisTest {
exception = intercept[ParseException](
parsePlan("ALTER TABLE my_tab ADD COLUMN b STRING DEFAULT \"123\" NOT NULL DEFAULT \"abc\"")
),
- errorClass = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
+ condition = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
parameters = Map("type" -> "ADD", "columnName" -> "b", "optionName" -> "DEFAULT"),
context = ExpectedContext(
fragment = "b STRING DEFAULT \"123\" NOT NULL DEFAULT \"abc\"",
@@ -2911,7 +3120,7 @@ class DDLParserSuite extends AnalysisTest {
exception = intercept[ParseException](
parsePlan("ALTER TABLE my_tab ADD COLUMN b STRING COMMENT \"abc\" NOT NULL COMMENT \"abc\"")
),
- errorClass = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
+ condition = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
parameters = Map("type" -> "ADD", "columnName" -> "b", "optionName" -> "COMMENT"),
context = ExpectedContext(
fragment = "b STRING COMMENT \"abc\" NOT NULL COMMENT \"abc\"",
@@ -2923,7 +3132,7 @@ class DDLParserSuite extends AnalysisTest {
exception = intercept[ParseException](
parsePlan("ALTER TABLE my_tab ADD COLUMN b STRING FIRST COMMENT \"abc\" AFTER y")
),
- errorClass = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
+ condition = "ALTER_TABLE_COLUMN_DESCRIPTOR_DUPLICATE",
parameters = Map("type" -> "ADD", "columnName" -> "b", "optionName" -> "FIRST|AFTER"),
context =
ExpectedContext(fragment = "b STRING FIRST COMMENT \"abc\" AFTER y", start = 30, stop = 65)
@@ -2935,7 +3144,7 @@ class DDLParserSuite extends AnalysisTest {
"USING parquet CLUSTERED BY (a) INTO 2 BUCKETS CLUSTER BY (a)"
checkError(
exception = parseException(sql1),
- errorClass = "SPECIFY_CLUSTER_BY_WITH_BUCKETING_IS_NOT_ALLOWED",
+ condition = "SPECIFY_CLUSTER_BY_WITH_BUCKETING_IS_NOT_ALLOWED",
parameters = Map.empty,
context = ExpectedContext(fragment = sql1, start = 0, stop = 96)
)
@@ -2946,7 +3155,7 @@ class DDLParserSuite extends AnalysisTest {
"USING parquet CLUSTERED BY (a) INTO 2 BUCKETS CLUSTER BY (a)"
checkError(
exception = parseException(sql1),
- errorClass = "SPECIFY_CLUSTER_BY_WITH_BUCKETING_IS_NOT_ALLOWED",
+ condition = "SPECIFY_CLUSTER_BY_WITH_BUCKETING_IS_NOT_ALLOWED",
parameters = Map.empty,
context = ExpectedContext(fragment = sql1, start = 0, stop = 97)
)
@@ -2957,7 +3166,7 @@ class DDLParserSuite extends AnalysisTest {
"USING parquet CLUSTER BY (a) PARTITIONED BY (a)"
checkError(
exception = parseException(sql1),
- errorClass = "SPECIFY_CLUSTER_BY_WITH_PARTITIONED_BY_IS_NOT_ALLOWED",
+ condition = "SPECIFY_CLUSTER_BY_WITH_PARTITIONED_BY_IS_NOT_ALLOWED",
parameters = Map.empty,
context = ExpectedContext(fragment = sql1, start = 0, stop = 83)
)
@@ -2968,7 +3177,7 @@ class DDLParserSuite extends AnalysisTest {
"USING parquet CLUSTER BY (a) PARTITIONED BY (a)"
checkError(
exception = parseException(sql1),
- errorClass = "SPECIFY_CLUSTER_BY_WITH_PARTITIONED_BY_IS_NOT_ALLOWED",
+ condition = "SPECIFY_CLUSTER_BY_WITH_PARTITIONED_BY_IS_NOT_ALLOWED",
parameters = Map.empty,
context = ExpectedContext(fragment = sql1, start = 0, stop = 84)
)
@@ -2986,7 +3195,7 @@ class DDLParserSuite extends AnalysisTest {
checkError(
exception = internalException(insertDirSql),
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> "INSERT OVERWRITE DIRECTORY is not supported."))
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala
index f11e920e4c07d..c416d21cfd4b0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala
@@ -138,12 +138,12 @@ class DataTypeParserSuite extends SparkFunSuite with SQLHelper {
test("Do not print empty parentheses for no params") {
checkError(
exception = intercept("unknown"),
- errorClass = "UNSUPPORTED_DATATYPE",
+ condition = "UNSUPPORTED_DATATYPE",
parameters = Map("typeName" -> "\"UNKNOWN\"")
)
checkError(
exception = intercept("unknown(1,2,3)"),
- errorClass = "UNSUPPORTED_DATATYPE",
+ condition = "UNSUPPORTED_DATATYPE",
parameters = Map("typeName" -> "\"UNKNOWN(1,2,3)\"")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala
index e4f9b54680dc7..603d5d779769d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala
@@ -34,7 +34,7 @@ class ErrorParserSuite extends AnalysisTest {
test("semantic errors") {
checkError(
exception = parseException("select *\nfrom r\norder by q\ncluster by q"),
- errorClass = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
+ condition = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
parameters = Map.empty,
context = ExpectedContext(fragment = "order by q\ncluster by q", start = 16, stop = 38))
}
@@ -43,42 +43,42 @@ class ErrorParserSuite extends AnalysisTest {
// scalastyle:off
checkError(
exception = parseException("USE \u0196pfel"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "\u0196pfel"))
checkError(
exception = parseException("USE \u88681"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "\u88681"))
// scalastyle:on
checkError(
exception = parseException("USE https://www.spa.rk/bucket/pa-th.json?=&#%"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "https://www.spa.rk/bucket/pa-th.json?=&#%"))
}
test("hyphen in identifier - DDL tests") {
checkError(
exception = parseException("USE test-test"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-test"))
checkError(
exception = parseException("SET CATALOG test-test"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-test"))
checkError(
exception = parseException("CREATE DATABASE IF NOT EXISTS my-database"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "my-database"))
checkError(
exception = parseException(
"""
|ALTER DATABASE my-database
|SET DBPROPERTIES ('p1'='v1')""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "my-database"))
checkError(
exception = parseException("DROP DATABASE my-database"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "my-database"))
checkError(
exception = parseException(
@@ -87,7 +87,7 @@ class ErrorParserSuite extends AnalysisTest {
|CHANGE COLUMN
|test-col TYPE BIGINT
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-col"))
checkError(
exception = parseException(
@@ -96,23 +96,23 @@ class ErrorParserSuite extends AnalysisTest {
|DROP COLUMN
|test-col, test
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-col"))
checkError(
exception = parseException("CREATE TABLE test (attri-bute INT)"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "attri-bute"))
checkError(
exception = parseException("CREATE FUNCTION test-func as org.test.func"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-func"))
checkError(
exception = parseException("DROP FUNCTION test-func as org.test.func"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-func"))
checkError(
exception = parseException("SHOW FUNCTIONS LIKE test-func"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-func"))
checkError(
exception = parseException(
@@ -123,7 +123,7 @@ class ErrorParserSuite extends AnalysisTest {
|LOCATION '/user/external/page_view'
|TBLPROPERTIES ('p1'='v1', 'p2'='v2')
|AS SELECT * FROM src""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "page-view"))
checkError(
exception = parseException(
@@ -131,31 +131,31 @@ class ErrorParserSuite extends AnalysisTest {
|CREATE TABLE IF NOT EXISTS tab
|USING test-provider
|AS SELECT * FROM src""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-provider"))
checkError(
exception = parseException("SHOW TABLES IN hyphen-database"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "hyphen-database"))
checkError(
exception = parseException("SHOW TABLE EXTENDED IN hyphen-db LIKE \"str\""),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "hyphen-db"))
checkError(
exception = parseException("DESC SCHEMA EXTENDED test-db"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-db"))
checkError(
exception = parseException("ANALYZE TABLE test-table PARTITION (part1)"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-table"))
checkError(
exception = parseException("CREATE TABLE t(c1 struct)"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-test"))
checkError(
exception = parseException("LOAD DATA INPATH \"path\" INTO TABLE my-tab"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "my-tab"))
}
@@ -163,28 +163,28 @@ class ErrorParserSuite extends AnalysisTest {
// dml tests
checkError(
exception = parseException("SELECT * FROM table-with-hyphen"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "table-with-hyphen"))
// special test case: minus in expression shouldn't be treated as hyphen in identifiers
checkError(
exception = parseException("SELECT a-b FROM table-with-hyphen"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "table-with-hyphen"))
checkError(
exception = parseException("SELECT a-b AS a-b FROM t"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "a-b"))
checkError(
exception = parseException("SELECT a-b FROM table-hyphen WHERE a-b = 0"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "table-hyphen"))
checkError(
exception = parseException("SELECT (a - test_func(b-c)) FROM test-table"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-table"))
checkError(
exception = parseException("WITH a-b AS (SELECT 1 FROM s) SELECT * FROM s;"),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "a-b"))
checkError(
exception = parseException(
@@ -193,7 +193,7 @@ class ErrorParserSuite extends AnalysisTest {
|FROM t1 JOIN t2
|USING (a, b, at-tr)
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "at-tr"))
checkError(
exception = parseException(
@@ -202,7 +202,7 @@ class ErrorParserSuite extends AnalysisTest {
|OVER (PARTITION BY category ORDER BY revenue DESC) as hyphen-rank
|FROM productRevenue
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "hyphen-rank"))
checkError(
exception = parseException(
@@ -213,7 +213,7 @@ class ErrorParserSuite extends AnalysisTest {
|GROUP BY fake-breaker
|ORDER BY c
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "grammar-breaker"))
assert(parsePlan(
"""
@@ -234,7 +234,7 @@ class ErrorParserSuite extends AnalysisTest {
|WINDOW hyphen-window AS
| (PARTITION BY a, b ORDER BY c rows BETWEEN 1 PRECEDING AND 1 FOLLOWING)
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "hyphen-window"))
checkError(
exception = parseException(
@@ -242,7 +242,7 @@ class ErrorParserSuite extends AnalysisTest {
|SELECT * FROM tab
|WINDOW window_ref AS window-ref
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "window-ref"))
checkError(
exception = parseException(
@@ -251,7 +251,7 @@ class ErrorParserSuite extends AnalysisTest {
|FROM t-a INNER JOIN tb
|ON ta.a = tb.a AND ta.tag = tb.tag
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "t-a"))
checkError(
exception = parseException(
@@ -260,7 +260,7 @@ class ErrorParserSuite extends AnalysisTest {
|SELECT a
|SELECT b
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-table"))
checkError(
exception = parseException(
@@ -273,7 +273,7 @@ class ErrorParserSuite extends AnalysisTest {
| FOR test-test IN ('dotNET', 'Java')
|);
""".stripMargin),
- errorClass = "INVALID_IDENTIFIER",
+ condition = "INVALID_IDENTIFIER",
parameters = Map("ident" -> "test-test"))
}
@@ -281,23 +281,23 @@ class ErrorParserSuite extends AnalysisTest {
// general bad types
checkError(
exception = parseException("SELECT cast(1 as badtype)"),
- errorClass = "UNSUPPORTED_DATATYPE",
+ condition = "UNSUPPORTED_DATATYPE",
parameters = Map("typeName" -> "\"BADTYPE\""),
context = ExpectedContext(fragment = "badtype", start = 17, stop = 23))
// special handling on char and varchar
checkError(
exception = parseException("SELECT cast('a' as CHAR)"),
- errorClass = "DATATYPE_MISSING_SIZE",
+ condition = "DATATYPE_MISSING_SIZE",
parameters = Map("type" -> "\"CHAR\""),
context = ExpectedContext(fragment = "CHAR", start = 19, stop = 22))
checkError(
exception = parseException("SELECT cast('a' as Varchar)"),
- errorClass = "DATATYPE_MISSING_SIZE",
+ condition = "DATATYPE_MISSING_SIZE",
parameters = Map("type" -> "\"VARCHAR\""),
context = ExpectedContext(fragment = "Varchar", start = 19, stop = 25))
checkError(
exception = parseException("SELECT cast('a' as Character)"),
- errorClass = "DATATYPE_MISSING_SIZE",
+ condition = "DATATYPE_MISSING_SIZE",
parameters = Map("type" -> "\"CHARACTER\""),
context = ExpectedContext(fragment = "Character", start = 19, stop = 27))
}
@@ -305,32 +305,32 @@ class ErrorParserSuite extends AnalysisTest {
test("'!' where only NOT should be allowed") {
checkError(
exception = parseException("SELECT 1 ! IN (2)"),
- errorClass = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
+ condition = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
parameters = Map("clause" -> "!"),
context = ExpectedContext(fragment = "!", start = 9, stop = 9))
checkError(
exception = parseException("SELECT 'a' ! LIKE 'b'"),
- errorClass = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
+ condition = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
parameters = Map("clause" -> "!"),
context = ExpectedContext(fragment = "!", start = 11, stop = 11))
checkError(
exception = parseException("SELECT 1 ! BETWEEN 1 AND 2"),
- errorClass = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
+ condition = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
parameters = Map("clause" -> "!"),
context = ExpectedContext(fragment = "!", start = 9, stop = 9))
checkError(
exception = parseException("SELECT 1 IS ! NULL"),
- errorClass = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
+ condition = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
parameters = Map("clause" -> "!"),
context = ExpectedContext(fragment = "!", start = 12, stop = 12))
checkError(
exception = parseException("CREATE TABLE IF ! EXISTS t(c1 INT)"),
- errorClass = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
+ condition = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
parameters = Map("clause" -> "!"),
context = ExpectedContext(fragment = "!", start = 16, stop = 16))
checkError(
exception = parseException("CREATE TABLE t(c1 INT ! NULL)"),
- errorClass = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
+ condition = "SYNTAX_DISCONTINUED.BANG_EQUALS_NOT",
parameters = Map("clause" -> "!"),
context = ExpectedContext(fragment = "!", start = 22, stop = 22))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 2654757177ee7..6d307d1cd9a87 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -201,7 +201,7 @@ class ExpressionParserSuite extends AnalysisTest {
checkError(
exception = parseException("a like 'pattern%' escape '##'"),
- errorClass = "INVALID_ESC",
+ condition = "INVALID_ESC",
parameters = Map("invalidEscape" -> "'##'"),
context = ExpectedContext(
fragment = "like 'pattern%' escape '##'",
@@ -210,7 +210,7 @@ class ExpressionParserSuite extends AnalysisTest {
checkError(
exception = parseException("a like 'pattern%' escape ''"),
- errorClass = "INVALID_ESC",
+ condition = "INVALID_ESC",
parameters = Map("invalidEscape" -> "''"),
context = ExpectedContext(
fragment = "like 'pattern%' escape ''",
@@ -222,7 +222,7 @@ class ExpressionParserSuite extends AnalysisTest {
checkError(
exception = parseException("a not like 'pattern%' escape '\"/'"),
- errorClass = "INVALID_ESC",
+ condition = "INVALID_ESC",
parameters = Map("invalidEscape" -> "'\"/'"),
context = ExpectedContext(
fragment = "not like 'pattern%' escape '\"/'",
@@ -231,7 +231,7 @@ class ExpressionParserSuite extends AnalysisTest {
checkError(
exception = parseException("a not like 'pattern%' escape ''"),
- errorClass = "INVALID_ESC",
+ condition = "INVALID_ESC",
parameters = Map("invalidEscape" -> "''"),
context = ExpectedContext(
fragment = "not like 'pattern%' escape ''",
@@ -261,7 +261,7 @@ class ExpressionParserSuite extends AnalysisTest {
Seq("any", "some", "all").foreach { quantifier =>
checkError(
exception = parseException(s"a like $quantifier()"),
- errorClass = "_LEGACY_ERROR_TEMP_0064",
+ condition = "_LEGACY_ERROR_TEMP_0064",
parameters = Map("msg" -> "Expected something between '(' and ')'."),
context = ExpectedContext(
fragment = s"like $quantifier()",
@@ -328,7 +328,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("`select`(all a, b)", $"select".function($"a", $"b"))
checkError(
exception = parseException("foo(a x)"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'x'", "hint" -> ": extra input 'x'"))
}
@@ -461,7 +461,7 @@ class ExpressionParserSuite extends AnalysisTest {
// We cannot use an arbitrary expression.
checkError(
exception = parseException("foo(*) over (partition by a order by b rows exp(b) preceding)"),
- errorClass = "_LEGACY_ERROR_TEMP_0064",
+ condition = "_LEGACY_ERROR_TEMP_0064",
parameters = Map("msg" -> "Frame bound value must be a literal."),
context = ExpectedContext(
fragment = "exp(b) preceding",
@@ -540,7 +540,7 @@ class ExpressionParserSuite extends AnalysisTest {
Literal(Timestamp.valueOf("2016-03-11 20:54:00.000")))
checkError(
exception = parseException("timestamP_LTZ '2016-33-11 20:54:00.000'"),
- errorClass = "INVALID_TYPED_LITERAL",
+ condition = "INVALID_TYPED_LITERAL",
sqlState = "42604",
parameters = Map(
"valueType" -> "\"TIMESTAMP_LTZ\"",
@@ -556,7 +556,7 @@ class ExpressionParserSuite extends AnalysisTest {
Literal(LocalDateTime.parse("2016-03-11T20:54:00.000")))
checkError(
exception = parseException("tImEstAmp_Ntz '2016-33-11 20:54:00.000'"),
- errorClass = "INVALID_TYPED_LITERAL",
+ condition = "INVALID_TYPED_LITERAL",
sqlState = "42604",
parameters = Map(
"valueType" -> "\"TIMESTAMP_NTZ\"",
@@ -572,7 +572,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("dAte '2016-03-11'", Literal(Date.valueOf("2016-03-11")))
checkError(
exception = parseException("DAtE 'mar 11 2016'"),
- errorClass = "INVALID_TYPED_LITERAL",
+ condition = "INVALID_TYPED_LITERAL",
sqlState = "42604",
parameters = Map("valueType" -> "\"DATE\"", "value" -> "'mar 11 2016'"),
context = ExpectedContext(
@@ -585,7 +585,7 @@ class ExpressionParserSuite extends AnalysisTest {
Literal(Timestamp.valueOf("2016-03-11 20:54:00.000")))
checkError(
exception = parseException("timestamP '2016-33-11 20:54:00.000'"),
- errorClass = "INVALID_TYPED_LITERAL",
+ condition = "INVALID_TYPED_LITERAL",
sqlState = "42604",
parameters = Map("valueType" -> "\"TIMESTAMP\"", "value" -> "'2016-33-11 20:54:00.000'"),
context = ExpectedContext(
@@ -600,7 +600,7 @@ class ExpressionParserSuite extends AnalysisTest {
checkError(
exception = parseException("timestamP '2016-33-11 20:54:00.000'"),
- errorClass = "INVALID_TYPED_LITERAL",
+ condition = "INVALID_TYPED_LITERAL",
sqlState = "42604",
parameters = Map("valueType" -> "\"TIMESTAMP\"", "value" -> "'2016-33-11 20:54:00.000'"),
context = ExpectedContext(
@@ -621,7 +621,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("INTERVAL '1 year 2 month'", ymIntervalLiteral)
checkError(
exception = parseException("Interval 'interval 1 yearsss 2 monthsss'"),
- errorClass = "INVALID_TYPED_LITERAL",
+ condition = "INVALID_TYPED_LITERAL",
parameters = Map(
"valueType" -> "\"INTERVAL\"",
"value" -> "'interval 1 yearsss 2 monthsss'"
@@ -638,7 +638,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("INTERVAL '1 day 2 hour 3 minute 4.005006 second'", dtIntervalLiteral)
checkError(
exception = parseException("Interval 'interval 1 daysss 2 hoursss'"),
- errorClass = "INVALID_TYPED_LITERAL",
+ condition = "INVALID_TYPED_LITERAL",
parameters = Map(
"valueType" -> "\"INTERVAL\"",
"value" -> "'interval 1 daysss 2 hoursss'"
@@ -651,7 +651,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("-interval '1 day 2 hour 3 minute 4.005006 second'", UnaryMinus(dtIntervalLiteral))
checkError(
exception = parseException("INTERVAL '1 year 2 second'"),
- errorClass = "_LEGACY_ERROR_TEMP_0029",
+ condition = "_LEGACY_ERROR_TEMP_0029",
parameters = Map("literal" -> "INTERVAL '1 year 2 second'"),
context = ExpectedContext(
fragment = "INTERVAL '1 year 2 second'",
@@ -664,7 +664,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("INTERVAL '3 month 1 hour'", intervalLiteral)
checkError(
exception = parseException("Interval 'interval 3 monthsss 1 hoursss'"),
- errorClass = "INVALID_TYPED_LITERAL",
+ condition = "INVALID_TYPED_LITERAL",
parameters = Map(
"valueType" -> "\"INTERVAL\"",
"value" -> "'interval 3 monthsss 1 hoursss'"
@@ -688,7 +688,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("x'A10C'", Literal(Array(0xa1, 0x0c).map(_.toByte)))
checkError(
exception = parseException("x'A1OC'"),
- errorClass = "INVALID_TYPED_LITERAL",
+ condition = "INVALID_TYPED_LITERAL",
sqlState = "42604",
parameters = Map(
"valueType" -> "\"X\"",
@@ -701,7 +701,7 @@ class ExpressionParserSuite extends AnalysisTest {
checkError(
exception = parseException("GEO '(10,-6)'"),
- errorClass = "UNSUPPORTED_TYPED_LITERAL",
+ condition = "UNSUPPORTED_TYPED_LITERAL",
parameters = Map(
"unsupportedType" -> "\"GEO\"",
"supportedTypes" ->
@@ -743,14 +743,14 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("9.e+1BD", Literal(BigDecimal("9.e+1").underlying()))
checkError(
exception = parseException(".e3"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'.'", "hint" -> ""))
// Tiny Int Literal
assertEqual("10Y", Literal(10.toByte))
checkError(
exception = parseException("1000Y"),
- errorClass = "INVALID_NUMERIC_LITERAL_RANGE",
+ condition = "INVALID_NUMERIC_LITERAL_RANGE",
parameters = Map(
"rawStrippedQualifier" -> "1000",
"minValue" -> Byte.MinValue.toString,
@@ -765,7 +765,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("10S", Literal(10.toShort))
checkError(
exception = parseException("40000S"),
- errorClass = "INVALID_NUMERIC_LITERAL_RANGE",
+ condition = "INVALID_NUMERIC_LITERAL_RANGE",
parameters = Map(
"rawStrippedQualifier" -> "40000",
"minValue" -> Short.MinValue.toString,
@@ -780,7 +780,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("10L", Literal(10L))
checkError(
exception = parseException("78732472347982492793712334L"),
- errorClass = "INVALID_NUMERIC_LITERAL_RANGE",
+ condition = "INVALID_NUMERIC_LITERAL_RANGE",
parameters = Map(
"rawStrippedQualifier" -> "78732472347982492793712334",
"minValue" -> Long.MinValue.toString,
@@ -795,7 +795,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("10.0D", Literal(10.0D))
checkError(
exception = parseException("-1.8E308D"),
- errorClass = "INVALID_NUMERIC_LITERAL_RANGE",
+ condition = "INVALID_NUMERIC_LITERAL_RANGE",
parameters = Map(
"rawStrippedQualifier" -> "-1.8E308",
"minValue" -> BigDecimal(Double.MinValue).toString,
@@ -807,7 +807,7 @@ class ExpressionParserSuite extends AnalysisTest {
stop = 8))
checkError(
exception = parseException("1.8E308D"),
- errorClass = "INVALID_NUMERIC_LITERAL_RANGE",
+ condition = "INVALID_NUMERIC_LITERAL_RANGE",
parameters = Map(
"rawStrippedQualifier" -> "1.8E308",
"minValue" -> BigDecimal(Double.MinValue).toString,
@@ -825,7 +825,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("123.08BD", Literal(BigDecimal("123.08").underlying()))
checkError(
exception = parseException("1.20E-38BD"),
- errorClass = "DECIMAL_PRECISION_EXCEEDS_MAX_PRECISION",
+ condition = "DECIMAL_PRECISION_EXCEEDS_MAX_PRECISION",
parameters = Map(
"precision" -> "40",
"maxPrecision" -> "38"),
@@ -899,7 +899,7 @@ class ExpressionParserSuite extends AnalysisTest {
// when ESCAPED_STRING_LITERALS is enabled.
checkError(
exception = parseException("'\''"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'''", "hint" -> ": extra input '''"))
// The unescape special characters (e.g., "\\t") for 2.0+ don't work
@@ -1082,7 +1082,7 @@ class ExpressionParserSuite extends AnalysisTest {
// Unknown FROM TO intervals
checkError(
exception = parseException("interval '10' month to second"),
- errorClass = "_LEGACY_ERROR_TEMP_0028",
+ condition = "_LEGACY_ERROR_TEMP_0028",
parameters = Map("from" -> "month", "to" -> "second"),
context = ExpectedContext(
fragment = "'10' month to second",
@@ -1104,7 +1104,7 @@ class ExpressionParserSuite extends AnalysisTest {
} else {
checkError(
exception = parseException(s"interval $intervalStr"),
- errorClass = "_LEGACY_ERROR_TEMP_0029",
+ condition = "_LEGACY_ERROR_TEMP_0029",
parameters = Map("literal" -> "interval 3 monThs 4 dayS 22 sEcond 1 millisecond"),
context = ExpectedContext(
fragment = s"interval $intervalStr",
@@ -1120,7 +1120,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual("1 - f('o', o(bar))", Literal(1) - $"f".function("o", $"o".function($"bar")))
checkError(
exception = parseException("1 - f('o', o(bar)) hello * world"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'*'", "hint" -> ""))
}
@@ -1142,7 +1142,7 @@ class ExpressionParserSuite extends AnalysisTest {
assertEqual(complexName.quotedString, UnresolvedAttribute(Seq("`fo`o", "`ba`r")))
checkError(
exception = parseException(complexName.unquotedString),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'.'", "hint" -> ""))
// Function identifier contains continuous backticks should be treated correctly.
@@ -1225,7 +1225,7 @@ class ExpressionParserSuite extends AnalysisTest {
Seq("any", "some", "all").foreach { quantifier =>
checkError(
exception = parseException(s"a ilike $quantifier()"),
- errorClass = "_LEGACY_ERROR_TEMP_0064",
+ condition = "_LEGACY_ERROR_TEMP_0064",
parameters = Map("msg" -> "Expected something between '(' and ')'."),
context = ExpectedContext(
fragment = s"ilike $quantifier()",
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala
index 83d2557108c57..93afef60a9ddf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParserUtilsSuite.scala
@@ -159,7 +159,7 @@ class ParserUtilsSuite extends SparkFunSuite {
exception = intercept[ParseException] {
operationNotAllowed(errorMessage, showFuncContext)
},
- errorClass = "_LEGACY_ERROR_TEMP_0035",
+ condition = "_LEGACY_ERROR_TEMP_0035",
parameters = Map("message" -> errorMessage))
}
@@ -172,7 +172,7 @@ class ParserUtilsSuite extends SparkFunSuite {
exception = intercept[ParseException] {
checkDuplicateKeys(properties2, createDbContext)
},
- errorClass = "DUPLICATE_KEY",
+ condition = "DUPLICATE_KEY",
parameters = Map("keyColumn" -> "`a`"))
}
@@ -223,7 +223,7 @@ class ParserUtilsSuite extends SparkFunSuite {
exception = intercept[ParseException] {
validate(f1(emptyContext), message, emptyContext)
},
- errorClass = "_LEGACY_ERROR_TEMP_0064",
+ condition = "_LEGACY_ERROR_TEMP_0064",
parameters = Map("msg" -> message))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
index a6a32e87b7421..6901f6e928c8a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
@@ -204,7 +204,7 @@ class PlanParserSuite extends AnalysisTest {
|""".stripMargin
checkError(
exception = parseException(query),
- errorClass = "UNCLOSED_BRACKETED_COMMENT",
+ condition = "UNCLOSED_BRACKETED_COMMENT",
parameters = Map.empty)
}
@@ -222,7 +222,7 @@ class PlanParserSuite extends AnalysisTest {
|""".stripMargin
checkError(
exception = parseException(query),
- errorClass = "UNCLOSED_BRACKETED_COMMENT",
+ condition = "UNCLOSED_BRACKETED_COMMENT",
parameters = Map.empty)
}
@@ -237,7 +237,7 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "EXPLAIN logical SELECT 1"
checkError(
exception = parseException(sql1),
- errorClass = "_LEGACY_ERROR_TEMP_0039",
+ condition = "_LEGACY_ERROR_TEMP_0039",
parameters = Map.empty,
context = ExpectedContext(
fragment = sql1,
@@ -247,7 +247,7 @@ class PlanParserSuite extends AnalysisTest {
val sql2 = "EXPLAIN formatted SELECT 1"
checkError(
exception = parseException(sql2),
- errorClass = "_LEGACY_ERROR_TEMP_0039",
+ condition = "_LEGACY_ERROR_TEMP_0039",
parameters = Map.empty,
context = ExpectedContext(
fragment = sql2,
@@ -295,7 +295,7 @@ class PlanParserSuite extends AnalysisTest {
val sql = "with cte1 (select 1), cte1 as (select 1 from cte1) select * from cte1"
checkError(
exception = parseException(sql),
- errorClass = "_LEGACY_ERROR_TEMP_0038",
+ condition = "_LEGACY_ERROR_TEMP_0038",
parameters = Map("duplicateNames" -> "'cte1'"),
context = ExpectedContext(
fragment = sql,
@@ -328,13 +328,13 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "from a"
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "end of input", "hint" -> ""))
val sql2 = "from (from a union all from b) c select *"
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'union'", "hint" -> ""))
}
@@ -345,12 +345,12 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "from a select * select * from x where a.s < 10"
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'from'", "hint" -> ""))
val sql2 = "from a select * from b"
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'from'", "hint" -> ""))
assertEqual(
"from a insert into tbl1 select * insert into tbl2 select * where s < 10",
@@ -393,7 +393,7 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = s"$baseSql order by a sort by a"
checkError(
exception = parseException(sql1),
- errorClass = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
+ condition = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
parameters = Map.empty,
context = ExpectedContext(
fragment = "order by a sort by a",
@@ -403,7 +403,7 @@ class PlanParserSuite extends AnalysisTest {
val sql2 = s"$baseSql cluster by a distribute by a"
checkError(
exception = parseException(sql2),
- errorClass = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
+ condition = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
parameters = Map.empty,
context = ExpectedContext(
fragment = "cluster by a distribute by a",
@@ -413,7 +413,7 @@ class PlanParserSuite extends AnalysisTest {
val sql3 = s"$baseSql order by a cluster by a"
checkError(
exception = parseException(sql3),
- errorClass = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
+ condition = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
parameters = Map.empty,
context = ExpectedContext(
fragment = "order by a cluster by a",
@@ -423,7 +423,7 @@ class PlanParserSuite extends AnalysisTest {
val sql4 = s"$baseSql order by a distribute by a"
checkError(
exception = parseException(sql4),
- errorClass = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
+ condition = "UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES",
parameters = Map.empty,
context = ExpectedContext(
fragment = "order by a distribute by a",
@@ -499,7 +499,7 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "SELECT a, b, count(distinct a, distinct b) as c FROM d GROUP BY a, b"
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'b'", "hint" -> ": extra input 'b'"))
}
@@ -595,7 +595,7 @@ class PlanParserSuite extends AnalysisTest {
|)""".stripMargin
checkError(
exception = parseException(sql1),
- errorClass = "NOT_ALLOWED_IN_FROM.LATERAL_WITH_PIVOT",
+ condition = "NOT_ALLOWED_IN_FROM.LATERAL_WITH_PIVOT",
parameters = Map.empty,
context = ExpectedContext(
fragment = fragment1,
@@ -617,7 +617,7 @@ class PlanParserSuite extends AnalysisTest {
|)""".stripMargin
checkError(
exception = parseException(sql2),
- errorClass = "NOT_ALLOWED_IN_FROM.LATERAL_WITH_UNPIVOT",
+ condition = "NOT_ALLOWED_IN_FROM.LATERAL_WITH_UNPIVOT",
parameters = Map.empty,
context = ExpectedContext(
fragment = fragment2,
@@ -647,7 +647,7 @@ class PlanParserSuite extends AnalysisTest {
|)""".stripMargin
checkError(
exception = parseException(sql3),
- errorClass = "NOT_ALLOWED_IN_FROM.UNPIVOT_WITH_PIVOT",
+ condition = "NOT_ALLOWED_IN_FROM.UNPIVOT_WITH_PIVOT",
parameters = Map.empty,
context = ExpectedContext(
fragment = fragment3,
@@ -711,7 +711,7 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "select * from a natural cross join b"
checkError(
exception = parseException(sql1),
- errorClass = "INCOMPATIBLE_JOIN_TYPES",
+ condition = "INCOMPATIBLE_JOIN_TYPES",
parameters = Map("joinType1" -> "NATURAL", "joinType2" -> "CROSS"),
sqlState = "42613",
context = ExpectedContext(
@@ -723,7 +723,7 @@ class PlanParserSuite extends AnalysisTest {
val sql2 = "select * from a natural join b on a.id = b.id"
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'on'", "hint" -> ""))
// Test multiple consecutive joins
@@ -744,7 +744,7 @@ class PlanParserSuite extends AnalysisTest {
val sql3 = "select * from t1 inner join t2 inner join t3 on col3 = col2 on col3 = col1"
checkError(
exception = parseException(sql3),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'on'", "hint" -> ""))
// Parenthesis
@@ -834,7 +834,7 @@ class PlanParserSuite extends AnalysisTest {
val fragment1 = "tablesample(bucket 4 out of 10 on x)"
checkError(
exception = parseException(sql1),
- errorClass = "_LEGACY_ERROR_TEMP_0015",
+ condition = "_LEGACY_ERROR_TEMP_0015",
parameters = Map("msg" -> "BUCKET x OUT OF y ON colname"),
context = ExpectedContext(
fragment = fragment1,
@@ -845,7 +845,7 @@ class PlanParserSuite extends AnalysisTest {
val fragment2 = "tablesample(bucket 11 out of 10)"
checkError(
exception = parseException(sql2),
- errorClass = "_LEGACY_ERROR_TEMP_0064",
+ condition = "_LEGACY_ERROR_TEMP_0064",
parameters = Map("msg" -> "Sampling fraction (1.1) must be on interval [0, 1]"),
context = ExpectedContext(
fragment = fragment2,
@@ -856,7 +856,7 @@ class PlanParserSuite extends AnalysisTest {
val fragment3 = "TABLESAMPLE(300M)"
checkError(
exception = parseException(sql3),
- errorClass = "_LEGACY_ERROR_TEMP_0015",
+ condition = "_LEGACY_ERROR_TEMP_0015",
parameters = Map("msg" -> "byteLengthLiteral"),
context = ExpectedContext(
fragment = fragment3,
@@ -867,7 +867,7 @@ class PlanParserSuite extends AnalysisTest {
val fragment4 = "TABLESAMPLE(BUCKET 3 OUT OF 32 ON rand())"
checkError(
exception = parseException(sql4),
- errorClass = "_LEGACY_ERROR_TEMP_0015",
+ condition = "_LEGACY_ERROR_TEMP_0015",
parameters = Map("msg" -> "BUCKET x OUT OF y ON function"),
context = ExpectedContext(
fragment = fragment4,
@@ -925,7 +925,7 @@ class PlanParserSuite extends AnalysisTest {
val fragment1 = "default.range(2)"
checkError(
exception = parseException(sql1),
- errorClass = "INVALID_SQL_SYNTAX.INVALID_TABLE_VALUED_FUNC_NAME",
+ condition = "INVALID_SQL_SYNTAX.INVALID_TABLE_VALUED_FUNC_NAME",
parameters = Map("funcName" -> "`default`.`range`"),
context = ExpectedContext(
fragment = fragment1,
@@ -937,7 +937,7 @@ class PlanParserSuite extends AnalysisTest {
val fragment2 = "spark_catalog.default.range(2)"
checkError(
exception = parseException(sql2),
- errorClass = "INVALID_SQL_SYNTAX.INVALID_TABLE_VALUED_FUNC_NAME",
+ condition = "INVALID_SQL_SYNTAX.INVALID_TABLE_VALUED_FUNC_NAME",
parameters = Map("funcName" -> "`spark_catalog`.`default`.`range`"),
context = ExpectedContext(
fragment = fragment2,
@@ -1047,14 +1047,14 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "SELECT /*+ HINT() */ * FROM t"
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "')'", "hint" -> ""))
// Disallow space as the delimiter.
val sql2 = "SELECT /*+ INDEX(a b c) */ * from default.t"
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'b'", "hint" -> ""))
assertEqual(
@@ -1114,7 +1114,7 @@ class PlanParserSuite extends AnalysisTest {
val sql3 = "SELECT /*+ COALESCE(30 + 50) */ * FROM t"
checkError(
exception = parseException(sql3),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'+'", "hint" -> ""))
assertEqual(
@@ -1241,13 +1241,13 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "select ltrim(both 'S' from 'SS abc S'"
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'from'", "hint" -> "")) // expecting {')'
val sql2 = "select rtrim(trailing 'S' from 'SS abc S'"
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'from'", "hint" -> "")) // expecting {')'
assertTrimPlans(
@@ -1361,7 +1361,7 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "CREATE VIEW testView AS INSERT INTO jt VALUES(1, 1)"
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'INSERT'", "hint" -> ""))
// Multi insert query
@@ -1371,13 +1371,13 @@ class PlanParserSuite extends AnalysisTest {
|INSERT INTO tbl2 SELECT * WHERE jt.id > 4""".stripMargin
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'INSERT'", "hint" -> ""))
val sql3 = "ALTER VIEW testView AS INSERT INTO jt VALUES(1, 1)"
checkError(
exception = parseException(sql3),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'INSERT'", "hint" -> ""))
// Multi insert query
@@ -1387,7 +1387,7 @@ class PlanParserSuite extends AnalysisTest {
|INSERT INTO tbl2 SELECT * WHERE jt.id > 4""".stripMargin
checkError(
exception = parseException(sql4),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'INSERT'", "hint" -> ""))
}
@@ -1395,13 +1395,13 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "SELECT * FROM (INSERT INTO BAR VALUES (2))"
checkError(
exception = parseException(sql1),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'BAR'", "hint" -> ": missing ')'"))
val sql2 = "SELECT * FROM S WHERE C1 IN (INSERT INTO T VALUES (2))"
checkError(
exception = parseException(sql2),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'IN'", "hint" -> ""))
}
@@ -1506,7 +1506,7 @@ class PlanParserSuite extends AnalysisTest {
val sql1 = "select * from my_tvf(arg1 => table v1)"
checkError(
exception = parseException(sql1),
- errorClass =
+ condition =
"INVALID_SQL_SYNTAX.INVALID_TABLE_FUNCTION_IDENTIFIER_ARGUMENT_MISSING_PARENTHESES",
parameters = Map("argumentName" -> "`v1`"),
context = ExpectedContext(
@@ -1627,14 +1627,14 @@ class PlanParserSuite extends AnalysisTest {
val sql6 = "select * from my_tvf(arg1 => table(1) partition by col1 with single partition)"
checkError(
exception = parseException(sql6),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map(
"error" -> "'partition'",
"hint" -> ""))
val sql7 = "select * from my_tvf(arg1 => table(1) order by col1)"
checkError(
exception = parseException(sql7),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map(
"error" -> "'order'",
"hint" -> ""))
@@ -1643,7 +1643,7 @@ class PlanParserSuite extends AnalysisTest {
val sql8 = s"select * from my_tvf(arg1 => $sql8tableArg $sql8partition)"
checkError(
exception = parseException(sql8),
- errorClass = "_LEGACY_ERROR_TEMP_0064",
+ condition = "_LEGACY_ERROR_TEMP_0064",
parameters = Map(
"msg" ->
("The table function call includes a table argument with an invalid " +
@@ -1766,7 +1766,7 @@ class PlanParserSuite extends AnalysisTest {
|FROM testData""".stripMargin
checkError(
exception = parseException(sql),
- errorClass = "UNSUPPORTED_FEATURE.TRANSFORM_NON_HIVE",
+ condition = "UNSUPPORTED_FEATURE.TRANSFORM_NON_HIVE",
parameters = Map.empty,
context = ExpectedContext(
fragment = sql,
@@ -1824,7 +1824,7 @@ class PlanParserSuite extends AnalysisTest {
val fragment = "TIMESTAMP AS OF col"
checkError(
exception = parseException(sql),
- errorClass = "_LEGACY_ERROR_TEMP_0056",
+ condition = "_LEGACY_ERROR_TEMP_0056",
parameters = Map("reason" -> "timestamp expression cannot refer to any columns"),
context = ExpectedContext(
fragment = fragment,
@@ -1919,11 +1919,11 @@ class PlanParserSuite extends AnalysisTest {
// Invalid empty name and invalid symbol in a name
checkError(
exception = parseException(s"SELECT :-"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'-'", "hint" -> ""))
checkError(
exception = parseException(s"SELECT :"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "end of input", "hint" -> ""))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingParserSuite.scala
index 5fc3ade408bd9..ba634333e06fb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/SqlScriptingParserSuite.scala
@@ -18,8 +18,10 @@
package org.apache.spark.sql.catalyst.parser
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.expressions.{Alias, EqualTo, Expression, In, Literal, ScalarSubquery}
import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.plans.logical.CreateVariable
+import org.apache.spark.sql.catalyst.plans.logical.{CreateVariable, Project}
+import org.apache.spark.sql.errors.DataTypeErrors.toSQLId
import org.apache.spark.sql.exceptions.SqlScriptingException
class SqlScriptingParserSuite extends SparkFunSuite with SQLHelper {
@@ -205,13 +207,14 @@ class SqlScriptingParserSuite extends SparkFunSuite with SQLHelper {
| SELECT a, b, c FROM T;
| SELECT * FROM T;
|END lbl_end""".stripMargin
-
+ val exception = intercept[SqlScriptingException] {
+ parseScript(sqlScriptText)
+ }
checkError(
- exception = intercept[SqlScriptingException] {
- parseScript(sqlScriptText)
- },
- errorClass = "LABELS_MISMATCH",
- parameters = Map("beginLabel" -> "lbl_begin", "endLabel" -> "lbl_end"))
+ exception = exception,
+ condition = "LABELS_MISMATCH",
+ parameters = Map("beginLabel" -> toSQLId("lbl_begin"), "endLabel" -> toSQLId("lbl_end")))
+ assert(exception.origin.line.contains(2))
}
test("compound: endLabel") {
@@ -224,13 +227,14 @@ class SqlScriptingParserSuite extends SparkFunSuite with SQLHelper {
| SELECT a, b, c FROM T;
| SELECT * FROM T;
|END lbl""".stripMargin
-
+ val exception = intercept[SqlScriptingException] {
+ parseScript(sqlScriptText)
+ }
checkError(
- exception = intercept[SqlScriptingException] {
- parseScript(sqlScriptText)
- },
- errorClass = "END_LABEL_WITHOUT_BEGIN_LABEL",
- parameters = Map("endLabel" -> "lbl"))
+ exception = exception,
+ condition = "END_LABEL_WITHOUT_BEGIN_LABEL",
+ parameters = Map("endLabel" -> toSQLId("lbl")))
+ assert(exception.origin.line.contains(8))
}
test("compound: beginLabel + endLabel with different casing") {
@@ -286,12 +290,14 @@ class SqlScriptingParserSuite extends SparkFunSuite with SQLHelper {
| SELECT 1;
| DECLARE testVariable INTEGER;
|END""".stripMargin
+ val exception = intercept[SqlScriptingException] {
+ parseScript(sqlScriptText)
+ }
checkError(
- exception = intercept[SqlScriptingException] {
- parseScript(sqlScriptText)
- },
- errorClass = "INVALID_VARIABLE_DECLARATION.ONLY_AT_BEGINNING",
- parameters = Map("varName" -> "`testVariable`", "lineNumber" -> "4"))
+ exception = exception,
+ condition = "INVALID_VARIABLE_DECLARATION.ONLY_AT_BEGINNING",
+ parameters = Map("varName" -> "`testVariable`"))
+ assert(exception.origin.line.contains(4))
}
test("declare in wrong scope") {
@@ -302,12 +308,14 @@ class SqlScriptingParserSuite extends SparkFunSuite with SQLHelper {
| DECLARE testVariable INTEGER;
| END IF;
|END""".stripMargin
+ val exception = intercept[SqlScriptingException] {
+ parseScript(sqlScriptText)
+ }
checkError(
- exception = intercept[SqlScriptingException] {
- parseScript(sqlScriptText)
- },
- errorClass = "INVALID_VARIABLE_DECLARATION.NOT_ALLOWED_IN_SCOPE",
- parameters = Map("varName" -> "`testVariable`", "lineNumber" -> "4"))
+ exception = exception,
+ condition = "INVALID_VARIABLE_DECLARATION.NOT_ALLOWED_IN_SCOPE",
+ parameters = Map("varName" -> "`testVariable`"))
+ assert(exception.origin.line.contains(4))
}
test("SET VAR statement test") {
@@ -666,7 +674,730 @@ class SqlScriptingParserSuite extends SparkFunSuite with SQLHelper {
head.asInstanceOf[SingleStatement].getText == "SELECT 42")
assert(whileStmt.label.contains("lbl"))
+ }
+
+ test("leave compound block") {
+ val sqlScriptText =
+ """
+ |lbl: BEGIN
+ | SELECT 1;
+ | LEAVE lbl;
+ |END""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 2)
+ assert(tree.collection.head.isInstanceOf[SingleStatement])
+ assert(tree.collection(1).isInstanceOf[LeaveStatement])
+ }
+
+ test("leave while loop") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | lbl: WHILE 1 = 1 DO
+ | SELECT 1;
+ | LEAVE lbl;
+ | END WHILE;
+ |END""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[WhileStatement])
+
+ val whileStmt = tree.collection.head.asInstanceOf[WhileStatement]
+ assert(whileStmt.condition.isInstanceOf[SingleStatement])
+ assert(whileStmt.condition.getText == "1 = 1")
+
+ assert(whileStmt.body.isInstanceOf[CompoundBody])
+ assert(whileStmt.body.collection.length == 2)
+
+ assert(whileStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(whileStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 1")
+
+ assert(whileStmt.body.collection(1).isInstanceOf[LeaveStatement])
+ assert(whileStmt.body.collection(1).asInstanceOf[LeaveStatement].label == "lbl")
+ }
+
+ test("leave repeat loop") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | lbl: REPEAT
+ | SELECT 1;
+ | LEAVE lbl;
+ | UNTIL 1 = 2
+ | END REPEAT;
+ |END""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[RepeatStatement])
+
+ val repeatStmt = tree.collection.head.asInstanceOf[RepeatStatement]
+ assert(repeatStmt.condition.isInstanceOf[SingleStatement])
+ assert(repeatStmt.condition.getText == "1 = 2")
+
+ assert(repeatStmt.body.isInstanceOf[CompoundBody])
+ assert(repeatStmt.body.collection.length == 2)
+
+ assert(repeatStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(repeatStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 1")
+
+ assert(repeatStmt.body.collection(1).isInstanceOf[LeaveStatement])
+ assert(repeatStmt.body.collection(1).asInstanceOf[LeaveStatement].label == "lbl")
+ }
+
+ test ("iterate compound block - should fail") {
+ val sqlScriptText =
+ """
+ |lbl: BEGIN
+ | SELECT 1;
+ | ITERATE lbl;
+ |END""".stripMargin
+ checkError(
+ exception = intercept[SqlScriptingException] {
+ parseScript(sqlScriptText)
+ },
+ condition = "INVALID_LABEL_USAGE.ITERATE_IN_COMPOUND",
+ parameters = Map("labelName" -> "LBL"))
+ }
+
+ test("iterate while loop") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | lbl: WHILE 1 = 1 DO
+ | SELECT 1;
+ | ITERATE lbl;
+ | END WHILE;
+ |END""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[WhileStatement])
+
+ val whileStmt = tree.collection.head.asInstanceOf[WhileStatement]
+ assert(whileStmt.condition.isInstanceOf[SingleStatement])
+ assert(whileStmt.condition.getText == "1 = 1")
+
+ assert(whileStmt.body.isInstanceOf[CompoundBody])
+ assert(whileStmt.body.collection.length == 2)
+
+ assert(whileStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(whileStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 1")
+
+ assert(whileStmt.body.collection(1).isInstanceOf[IterateStatement])
+ assert(whileStmt.body.collection(1).asInstanceOf[IterateStatement].label == "lbl")
+ }
+
+ test("iterate repeat loop") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | lbl: REPEAT
+ | SELECT 1;
+ | ITERATE lbl;
+ | UNTIL 1 = 2
+ | END REPEAT;
+ |END""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[RepeatStatement])
+
+ val repeatStmt = tree.collection.head.asInstanceOf[RepeatStatement]
+ assert(repeatStmt.condition.isInstanceOf[SingleStatement])
+ assert(repeatStmt.condition.getText == "1 = 2")
+
+ assert(repeatStmt.body.isInstanceOf[CompoundBody])
+ assert(repeatStmt.body.collection.length == 2)
+
+ assert(repeatStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(repeatStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 1")
+
+ assert(repeatStmt.body.collection(1).isInstanceOf[IterateStatement])
+ assert(repeatStmt.body.collection(1).asInstanceOf[IterateStatement].label == "lbl")
+ }
+
+ test("leave with wrong label - should fail") {
+ val sqlScriptText =
+ """
+ |lbl: BEGIN
+ | SELECT 1;
+ | LEAVE randomlbl;
+ |END""".stripMargin
+ checkError(
+ exception = intercept[SqlScriptingException] {
+ parseScript(sqlScriptText)
+ },
+ condition = "INVALID_LABEL_USAGE.DOES_NOT_EXIST",
+ parameters = Map("labelName" -> "RANDOMLBL", "statementType" -> "LEAVE"))
+ }
+
+ test("iterate with wrong label - should fail") {
+ val sqlScriptText =
+ """
+ |lbl: BEGIN
+ | SELECT 1;
+ | ITERATE randomlbl;
+ |END""".stripMargin
+ checkError(
+ exception = intercept[SqlScriptingException] {
+ parseScript(sqlScriptText)
+ },
+ condition = "INVALID_LABEL_USAGE.DOES_NOT_EXIST",
+ parameters = Map("labelName" -> "RANDOMLBL", "statementType" -> "ITERATE"))
+ }
+
+ test("leave outer loop from nested while loop") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | lbl: WHILE 1 = 1 DO
+ | lbl2: WHILE 2 = 2 DO
+ | SELECT 1;
+ | LEAVE lbl;
+ | END WHILE;
+ | END WHILE;
+ |END""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[WhileStatement])
+
+ val whileStmt = tree.collection.head.asInstanceOf[WhileStatement]
+ assert(whileStmt.condition.isInstanceOf[SingleStatement])
+ assert(whileStmt.condition.getText == "1 = 1")
+
+ assert(whileStmt.body.isInstanceOf[CompoundBody])
+ assert(whileStmt.body.collection.length == 1)
+
+ val nestedWhileStmt = whileStmt.body.collection.head.asInstanceOf[WhileStatement]
+ assert(nestedWhileStmt.condition.isInstanceOf[SingleStatement])
+ assert(nestedWhileStmt.condition.getText == "2 = 2")
+
+ assert(nestedWhileStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(nestedWhileStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 1")
+
+ assert(nestedWhileStmt.body.collection(1).isInstanceOf[LeaveStatement])
+ assert(nestedWhileStmt.body.collection(1).asInstanceOf[LeaveStatement].label == "lbl")
+ }
+
+ test("leave outer loop from nested repeat loop") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | lbl: REPEAT
+ | lbl2: REPEAT
+ | SELECT 1;
+ | LEAVE lbl;
+ | UNTIL 2 = 2
+ | END REPEAT;
+ | UNTIL 1 = 1
+ | END REPEAT;
+ |END""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[RepeatStatement])
+
+ val repeatStmt = tree.collection.head.asInstanceOf[RepeatStatement]
+ assert(repeatStmt.condition.isInstanceOf[SingleStatement])
+ assert(repeatStmt.condition.getText == "1 = 1")
+
+ assert(repeatStmt.body.isInstanceOf[CompoundBody])
+ assert(repeatStmt.body.collection.length == 1)
+
+ val nestedRepeatStmt = repeatStmt.body.collection.head.asInstanceOf[RepeatStatement]
+ assert(nestedRepeatStmt.condition.isInstanceOf[SingleStatement])
+ assert(nestedRepeatStmt.condition.getText == "2 = 2")
+
+ assert(nestedRepeatStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(
+ nestedRepeatStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 1")
+
+ assert(nestedRepeatStmt.body.collection(1).isInstanceOf[LeaveStatement])
+ assert(nestedRepeatStmt.body.collection(1).asInstanceOf[LeaveStatement].label == "lbl")
+ }
+
+ test("iterate outer loop from nested while loop") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | lbl: WHILE 1 = 1 DO
+ | lbl2: WHILE 2 = 2 DO
+ | SELECT 1;
+ | ITERATE lbl;
+ | END WHILE;
+ | END WHILE;
+ |END""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[WhileStatement])
+
+ val whileStmt = tree.collection.head.asInstanceOf[WhileStatement]
+ assert(whileStmt.condition.isInstanceOf[SingleStatement])
+ assert(whileStmt.condition.getText == "1 = 1")
+
+ assert(whileStmt.body.isInstanceOf[CompoundBody])
+ assert(whileStmt.body.collection.length == 1)
+
+ val nestedWhileStmt = whileStmt.body.collection.head.asInstanceOf[WhileStatement]
+ assert(nestedWhileStmt.condition.isInstanceOf[SingleStatement])
+ assert(nestedWhileStmt.condition.getText == "2 = 2")
+
+ assert(nestedWhileStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(nestedWhileStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 1")
+
+ assert(nestedWhileStmt.body.collection(1).isInstanceOf[IterateStatement])
+ assert(nestedWhileStmt.body.collection(1).asInstanceOf[IterateStatement].label == "lbl")
+ }
+
+ test("iterate outer loop from nested repeat loop") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | lbl: REPEAT
+ | lbl2: REPEAT
+ | SELECT 1;
+ | ITERATE lbl;
+ | UNTIL 2 = 2
+ | END REPEAT;
+ | UNTIL 1 = 1
+ | END REPEAT;
+ |END""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[RepeatStatement])
+
+ val repeatStmt = tree.collection.head.asInstanceOf[RepeatStatement]
+ assert(repeatStmt.condition.isInstanceOf[SingleStatement])
+ assert(repeatStmt.condition.getText == "1 = 1")
+
+ assert(repeatStmt.body.isInstanceOf[CompoundBody])
+ assert(repeatStmt.body.collection.length == 1)
+
+ val nestedRepeatStmt = repeatStmt.body.collection.head.asInstanceOf[RepeatStatement]
+ assert(nestedRepeatStmt.condition.isInstanceOf[SingleStatement])
+ assert(nestedRepeatStmt.condition.getText == "2 = 2")
+
+ assert(nestedRepeatStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(
+ nestedRepeatStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 1")
+
+ assert(nestedRepeatStmt.body.collection(1).isInstanceOf[IterateStatement])
+ assert(nestedRepeatStmt.body.collection(1).asInstanceOf[IterateStatement].label == "lbl")
+ }
+
+ test("repeat") {
+ val sqlScriptText =
+ """BEGIN
+ |lbl: REPEAT
+ | SELECT 1;
+ | UNTIL 1 = 1
+ |END REPEAT lbl;
+ |END
+ """.stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[RepeatStatement])
+
+ val repeatStmt = tree.collection.head.asInstanceOf[RepeatStatement]
+ assert(repeatStmt.condition.isInstanceOf[SingleStatement])
+ assert(repeatStmt.condition.getText == "1 = 1")
+
+ assert(repeatStmt.body.isInstanceOf[CompoundBody])
+ assert(repeatStmt.body.collection.length == 1)
+ assert(repeatStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(repeatStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 1")
+
+ assert(repeatStmt.label.contains("lbl"))
+ }
+
+ test("repeat with complex condition") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ |CREATE TABLE t (a INT, b STRING, c DOUBLE) USING parquet;
+ |REPEAT
+ | SELECT 42;
+ |UNTIL
+ | (SELECT COUNT(*) < 2 FROM t)
+ |END REPEAT;
+ |END
+ |""".stripMargin
+
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 2)
+ assert(tree.collection(1).isInstanceOf[RepeatStatement])
+
+ val repeatStmt = tree.collection(1).asInstanceOf[RepeatStatement]
+ assert(repeatStmt.condition.isInstanceOf[SingleStatement])
+ assert(repeatStmt.condition.getText == "(SELECT COUNT(*) < 2 FROM t)")
+
+ assert(repeatStmt.body.isInstanceOf[CompoundBody])
+ assert(repeatStmt.body.collection.length == 1)
+ assert(repeatStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(repeatStmt.body.collection.head.asInstanceOf[SingleStatement].getText == "SELECT 42")
+ }
+
+ test("repeat with if else block") {
+ val sqlScriptText =
+ """BEGIN
+ |lbl: REPEAT
+ | IF 1 = 1 THEN
+ | SELECT 1;
+ | ELSE
+ | SELECT 2;
+ | END IF;
+ |UNTIL
+ | 1 = 1
+ |END REPEAT lbl;
+ |END
+ """.stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[RepeatStatement])
+
+ val whileStmt = tree.collection.head.asInstanceOf[RepeatStatement]
+ assert(whileStmt.condition.isInstanceOf[SingleStatement])
+ assert(whileStmt.condition.getText == "1 = 1")
+
+ assert(whileStmt.body.isInstanceOf[CompoundBody])
+ assert(whileStmt.body.collection.length == 1)
+ assert(whileStmt.body.collection.head.isInstanceOf[IfElseStatement])
+ val ifStmt = whileStmt.body.collection.head.asInstanceOf[IfElseStatement]
+
+ assert(ifStmt.conditions.length == 1)
+ assert(ifStmt.conditionalBodies.length == 1)
+ assert(ifStmt.elseBody.isDefined)
+
+ assert(ifStmt.conditions.head.isInstanceOf[SingleStatement])
+ assert(ifStmt.conditions.head.getText == "1 = 1")
+
+ assert(ifStmt.conditionalBodies.head.collection.head.isInstanceOf[SingleStatement])
+ assert(ifStmt.conditionalBodies.head.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 1")
+
+ assert(ifStmt.elseBody.get.collection.head.isInstanceOf[SingleStatement])
+ assert(ifStmt.elseBody.get.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 2")
+
+ assert(whileStmt.label.contains("lbl"))
+ }
+
+ test("nested repeat") {
+ val sqlScriptText =
+ """BEGIN
+ |lbl: REPEAT
+ | REPEAT
+ | SELECT 42;
+ | UNTIL
+ | 2 = 2
+ | END REPEAT;
+ |UNTIL
+ | 1 = 1
+ |END REPEAT lbl;
+ |END
+ """.stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[RepeatStatement])
+
+ val whileStmt = tree.collection.head.asInstanceOf[RepeatStatement]
+ assert(whileStmt.condition.isInstanceOf[SingleStatement])
+ assert(whileStmt.condition.getText == "1 = 1")
+
+ assert(whileStmt.body.isInstanceOf[CompoundBody])
+ assert(whileStmt.body.collection.length == 1)
+ assert(whileStmt.body.collection.head.isInstanceOf[RepeatStatement])
+ val nestedWhileStmt = whileStmt.body.collection.head.asInstanceOf[RepeatStatement]
+
+ assert(nestedWhileStmt.condition.isInstanceOf[SingleStatement])
+ assert(nestedWhileStmt.condition.getText == "2 = 2")
+
+ assert(nestedWhileStmt.body.isInstanceOf[CompoundBody])
+ assert(nestedWhileStmt.body.collection.length == 1)
+ assert(nestedWhileStmt.body.collection.head.isInstanceOf[SingleStatement])
+ assert(nestedWhileStmt.body.collection.
+ head.asInstanceOf[SingleStatement].getText == "SELECT 42")
+
+ assert(whileStmt.label.contains("lbl"))
+
+ }
+
+ test("searched case statement") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | CASE
+ | WHEN 1 = 1 THEN
+ | SELECT 42;
+ | END CASE;
+ |END
+ |""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[CaseStatement])
+ val caseStmt = tree.collection.head.asInstanceOf[CaseStatement]
+ assert(caseStmt.conditions.length == 1)
+ assert(caseStmt.conditions.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditions.head.getText == "1 = 1")
+ }
+
+ test("searched case statement - multi when") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | CASE
+ | WHEN 1 IN (1,2,3) THEN
+ | SELECT 1;
+ | WHEN (SELECT * FROM t) THEN
+ | SELECT * FROM b;
+ | WHEN 1 = 1 THEN
+ | SELECT 42;
+ | END CASE;
+ |END
+ |""".stripMargin
+ val tree = parseScript(sqlScriptText)
+
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[CaseStatement])
+
+ val caseStmt = tree.collection.head.asInstanceOf[CaseStatement]
+ assert(caseStmt.conditions.length == 3)
+ assert(caseStmt.conditionalBodies.length == 3)
+ assert(caseStmt.elseBody.isEmpty)
+
+ assert(caseStmt.conditions.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditions.head.getText == "1 IN (1,2,3)")
+
+ assert(caseStmt.conditionalBodies.head.collection.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditionalBodies.head.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 1")
+
+ assert(caseStmt.conditions(1).isInstanceOf[SingleStatement])
+ assert(caseStmt.conditions(1).getText == "(SELECT * FROM t)")
+
+ assert(caseStmt.conditionalBodies(1).collection.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditionalBodies(1).collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT * FROM b")
+
+ assert(caseStmt.conditions(2).isInstanceOf[SingleStatement])
+ assert(caseStmt.conditions(2).getText == "1 = 1")
+
+ assert(caseStmt.conditionalBodies(2).collection.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditionalBodies(2).collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 42")
+ }
+
+ test("searched case statement with else") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | CASE
+ | WHEN 1 = 1 THEN
+ | SELECT 42;
+ | ELSE
+ | SELECT 43;
+ | END CASE;
+ |END
+ |""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[CaseStatement])
+ val caseStmt = tree.collection.head.asInstanceOf[CaseStatement]
+ assert(caseStmt.elseBody.isDefined)
+ assert(caseStmt.conditions.length == 1)
+ assert(caseStmt.conditions.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditions.head.getText == "1 = 1")
+
+ assert(caseStmt.elseBody.get.collection.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.elseBody.get.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 43")
+ }
+ test("searched case statement nested") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | CASE
+ | WHEN 1 = 1 THEN
+ | CASE
+ | WHEN 2 = 1 THEN
+ | SELECT 41;
+ | ELSE
+ | SELECT 42;
+ | END CASE;
+ | END CASE;
+ |END
+ |""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[CaseStatement])
+
+ val caseStmt = tree.collection.head.asInstanceOf[CaseStatement]
+ assert(caseStmt.conditions.length == 1)
+ assert(caseStmt.conditionalBodies.length == 1)
+ assert(caseStmt.elseBody.isEmpty)
+
+ assert(caseStmt.conditions.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditions.head.getText == "1 = 1")
+
+ assert(caseStmt.conditionalBodies.head.collection.head.isInstanceOf[CaseStatement])
+ val nestedCaseStmt =
+ caseStmt.conditionalBodies.head.collection.head.asInstanceOf[CaseStatement]
+
+ assert(nestedCaseStmt.conditions.length == 1)
+ assert(nestedCaseStmt.conditionalBodies.length == 1)
+ assert(nestedCaseStmt.elseBody.isDefined)
+
+ assert(nestedCaseStmt.conditions.head.isInstanceOf[SingleStatement])
+ assert(nestedCaseStmt.conditions.head.getText == "2 = 1")
+
+ assert(nestedCaseStmt.conditionalBodies.head.collection.head.isInstanceOf[SingleStatement])
+ assert(nestedCaseStmt.conditionalBodies.head.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 41")
+
+ assert(nestedCaseStmt.elseBody.get.collection.head.isInstanceOf[SingleStatement])
+ assert(nestedCaseStmt.elseBody.get.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 42")
+ }
+
+ test("simple case statement") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | CASE 1
+ | WHEN 1 THEN
+ | SELECT 1;
+ | END CASE;
+ |END
+ |""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[CaseStatement])
+ val caseStmt = tree.collection.head.asInstanceOf[CaseStatement]
+ assert(caseStmt.conditions.length == 1)
+ assert(caseStmt.conditions.head.isInstanceOf[SingleStatement])
+ checkSimpleCaseStatementCondition(caseStmt.conditions.head, _ == Literal(1), _ == Literal(1))
+ }
+
+
+ test("simple case statement - multi when") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | CASE 1
+ | WHEN 1 THEN
+ | SELECT 1;
+ | WHEN (SELECT 2) THEN
+ | SELECT * FROM b;
+ | WHEN 3 IN (1,2,3) THEN
+ | SELECT 42;
+ | END CASE;
+ |END
+ |""".stripMargin
+ val tree = parseScript(sqlScriptText)
+
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[CaseStatement])
+
+ val caseStmt = tree.collection.head.asInstanceOf[CaseStatement]
+ assert(caseStmt.conditions.length == 3)
+ assert(caseStmt.conditionalBodies.length == 3)
+ assert(caseStmt.elseBody.isEmpty)
+
+ assert(caseStmt.conditions.head.isInstanceOf[SingleStatement])
+ checkSimpleCaseStatementCondition(caseStmt.conditions.head, _ == Literal(1), _ == Literal(1))
+
+ assert(caseStmt.conditionalBodies.head.collection.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditionalBodies.head.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 1")
+
+ assert(caseStmt.conditions(1).isInstanceOf[SingleStatement])
+ checkSimpleCaseStatementCondition(
+ caseStmt.conditions(1), _ == Literal(1), _.isInstanceOf[ScalarSubquery])
+
+ assert(caseStmt.conditionalBodies(1).collection.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditionalBodies(1).collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT * FROM b")
+
+ assert(caseStmt.conditions(2).isInstanceOf[SingleStatement])
+ checkSimpleCaseStatementCondition(
+ caseStmt.conditions(2), _ == Literal(1), _.isInstanceOf[In])
+
+ assert(caseStmt.conditionalBodies(2).collection.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.conditionalBodies(2).collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 42")
+ }
+
+ test("simple case statement with else") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | CASE 1
+ | WHEN 1 THEN
+ | SELECT 42;
+ | ELSE
+ | SELECT 43;
+ | END CASE;
+ |END
+ |""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[CaseStatement])
+ val caseStmt = tree.collection.head.asInstanceOf[CaseStatement]
+ assert(caseStmt.elseBody.isDefined)
+ assert(caseStmt.conditions.length == 1)
+ assert(caseStmt.conditions.head.isInstanceOf[SingleStatement])
+ checkSimpleCaseStatementCondition(caseStmt.conditions.head, _ == Literal(1), _ == Literal(1))
+
+ assert(caseStmt.elseBody.get.collection.head.isInstanceOf[SingleStatement])
+ assert(caseStmt.elseBody.get.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 43")
+ }
+
+ test("simple case statement nested") {
+ val sqlScriptText =
+ """
+ |BEGIN
+ | CASE (SELECT 1)
+ | WHEN 1 THEN
+ | CASE 2
+ | WHEN 2 THEN
+ | SELECT 41;
+ | ELSE
+ | SELECT 42;
+ | END CASE;
+ | END CASE;
+ |END
+ |""".stripMargin
+ val tree = parseScript(sqlScriptText)
+ assert(tree.collection.length == 1)
+ assert(tree.collection.head.isInstanceOf[CaseStatement])
+
+ val caseStmt = tree.collection.head.asInstanceOf[CaseStatement]
+ assert(caseStmt.conditions.length == 1)
+ assert(caseStmt.conditionalBodies.length == 1)
+ assert(caseStmt.elseBody.isEmpty)
+
+ assert(caseStmt.conditions.head.isInstanceOf[SingleStatement])
+ checkSimpleCaseStatementCondition(
+ caseStmt.conditions.head, _.isInstanceOf[ScalarSubquery], _ == Literal(1))
+
+ assert(caseStmt.conditionalBodies.head.collection.head.isInstanceOf[CaseStatement])
+ val nestedCaseStmt =
+ caseStmt.conditionalBodies.head.collection.head.asInstanceOf[CaseStatement]
+
+ assert(nestedCaseStmt.conditions.length == 1)
+ assert(nestedCaseStmt.conditionalBodies.length == 1)
+ assert(nestedCaseStmt.elseBody.isDefined)
+
+ assert(nestedCaseStmt.conditions.head.isInstanceOf[SingleStatement])
+ checkSimpleCaseStatementCondition(
+ nestedCaseStmt.conditions.head, _ == Literal(2), _ == Literal(2))
+
+ assert(nestedCaseStmt.conditionalBodies.head.collection.head.isInstanceOf[SingleStatement])
+ assert(nestedCaseStmt.conditionalBodies.head.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 41")
+
+ assert(nestedCaseStmt.elseBody.get.collection.head.isInstanceOf[SingleStatement])
+ assert(nestedCaseStmt.elseBody.get.collection.head.asInstanceOf[SingleStatement]
+ .getText == "SELECT 42")
}
// Helper methods
@@ -677,4 +1408,17 @@ class SqlScriptingParserSuite extends SparkFunSuite with SQLHelper {
.replace("END", "")
.trim
}
+
+ private def checkSimpleCaseStatementCondition(
+ conditionStatement: SingleStatement,
+ predicateLeft: Expression => Boolean,
+ predicateRight: Expression => Boolean): Unit = {
+ assert(conditionStatement.parsedPlan.isInstanceOf[Project])
+ val project = conditionStatement.parsedPlan.asInstanceOf[Project]
+ assert(project.projectList.head.isInstanceOf[Alias])
+ assert(project.projectList.head.asInstanceOf[Alias].child.isInstanceOf[EqualTo])
+ val equalTo = project.projectList.head.asInstanceOf[Alias].child.asInstanceOf[EqualTo]
+ assert(predicateLeft(equalTo.left))
+ assert(predicateRight(equalTo.right))
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index 62557ead1d2ee..0f32922728814 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -296,10 +296,10 @@ class TableIdentifierParserSuite extends SQLKeywordUtils {
"t:" -> ("PARSE_SYNTAX_ERROR", Map("error" -> "':'", "hint" -> ": extra input ':'")),
"${some.var.x}" -> ("PARSE_SYNTAX_ERROR", Map("error" -> "'$'", "hint" -> "")),
"tab:1" -> ("PARSE_SYNTAX_ERROR", Map("error" -> "':'", "hint" -> ""))
- ).foreach { case (identifier, (errorClass, parameters)) =>
+ ).foreach { case (identifier, (condition, parameters)) =>
checkError(
exception = intercept[ParseException](parseTableIdentifier(identifier)),
- errorClass = errorClass,
+ condition = condition,
parameters = parameters)
}
}
@@ -318,7 +318,7 @@ class TableIdentifierParserSuite extends SQLKeywordUtils {
reservedKeywordsInAnsiMode.foreach { keyword =>
checkError(
exception = intercept[ParseException](parseTableIdentifier(keyword)),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> s"'$keyword'", "hint" -> ""))
assert(TableIdentifier(keyword) === parseTableIdentifier(s"`$keyword`"))
assert(TableIdentifier(keyword, Option("db")) === parseTableIdentifier(s"db.`$keyword`"))
@@ -374,7 +374,7 @@ class TableIdentifierParserSuite extends SQLKeywordUtils {
assert(complexName === parseTableIdentifier(complexName.quotedString))
checkError(
exception = intercept[ParseException](parseTableIdentifier(complexName.unquotedString)),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'b'", "hint" -> ""))
// Table identifier contains continuous backticks should be treated correctly.
val complexName2 = TableIdentifier("x``y", Some("d``b"))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableSchemaParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableSchemaParserSuite.scala
index a56ab8616df0f..74fb5a44ab0bc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableSchemaParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableSchemaParserSuite.scala
@@ -75,26 +75,26 @@ class TableSchemaParserSuite extends SparkFunSuite {
checkError(
exception = parseException(""),
- errorClass = "PARSE_EMPTY_STATEMENT")
+ condition = "PARSE_EMPTY_STATEMENT")
checkError(
exception = parseException("a"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "end of input", "hint" -> ""))
checkError(
exception = parseException("a INT b long"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "'b'", "hint" -> ""))
checkError(
exception = parseException("a INT,, b long"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "','", "hint" -> ""))
checkError(
exception = parseException("a INT, b long,,"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "','", "hint" -> ""))
checkError(
exception = parseException("a INT, b long, c int,"),
- errorClass = "PARSE_SYNTAX_ERROR",
+ condition = "PARSE_SYNTAX_ERROR",
parameters = Map("error" -> "end of input", "hint" -> ""))
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/UnpivotParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/UnpivotParserSuite.scala
index 3012ef6f1544d..3f59f8de95429 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/UnpivotParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/UnpivotParserSuite.scala
@@ -31,8 +31,8 @@ class UnpivotParserSuite extends AnalysisTest {
comparePlans(parsePlan(sqlCommand), plan, checkAnalysis = false)
}
- private def intercept(sqlCommand: String, errorClass: Option[String], messages: String*): Unit =
- interceptParseException(parsePlan)(sqlCommand, messages: _*)(errorClass)
+ private def intercept(sqlCommand: String, condition: Option[String], messages: String*): Unit =
+ interceptParseException(parsePlan)(sqlCommand, messages: _*)(condition)
test("unpivot - single value") {
assertEqual(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala
index 886b043ad79e6..7fa1935ccb058 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/JoinTypesTest.scala
@@ -68,7 +68,7 @@ class JoinTypesTest extends SparkFunSuite {
exception = intercept[AnalysisException](
JoinType(joinType)
),
- errorClass = "UNSUPPORTED_JOIN_TYPE",
+ condition = "UNSUPPORTED_JOIN_TYPE",
sqlState = "0A000",
parameters = Map(
"typ" -> joinType,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/streaming/InternalOutputModesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/streaming/InternalOutputModesSuite.scala
index 55afbc3acb096..3a739ccbecb64 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/streaming/InternalOutputModesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/streaming/InternalOutputModesSuite.scala
@@ -41,7 +41,7 @@ class InternalOutputModesSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
InternalOutputModes(outputMode)
},
- errorClass = "STREAMING_OUTPUT_MODE.INVALID",
+ condition = "STREAMING_OUTPUT_MODE.INVALID",
parameters = Map("outputMode" -> outputMode))
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
index 1d3fb835f5a77..0e872dcdb6262 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
@@ -43,7 +43,7 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
builder.put(1, null) // null value is OK
checkError(
exception = intercept[SparkRuntimeException](builder.put(null, 1)),
- errorClass = "NULL_MAP_KEY",
+ condition = "NULL_MAP_KEY",
parameters = Map.empty
)
}
@@ -53,7 +53,7 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
builder.put(1, 1)
checkError(
exception = intercept[SparkRuntimeException](builder.put(1, 2)),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "1",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -65,7 +65,7 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
builderDouble.put(-0.0, 1)
checkError(
exception = intercept[SparkRuntimeException](builderDouble.put(0.0, 2)),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "0.0",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -110,7 +110,7 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
val arr = Array(1.toByte)
checkError(
exception = intercept[SparkRuntimeException](builder.put(arr, 3)),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> arr.toString,
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -146,7 +146,7 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
// By default duplicated map key fails the query.
checkError(
exception = intercept[SparkRuntimeException](builder.put(unsafeRow, 3)),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> "[0,1]",
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
@@ -180,7 +180,7 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
// By default duplicated map key fails the query.
checkError(
exception = intercept[SparkRuntimeException](builder.put(unsafeArray, 3)),
- errorClass = "DUPLICATED_MAP_KEY",
+ condition = "DUPLICATED_MAP_KEY",
parameters = Map(
"key" -> unsafeArray.toString,
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
index d55e672079484..632109a0cc8d0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
@@ -58,7 +58,7 @@ class ArrayDataIndexedSeqSuite extends SparkFunSuite {
exception = intercept[SparkException] {
seq(index)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map(
"message" -> s"Index $index must be between 0 and the length of the ArrayData."))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
index 034010f5825b8..095dc3869571d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
@@ -42,7 +42,7 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
convertIncompatiblePattern(s"yyyy-MM-dd $l G")
},
- errorClass = "INCONSISTENT_BEHAVIOR_CROSS_VERSION.DATETIME_WEEK_BASED_PATTERN",
+ condition = "INCONSISTENT_BEHAVIOR_CROSS_VERSION.DATETIME_WEEK_BASED_PATTERN",
parameters = Map("c" -> l.toString))
}
unsupportedLetters.foreach { l =>
@@ -50,7 +50,7 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
convertIncompatiblePattern(s"yyyy-MM-dd $l G")
},
- errorClass = "INVALID_DATETIME_PATTERN.ILLEGAL_CHARACTER",
+ condition = "INVALID_DATETIME_PATTERN.ILLEGAL_CHARACTER",
parameters = Map(
"c" -> l.toString,
"pattern" -> s"yyyy-MM-dd $l G"))
@@ -60,7 +60,7 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
DateTimeFormatterHelper.convertIncompatiblePattern(s"$l", isParsing = true)
},
- errorClass = "INVALID_DATETIME_PATTERN.ILLEGAL_CHARACTER",
+ condition = "INVALID_DATETIME_PATTERN.ILLEGAL_CHARACTER",
parameters = Map(
"c" -> l.toString,
"pattern" -> s"$l"))
@@ -70,17 +70,27 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
convertIncompatiblePattern(s"yyyy-MM-dd $style")
},
- errorClass = "INVALID_DATETIME_PATTERN.LENGTH",
+ condition = "INVALID_DATETIME_PATTERN.LENGTH",
parameters = Map("pattern" -> style))
checkError(
exception = intercept[SparkIllegalArgumentException] {
convertIncompatiblePattern(s"yyyy-MM-dd $style${style.head}")
},
- errorClass = "INVALID_DATETIME_PATTERN.LENGTH",
+ condition = "INVALID_DATETIME_PATTERN.LENGTH",
parameters = Map("pattern" -> style))
}
assert(convertIncompatiblePattern("yyyy-MM-dd EEEE") === "uuuu-MM-dd EEEE")
assert(convertIncompatiblePattern("yyyy-MM-dd'e'HH:mm:ss") === "uuuu-MM-dd'e'HH:mm:ss")
assert(convertIncompatiblePattern("yyyy-MM-dd'T'") === "uuuu-MM-dd'T'")
}
+
+ test("SPARK-49583: invalid var length second fraction") {
+ val pattern = "\nSSSS\r"
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ createBuilderWithVarLengthSecondFraction(pattern)
+ },
+ condition = "INVALID_DATETIME_PATTERN.SECONDS_FRACTION",
+ parameters = Map("pattern" -> pattern))
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 8d8669aece894..96aaf13052b02 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -542,7 +542,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
checkError(
exception = intercept[SparkIllegalArgumentException](
dateAddInterval(input, new CalendarInterval(36, 47, 1))),
- errorClass = "_LEGACY_ERROR_TEMP_2000",
+ condition = "_LEGACY_ERROR_TEMP_2000",
parameters = Map(
"message" -> "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date",
"ansiConfig" -> "\"spark.sql.ansi.enabled\""))
@@ -896,13 +896,13 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
getDayOfWeekFromString(UTF8String.fromString("xx"))
},
- errorClass = "_LEGACY_ERROR_TEMP_3209",
+ condition = "ILLEGAL_DAY_OF_WEEK",
parameters = Map("string" -> "xx"))
checkError(
exception = intercept[SparkIllegalArgumentException] {
getDayOfWeekFromString(UTF8String.fromString("\"quote"))
},
- errorClass = "_LEGACY_ERROR_TEMP_3209",
+ condition = "ILLEGAL_DAY_OF_WEEK",
parameters = Map("string" -> "\"quote"))
}
@@ -1043,7 +1043,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
timestampAdd("SECS", 1, date(1969, 1, 1, 0, 0, 0, 1, getZoneId("UTC")), getZoneId("UTC"))
},
- errorClass = "INVALID_PARAMETER_VALUE.DATETIME_UNIT",
+ condition = "INVALID_PARAMETER_VALUE.DATETIME_UNIT",
parameters = Map(
"functionName" -> "`TIMESTAMPADD`",
"parameter" -> "`unit`",
@@ -1102,7 +1102,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
date(2022, 1, 1, 0, 0, 0, 1, getZoneId("UTC")),
getZoneId("UTC"))
},
- errorClass = "INVALID_PARAMETER_VALUE.DATETIME_UNIT",
+ condition = "INVALID_PARAMETER_VALUE.DATETIME_UNIT",
parameters =
Map("functionName" -> "`TIMESTAMPDIFF`",
"parameter" -> "`unit`",
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index 001ae2728d10f..700dfe30a2389 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -44,7 +44,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.INPUT_IS_NULL",
+ condition = "INVALID_INTERVAL_FORMAT.INPUT_IS_NULL",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null")))
assert(safeStringToInterval(UTF8String.fromString(input)) === null)
@@ -55,7 +55,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.INPUT_IS_EMPTY",
+ condition = "INVALID_INTERVAL_FORMAT.INPUT_IS_EMPTY",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null")))
assert(safeStringToInterval(UTF8String.fromString(input)) === null)
@@ -66,7 +66,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.INVALID_PREFIX",
+ condition = "INVALID_INTERVAL_FORMAT.INVALID_PREFIX",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null"),
"prefix" -> prefix))
@@ -78,7 +78,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.UNRECOGNIZED_NUMBER",
+ condition = "INVALID_INTERVAL_FORMAT.UNRECOGNIZED_NUMBER",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null"),
"number" -> number))
@@ -90,7 +90,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.ARITHMETIC_EXCEPTION",
+ condition = "INVALID_INTERVAL_FORMAT.ARITHMETIC_EXCEPTION",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null")))
assert(safeStringToInterval(UTF8String.fromString(input)) === null)
@@ -101,7 +101,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.INVALID_VALUE",
+ condition = "INVALID_INTERVAL_FORMAT.INVALID_VALUE",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null"),
"value" -> value))
@@ -113,7 +113,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.INVALID_PRECISION",
+ condition = "INVALID_INTERVAL_FORMAT.INVALID_PRECISION",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null"),
"value" -> value))
@@ -125,7 +125,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.INVALID_FRACTION",
+ condition = "INVALID_INTERVAL_FORMAT.INVALID_FRACTION",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null"),
"unit" -> unit))
@@ -137,7 +137,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.INVALID_UNIT",
+ condition = "INVALID_INTERVAL_FORMAT.INVALID_UNIT",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null"),
"unit" -> unit))
@@ -149,7 +149,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.MISSING_NUMBER",
+ condition = "INVALID_INTERVAL_FORMAT.MISSING_NUMBER",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null"),
"word" -> word))
@@ -161,7 +161,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.MISSING_UNIT",
+ condition = "INVALID_INTERVAL_FORMAT.MISSING_UNIT",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null"),
"word" -> word))
@@ -173,7 +173,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkIllegalArgumentException] {
stringToInterval(UTF8String.fromString(input))
},
- errorClass = "INVALID_INTERVAL_FORMAT.UNKNOWN_PARSING_ERROR",
+ condition = "INVALID_INTERVAL_FORMAT.UNKNOWN_PARSING_ERROR",
parameters = Map(
"input" -> Option(input).map(_.toString).getOrElse("null"),
"word" -> word))
@@ -295,7 +295,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
assert(fromYearMonthString("99-10") === new CalendarInterval(99 * 12 + 10, 0, 0L))
assert(fromYearMonthString("+99-10") === new CalendarInterval(99 * 12 + 10, 0, 0L))
assert(fromYearMonthString("-8-10") === new CalendarInterval(-8 * 12 - 10, 0, 0L))
- failFuncWithInvalidInput("99-15", "month 15 outside range", fromYearMonthString)
+ failFuncWithInvalidInput("99-15", "year-month", fromYearMonthString)
failFuncWithInvalidInput("9a9-15", "Interval string does not match year-month format",
fromYearMonthString)
@@ -314,12 +314,12 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
val e1 = intercept[IllegalArgumentException]{
assert(fromYearMonthString("178956970-8") == new CalendarInterval(Int.MinValue, 0, 0))
}.getMessage
- assert(e1.contains("integer overflow"))
+ assert(e1.contains("year-month"))
assert(fromYearMonthString("-178956970-8") == new CalendarInterval(Int.MinValue, 0, 0))
val e2 = intercept[IllegalArgumentException]{
assert(fromYearMonthString("-178956970-9") == new CalendarInterval(Int.MinValue, 0, 0))
}.getMessage
- assert(e2.contains("integer overflow"))
+ assert(e2.contains("year-month"))
}
test("from day-time string - legacy") {
@@ -338,6 +338,29 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
12 * MICROS_PER_MINUTE + millisToMicros(888)))
assert(fromDayTimeString("-3 0:0:0") === new CalendarInterval(0, -3, 0L))
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ fromDayTimeString("5 30:12:20")
+ },
+ parameters = Map(
+ "msg" -> "requirement failed: hour 30 outside range [0, 23]",
+ "input" -> "5 30:12:20"),
+ condition = "INVALID_INTERVAL_FORMAT.DAY_TIME_PARSING",
+ sqlState = Some("22006")
+ )
+
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ fromDayTimeString("5 12:40:30.999999999", 0, 0)
+ },
+ parameters = Map(
+ "from" -> "day",
+ "to" -> "day",
+ "input" -> "5 12:40:30.999999999"),
+ condition = "INVALID_INTERVAL_FORMAT.UNSUPPORTED_FROM_TO_EXPRESSION",
+ sqlState = Some("22006")
+ )
+
failFuncWithInvalidInput("5 30:12:20", "hour 30 outside range", fromDayTimeString)
failFuncWithInvalidInput("5 30-12", "must match day-time format", fromDayTimeString)
}
@@ -379,6 +402,17 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
assert(negate(new CalendarInterval(1, 2, 3)) === new CalendarInterval(-1, -2, -3))
}
+ test("parsing second_nano string") {
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ toDTInterval("12", "33.33.33", 1)
+ },
+ condition = "INVALID_INTERVAL_FORMAT.SECOND_NANO_FORMAT",
+ parameters = Map("input" -> "33.33.33"),
+ sqlState = Some("22006")
+ )
+ }
+
test("subtract one interval by another") {
val input1 = new CalendarInterval(3, 1, 1 * MICROS_PER_HOUR)
val input2 = new CalendarInterval(2, 4, 100 * MICROS_PER_HOUR)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
index 6223f9aadb593..558d7eda78b4a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
@@ -553,7 +553,7 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite {
exception = intercept[SparkException] {
formatter.parseWithoutTimeZone(invalidTimestampStr, allowTimeZone = false)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map(
"message" -> ("Cannot parse field value '2021-13-01T25:61:61' for pattern " +
"'yyyy-MM-dd HH:mm:ss' as the target spark data type \"TIMESTAMP_NTZ\"."))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala
index e20dfd4f60512..aca6931a0688d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala
@@ -362,7 +362,7 @@ class CatalogSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
catalog.alterTable(testIdent, TableChange.addColumn(Array("data", "ts"), TimestampType))
},
- errorClass = "_LEGACY_ERROR_TEMP_3229",
+ condition = "_LEGACY_ERROR_TEMP_3229",
parameters = Map("name" -> "data"))
// the table has not changed
@@ -381,7 +381,7 @@ class CatalogSuite extends SparkFunSuite {
catalog.alterTable(testIdent,
TableChange.addColumn(Array("missing_col", "new_field"), StringType))
},
- errorClass = "_LEGACY_ERROR_TEMP_3227",
+ condition = "_LEGACY_ERROR_TEMP_3227",
parameters = Map("fieldName" -> "missing_col"))
}
@@ -427,7 +427,7 @@ class CatalogSuite extends SparkFunSuite {
catalog.alterTable(testIdent,
TableChange.updateColumnType(Array("missing_col"), LongType))
},
- errorClass = "_LEGACY_ERROR_TEMP_3227",
+ condition = "_LEGACY_ERROR_TEMP_3227",
parameters = Map("fieldName" -> "missing_col"))
}
@@ -478,7 +478,7 @@ class CatalogSuite extends SparkFunSuite {
catalog.alterTable(testIdent,
TableChange.updateColumnComment(Array("missing_col"), "comment"))
},
- errorClass = "_LEGACY_ERROR_TEMP_3227",
+ condition = "_LEGACY_ERROR_TEMP_3227",
parameters = Map("fieldName" -> "missing_col"))
}
@@ -546,7 +546,7 @@ class CatalogSuite extends SparkFunSuite {
catalog.alterTable(testIdent,
TableChange.renameColumn(Array("missing_col"), "new_name"))
},
- errorClass = "_LEGACY_ERROR_TEMP_3227",
+ condition = "_LEGACY_ERROR_TEMP_3227",
parameters = Map("fieldName" -> "missing_col"))
}
@@ -614,7 +614,7 @@ class CatalogSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
catalog.alterTable(testIdent, TableChange.deleteColumn(Array("missing_col"), false))
},
- errorClass = "_LEGACY_ERROR_TEMP_3227",
+ condition = "_LEGACY_ERROR_TEMP_3227",
parameters = Map("fieldName" -> "missing_col"))
// with if exists it should pass
@@ -636,7 +636,7 @@ class CatalogSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
catalog.alterTable(testIdent, TableChange.deleteColumn(Array("point", "z"), false))
},
- errorClass = "_LEGACY_ERROR_TEMP_3227",
+ condition = "_LEGACY_ERROR_TEMP_3227",
parameters = Map("fieldName" -> "z"))
// with if exists it should pass
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala
index 8d8d2317f0986..411a88b8765f6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala
@@ -24,10 +24,13 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.sql.catalyst.analysis.{NoSuchFunctionException, NoSuchNamespaceException}
import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
+import org.apache.spark.sql.connector.catalog.procedures.UnboundProcedure
-class InMemoryCatalog extends InMemoryTableCatalog with FunctionCatalog {
+class InMemoryCatalog extends InMemoryTableCatalog with FunctionCatalog with ProcedureCatalog {
protected val functions: util.Map[Identifier, UnboundFunction] =
new ConcurrentHashMap[Identifier, UnboundFunction]()
+ protected val procedures: util.Map[Identifier, UnboundProcedure] =
+ new ConcurrentHashMap[Identifier, UnboundProcedure]()
override protected def allNamespaces: Seq[Seq[String]] = {
(tables.keySet.asScala.map(_.namespace.toSeq) ++
@@ -63,4 +66,18 @@ class InMemoryCatalog extends InMemoryTableCatalog with FunctionCatalog {
def clearFunctions(): Unit = {
functions.clear()
}
+
+ override def loadProcedure(ident: Identifier): UnboundProcedure = {
+ val procedure = procedures.get(ident)
+ if (procedure == null) throw new RuntimeException("Procedure not found: " + ident)
+ procedure
+ }
+
+ def createProcedure(ident: Identifier, procedure: UnboundProcedure): UnboundProcedure = {
+ procedures.put(ident, procedure)
+ }
+
+ def clearProcedures(): Unit = {
+ procedures.clear()
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala
index 982de88e58847..56ed3bb243e19 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala
@@ -167,7 +167,8 @@ class InMemoryTableCatalog extends BasicInMemoryTableCatalog with SupportsNamesp
override def capabilities: java.util.Set[TableCatalogCapability] = {
Set(
TableCatalogCapability.SUPPORT_COLUMN_DEFAULT_VALUE,
- TableCatalogCapability.SUPPORTS_CREATE_TABLE_WITH_GENERATED_COLUMNS
+ TableCatalogCapability.SUPPORTS_CREATE_TABLE_WITH_GENERATED_COLUMNS,
+ TableCatalogCapability.SUPPORTS_CREATE_TABLE_WITH_IDENTITY_COLUMNS
).asJava
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagementSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagementSuite.scala
index 1aa0b408366bf..a9d8a69128ae2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagementSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsAtomicPartitionManagementSuite.scala
@@ -121,7 +121,7 @@ class SupportsAtomicPartitionManagementSuite extends SparkFunSuite {
exception = intercept[SparkUnsupportedOperationException] {
partTable.purgePartitions(partIdents)
},
- errorClass = "UNSUPPORTED_FEATURE.PURGE_PARTITION",
+ condition = "UNSUPPORTED_FEATURE.PURGE_PARTITION",
parameters = Map.empty
)
}
@@ -170,7 +170,7 @@ class SupportsAtomicPartitionManagementSuite extends SparkFunSuite {
partTable.truncatePartitions(Array(InternalRow("5"), InternalRow("6")))
}
checkError(e,
- errorClass = "PARTITIONS_NOT_FOUND",
+ condition = "PARTITIONS_NOT_FOUND",
parameters = Map("partitionList" -> "PARTITION (`dt` = 6)",
"tableName" -> "`test`.`ns`.`test_table`"))
assert(partTable.rows === InternalRow(2, "zyx", "5") :: Nil)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsPartitionManagementSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsPartitionManagementSuite.scala
index 06a23e7fda207..8581d4dec1fb8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsPartitionManagementSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/SupportsPartitionManagementSuite.scala
@@ -93,7 +93,7 @@ class SupportsPartitionManagementSuite extends SparkFunSuite {
exception = intercept[SparkUnsupportedOperationException] {
partTable.purgePartition(InternalRow.apply("3"))
},
- errorClass = "UNSUPPORTED_FEATURE.PURGE_PARTITION",
+ condition = "UNSUPPORTED_FEATURE.PURGE_PARTITION",
parameters = Map.empty
)
}
@@ -217,7 +217,7 @@ class SupportsPartitionManagementSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
partTable.partitionExists(InternalRow(0))
},
- errorClass = "_LEGACY_ERROR_TEMP_3208",
+ condition = "_LEGACY_ERROR_TEMP_3208",
parameters = Map("numFields" -> "1", "schemaLen" -> "2"))
}
@@ -228,7 +228,7 @@ class SupportsPartitionManagementSuite extends SparkFunSuite {
partTable.renamePartition(InternalRow(0, "abc"), InternalRow(1, "abc"))
}
checkError(e,
- errorClass = "PARTITIONS_ALREADY_EXIST",
+ condition = "PARTITIONS_ALREADY_EXIST",
parameters = Map("partitionList" -> "PARTITION (`part0` = 1, `part1` = abc)",
"tableName" -> "`test`.`ns`.`test_table`"))
@@ -237,7 +237,7 @@ class SupportsPartitionManagementSuite extends SparkFunSuite {
partTable.renamePartition(newPart, InternalRow(3, "abc"))
}
checkError(e2,
- errorClass = "PARTITIONS_NOT_FOUND",
+ condition = "PARTITIONS_NOT_FOUND",
parameters = Map("partitionList" -> "PARTITION (`part0` = 2, `part1` = xyz)",
"tableName" -> "`test`.`ns`.`test_table`"))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
index 4343e464b2c80..3241f031a706b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
@@ -161,7 +161,7 @@ class DataTypeSuite extends SparkFunSuite {
exception = intercept[SparkException] {
left.merge(right)
},
- errorClass = "CANNOT_MERGE_INCOMPATIBLE_DATA_TYPE",
+ condition = "CANNOT_MERGE_INCOMPATIBLE_DATA_TYPE",
parameters = Map("left" -> "\"FLOAT\"", "right" -> "\"BIGINT\""
)
)
@@ -299,21 +299,21 @@ class DataTypeSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
DataType.fromJson(""""abcd"""")
},
- errorClass = "INVALID_JSON_DATA_TYPE",
+ condition = "INVALID_JSON_DATA_TYPE",
parameters = Map("invalidType" -> "abcd"))
checkError(
exception = intercept[SparkIllegalArgumentException] {
DataType.fromJson("""{"abcd":"a"}""")
},
- errorClass = "INVALID_JSON_DATA_TYPE",
+ condition = "INVALID_JSON_DATA_TYPE",
parameters = Map("invalidType" -> """{"abcd":"a"}"""))
checkError(
exception = intercept[SparkIllegalArgumentException] {
DataType.fromJson("""{"fields": [{"a":123}], "type": "struct"}""")
},
- errorClass = "INVALID_JSON_DATA_TYPE",
+ condition = "INVALID_JSON_DATA_TYPE",
parameters = Map("invalidType" -> """{"a":123}"""))
// Malformed JSON string
@@ -900,7 +900,7 @@ class DataTypeSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
DataType.fromJson(json)
},
- errorClass = "INVALID_JSON_DATA_TYPE_FOR_COLLATIONS",
+ condition = "INVALID_JSON_DATA_TYPE_FOR_COLLATIONS",
parameters = Map("jsonType" -> "integer")
)
}
@@ -934,7 +934,7 @@ class DataTypeSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
DataType.fromJson(json)
},
- errorClass = "INVALID_JSON_DATA_TYPE_FOR_COLLATIONS",
+ condition = "INVALID_JSON_DATA_TYPE_FOR_COLLATIONS",
parameters = Map("jsonType" -> "integer")
)
}
@@ -968,7 +968,7 @@ class DataTypeSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
DataType.fromJson(json)
},
- errorClass = "INVALID_JSON_DATA_TYPE_FOR_COLLATIONS",
+ condition = "INVALID_JSON_DATA_TYPE_FOR_COLLATIONS",
parameters = Map("jsonType" -> "map")
)
}
@@ -997,7 +997,7 @@ class DataTypeSuite extends SparkFunSuite {
exception = intercept[SparkException] {
DataType.fromJson(json)
},
- errorClass = "COLLATION_INVALID_PROVIDER",
+ condition = "COLLATION_INVALID_PROVIDER",
parameters = Map("provider" -> "badProvider", "supportedProviders" -> "spark, icu")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
index 8c9196cc33ca5..f07ee8b35bbb2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
@@ -41,7 +41,7 @@ class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBa
DataTypeUtils.canWrite("", widerPoint2, point2, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`t`.`x`",
@@ -60,7 +60,7 @@ class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBa
DataTypeUtils.canWrite("", arrayOfLong, arrayOfInt, true,
analysis.caseSensitiveResolution, "arr", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`arr`.`element`",
@@ -79,7 +79,7 @@ class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBa
DataTypeUtils.canWrite("", mapOfLong, mapOfInt, true,
analysis.caseSensitiveResolution, "m", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`m`.`value`",
@@ -98,7 +98,7 @@ class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBa
DataTypeUtils.canWrite("", mapKeyLong, mapKeyInt, true,
analysis.caseSensitiveResolution, "m", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`m`.`key`",
@@ -116,7 +116,7 @@ class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBa
analysis.caseSensitiveResolution, "nulls", storeAssignmentPolicy,
errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`nulls`",
@@ -143,7 +143,7 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase
DataTypeUtils.canWrite("", mapOfString, mapOfInt, true,
analysis.caseSensitiveResolution, "m", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`m`.`value`",
@@ -163,7 +163,7 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase
DataTypeUtils.canWrite("", stringPoint2, point2, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`t`.`x`",
@@ -182,7 +182,7 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase
DataTypeUtils.canWrite("", arrayOfString, arrayOfInt, true,
analysis.caseSensitiveResolution, "arr", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`arr`.`element`",
@@ -201,7 +201,7 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase
DataTypeUtils.canWrite("", mapKeyString, mapKeyInt, true,
analysis.caseSensitiveResolution, "arr", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`arr`.`key`",
@@ -218,7 +218,7 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase
analysis.caseSensitiveResolution, "longToTimestamp", storeAssignmentPolicy,
errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`longToTimestamp`",
@@ -231,7 +231,7 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase
analysis.caseSensitiveResolution, "timestampToLong", storeAssignmentPolicy,
errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`timestampToLong`",
@@ -306,7 +306,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", w, r, true, analysis.caseSensitiveResolution, "t",
storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`t`",
@@ -328,7 +328,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", missingRequiredField, point2, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.STRUCT_MISSING_FIELDS",
parameters = Map("tableName" -> "``", "colName" -> "`t`", "missingFields" -> "`y`")
)
}
@@ -341,7 +341,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", missingRequiredField, point2, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.UNEXPECTED_COLUMN_NAME",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.UNEXPECTED_COLUMN_NAME",
parameters = Map(
"expected" -> "`x`",
"found" -> "`y`",
@@ -369,7 +369,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", missingMiddleField, expectedStruct, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.UNEXPECTED_COLUMN_NAME",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.UNEXPECTED_COLUMN_NAME",
parameters = Map(
"expected" -> "`y`",
"found" -> "`z`",
@@ -406,7 +406,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", requiredFieldIsOptional, point2, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_COLUMN",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_COLUMN",
parameters = Map("tableName" -> "``", "colName" -> "`t`.`x`")
)
}
@@ -418,7 +418,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", point3, point2, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.EXTRA_STRUCT_FIELDS",
parameters = Map("tableName" -> "``", "colName" -> "`t`", "extraFields" -> "`z`")
)
}
@@ -459,7 +459,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", arrayOfOptional, arrayOfRequired, true,
analysis.caseSensitiveResolution, "arr", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_ARRAY_ELEMENTS",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_ARRAY_ELEMENTS",
parameters = Map("tableName" -> "``", "colName" -> "`arr`")
)
}
@@ -489,7 +489,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", mapOfOptional, mapOfRequired, true,
analysis.caseSensitiveResolution, "m", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_MAP_VALUES",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_MAP_VALUES",
parameters = Map("tableName" -> "``", "colName" -> "`m`")
)
}
@@ -560,7 +560,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", sqlType, udtType, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_COLUMN",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.NULLABLE_COLUMN",
parameters = Map(
"tableName" -> "``",
"colName" -> "`t`.`col2`"
@@ -595,7 +595,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", udtType, sqlType, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`t`.`col2`",
@@ -633,7 +633,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", sqlType, udtType, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`t`.`col2`",
@@ -675,7 +675,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
DataTypeUtils.canWrite("", writeType, readType, true,
analysis.caseSensitiveResolution, "t", storeAssignmentPolicy, errMsg => errs += errMsg)
),
- errorClass = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
+ condition = "INCOMPATIBLE_DATA_FOR_TABLE.CANNOT_SAFELY_CAST",
parameters = Map(
"tableName" -> "``",
"colName" -> "`t`.`a`.`element`",
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index e6d915903f9bc..794112db5502a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -65,7 +65,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
checkError(
exception = intercept[SparkArithmeticException](Decimal(170L, 2, 1)),
- errorClass = "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION",
+ condition = "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION",
parameters = Map(
"value" -> "0",
"precision" -> "2",
@@ -73,7 +73,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
"config" -> "\"spark.sql.ansi.enabled\""))
checkError(
exception = intercept[SparkArithmeticException](Decimal(170L, 2, 0)),
- errorClass = "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION",
+ condition = "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION",
parameters = Map(
"value" -> "0",
"precision" -> "2",
@@ -81,7 +81,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
"config" -> "\"spark.sql.ansi.enabled\""))
checkError(
exception = intercept[SparkArithmeticException](Decimal(BigDecimal("10.030"), 2, 1)),
- errorClass = "NUMERIC_VALUE_OUT_OF_RANGE.WITHOUT_SUGGESTION",
+ condition = "NUMERIC_VALUE_OUT_OF_RANGE.WITHOUT_SUGGESTION",
parameters = Map(
"roundedValue" -> "10.0",
"originalValue" -> "10.030",
@@ -89,7 +89,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
"scale" -> "1"))
checkError(
exception = intercept[SparkArithmeticException](Decimal(BigDecimal("-9.95"), 2, 1)),
- errorClass = "NUMERIC_VALUE_OUT_OF_RANGE.WITHOUT_SUGGESTION",
+ condition = "NUMERIC_VALUE_OUT_OF_RANGE.WITHOUT_SUGGESTION",
parameters = Map(
"roundedValue" -> "-10.0",
"originalValue" -> "-9.95",
@@ -97,7 +97,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
"scale" -> "1"))
checkError(
exception = intercept[SparkArithmeticException](Decimal(1e17.toLong, 17, 0)),
- errorClass = "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION",
+ condition = "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION",
parameters = Map(
"value" -> "0",
"precision" -> "17",
@@ -120,7 +120,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
def checkNegativeScaleDecimal(d: => Decimal): Unit = {
checkError(
exception = intercept[SparkException] (d),
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> ("Negative scale is not allowed: -3. " +
"Set the config \"spark.sql.legacy.allowNegativeScaleOfDecimal\" " +
"to \"true\" to allow it."))
@@ -317,7 +317,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
exception = intercept[SparkException] {
d.toPrecision(5, 50, BigDecimal.RoundingMode.HALF_DOWN)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> "Not supported rounding mode: HALF_DOWN.")
)
}
@@ -350,7 +350,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
checkError(
exception = intercept[SparkArithmeticException](
Decimal.fromStringANSI(UTF8String.fromString(string))),
- errorClass = "NUMERIC_OUT_OF_SUPPORTED_RANGE",
+ condition = "NUMERIC_OUT_OF_SUPPORTED_RANGE",
parameters = Map("value" -> string))
}
@@ -370,12 +370,11 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
checkError(
exception = intercept[SparkNumberFormatException](
Decimal.fromStringANSI(UTF8String.fromString("str"))),
- errorClass = "CAST_INVALID_INPUT",
+ condition = "CAST_INVALID_INPUT",
parameters = Map(
"expression" -> "'str'",
"sourceType" -> "\"STRING\"",
- "targetType" -> "\"DECIMAL(10,0)\"",
- "ansiConfig" -> "\"spark.sql.ansi.enabled\""))
+ "targetType" -> "\"DECIMAL(10,0)\""))
}
test("SPARK-35841: Casting string to decimal type doesn't work " +
@@ -398,7 +397,7 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
checkError(
exception = intercept[SparkArithmeticException](
Decimal.fromStringANSI(UTF8String.fromString(string))),
- errorClass = "NUMERIC_OUT_OF_SUPPORTED_RANGE",
+ condition = "NUMERIC_OUT_OF_SUPPORTED_RANGE",
parameters = Map("value" -> string))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
index 562febe381130..6a67525dd02d3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
@@ -45,21 +45,21 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
test("lookup a single missing field should output existing fields") {
checkError(
exception = intercept[SparkIllegalArgumentException](s("c")),
- errorClass = "FIELD_NOT_FOUND",
+ condition = "FIELD_NOT_FOUND",
parameters = Map("fieldName" -> "`c`", "fields" -> "`a`, `b`"))
}
test("lookup a set of missing fields should output existing fields") {
checkError(
exception = intercept[SparkIllegalArgumentException](s(Set("a", "c"))),
- errorClass = "NONEXISTENT_FIELD_NAME_IN_LIST",
+ condition = "NONEXISTENT_FIELD_NAME_IN_LIST",
parameters = Map("nonExistFields" -> "`c`", "fieldNames" -> "`a`, `b`"))
}
test("lookup fieldIndex for missing field should output existing fields") {
checkError(
exception = intercept[SparkIllegalArgumentException](s.fieldIndex("c")),
- errorClass = "FIELD_NOT_FOUND",
+ condition = "FIELD_NOT_FOUND",
parameters = Map("fieldName" -> "`c`", "fields" -> "`a`, `b`"))
}
@@ -341,7 +341,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`S1`.`S12`.`S123`",
"path" -> "`s1`.`s12`"))
@@ -352,7 +352,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "AMBIGUOUS_COLUMN_OR_FIELD",
+ condition = "AMBIGUOUS_COLUMN_OR_FIELD",
parameters = Map("name" -> "`S2`.`x`", "n" -> "2"))
caseSensitiveCheck(Seq("s2", "x"), Some(Seq("s2") -> StructField("x", IntegerType)))
@@ -362,7 +362,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`m1`.`key`",
"path" -> "`m1`"))
@@ -373,7 +373,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`M1`.`key`.`name`",
"path" -> "`m1`.`key`"))
@@ -382,7 +382,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`M1`.`value`.`name`",
"path" -> "`m1`.`value`"))
@@ -399,7 +399,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`m2`.`key`.`A`.`name`",
"path" -> "`m2`.`key`.`a`"))
@@ -408,7 +408,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`M2`.`value`.`b`.`name`",
"path" -> "`m2`.`value`.`b`"))
@@ -418,7 +418,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`A1`.`element`",
"path" -> "`a1`"))
@@ -428,7 +428,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`A1`.`element`.`name`",
"path" -> "`a1`.`element`"))
@@ -442,7 +442,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`a2`.`element`.`C`.`name`",
"path" -> "`a2`.`element`.`c`"))
@@ -456,7 +456,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`M3`.`value`.`value`.`MA`.`name`",
"path" -> "`m3`.`value`.`value`.`ma`"))
@@ -470,7 +470,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
}
checkError(
exception = e,
- errorClass = "INVALID_FIELD_NAME",
+ condition = "INVALID_FIELD_NAME",
parameters = Map(
"fieldName" -> "`A3`.`element`.`element`.`D`.`name`",
"path" -> "`a3`.`element`.`element`.`d`")
@@ -522,7 +522,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkException] {
StructType.fromDDL("c1 DECIMAL(10, 5)").merge(StructType.fromDDL("c1 DECIMAL(12, 2)"))
},
- errorClass = "CANNOT_MERGE_INCOMPATIBLE_DATA_TYPE",
+ condition = "CANNOT_MERGE_INCOMPATIBLE_DATA_TYPE",
parameters = Map("left" -> "\"DECIMAL(10,5)\"", "right" -> "\"DECIMAL(12,2)\"")
)
@@ -530,7 +530,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
exception = intercept[SparkException] {
StructType.fromDDL("c1 DECIMAL(12, 5)").merge(StructType.fromDDL("c1 DECIMAL(12, 2)"))
},
- errorClass = "CANNOT_MERGE_INCOMPATIBLE_DATA_TYPE",
+ condition = "CANNOT_MERGE_INCOMPATIBLE_DATA_TYPE",
parameters = Map("left" -> "\"DECIMAL(12,5)\"", "right" -> "\"DECIMAL(12,2)\"")
)
}
@@ -564,7 +564,6 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
.putString(ResolveDefaultColumns.EXISTS_DEFAULT_COLUMN_METADATA_KEY, "1 + 1")
.putString(ResolveDefaultColumns.CURRENT_DEFAULT_COLUMN_METADATA_KEY, "1 + 1")
.build())))
- val error = "fails to parse as a valid literal value"
assert(ResolveDefaultColumns.existenceDefaultValues(source2).length == 1)
assert(ResolveDefaultColumns.existenceDefaultValues(source2)(0) == 2)
@@ -576,9 +575,13 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
.putString(ResolveDefaultColumns.EXISTS_DEFAULT_COLUMN_METADATA_KEY, "invalid")
.putString(ResolveDefaultColumns.CURRENT_DEFAULT_COLUMN_METADATA_KEY, "invalid")
.build())))
- assert(intercept[AnalysisException] {
- ResolveDefaultColumns.existenceDefaultValues(source3)
- }.getMessage.contains(error))
+
+ checkError(
+ exception = intercept[AnalysisException]{
+ ResolveDefaultColumns.existenceDefaultValues(source3)
+ },
+ condition = "INVALID_DEFAULT_VALUE.UNRESOLVED_EXPRESSION",
+ parameters = Map("statement" -> "", "colName" -> "`c1`", "defaultValue" -> "invalid"))
// Negative test: StructType.defaultValues fails because the existence default value fails to
// resolve.
@@ -592,9 +595,15 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
ResolveDefaultColumns.CURRENT_DEFAULT_COLUMN_METADATA_KEY,
"(SELECT 'abc' FROM missingtable)")
.build())))
- assert(intercept[AnalysisException] {
- ResolveDefaultColumns.existenceDefaultValues(source4)
- }.getMessage.contains(error))
+
+ checkError(
+ exception = intercept[AnalysisException]{
+ ResolveDefaultColumns.existenceDefaultValues(source4)
+ },
+ condition = "INVALID_DEFAULT_VALUE.SUBQUERY_EXPRESSION",
+ parameters = Map("statement" -> "",
+ "colName" -> "`c1`",
+ "defaultValue" -> "(SELECT 'abc' FROM missingtable)"))
}
test("SPARK-46629: Test STRUCT DDL with NOT NULL round trip") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
index c0fa43ff9bde0..c705a6b791bd1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
@@ -54,13 +54,13 @@ class ArrowUtilsSuite extends SparkFunSuite {
exception = intercept[SparkException] {
roundtrip(TimestampType)
},
- errorClass = "INTERNAL_ERROR",
+ condition = "INTERNAL_ERROR",
parameters = Map("message" -> "Missing timezoneId where it is mandatory."))
checkError(
exception = intercept[SparkUnsupportedOperationException] {
ArrowUtils.fromArrowType(new ArrowType.Int(8, false))
},
- errorClass = "UNSUPPORTED_ARROWTYPE",
+ condition = "UNSUPPORTED_ARROWTYPE",
parameters = Map("typeName" -> "Int(8, false)")
)
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/CaseInsensitiveStringMapSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/CaseInsensitiveStringMapSuite.scala
index 98c2a3d1e2726..932fb0a733371 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/CaseInsensitiveStringMapSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/CaseInsensitiveStringMapSuite.scala
@@ -67,7 +67,7 @@ class CaseInsensitiveStringMapSuite extends SparkFunSuite {
exception = intercept[SparkIllegalArgumentException] {
options.getBoolean("FOO", true)
},
- errorClass = "_LEGACY_ERROR_TEMP_3206",
+ condition = "_LEGACY_ERROR_TEMP_3206",
parameters = Map("value" -> "bar"))
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/SchemaUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/SchemaUtilsSuite.scala
index c5f19b438f27f..a277bb021c3f6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/SchemaUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/SchemaUtilsSuite.scala
@@ -46,20 +46,20 @@ class SchemaUtilsSuite extends SparkFunSuite {
exception = intercept[AnalysisException] {
SchemaUtils.checkSchemaColumnNameDuplication(schema, caseSensitive)
},
- errorClass = "COLUMN_ALREADY_EXISTS",
+ condition = "COLUMN_ALREADY_EXISTS",
parameters = Map("columnName" -> "`a`"))
checkError(
exception = intercept[AnalysisException] {
SchemaUtils.checkColumnNameDuplication(schema.map(_.name), resolver(caseSensitive))
},
- errorClass = "COLUMN_ALREADY_EXISTS",
+ condition = "COLUMN_ALREADY_EXISTS",
parameters = Map("columnName" -> "`a`"))
checkError(
exception = intercept[AnalysisException] {
SchemaUtils.checkColumnNameDuplication(
schema.map(_.name), caseSensitiveAnalysis = caseSensitive)
},
- errorClass = "COLUMN_ALREADY_EXISTS",
+ condition = "COLUMN_ALREADY_EXISTS",
parameters = Map("columnName" -> "`a`"))
}
@@ -106,7 +106,7 @@ class SchemaUtilsSuite extends SparkFunSuite {
exception = intercept[AnalysisException] {
SchemaUtils.checkSchemaColumnNameDuplication(schema)
},
- errorClass = "COLUMN_ALREADY_EXISTS",
+ condition = "COLUMN_ALREADY_EXISTS",
parameters = Map("columnName" -> "`camelcase`"))
}
}
diff --git a/sql/connect/common/src/main/protobuf/spark/connect/relations.proto b/sql/connect/common/src/main/protobuf/spark/connect/relations.proto
index 04fe21086097c..1003e5c21d639 100644
--- a/sql/connect/common/src/main/protobuf/spark/connect/relations.proto
+++ b/sql/connect/common/src/main/protobuf/spark/connect/relations.proto
@@ -76,6 +76,7 @@ message Relation {
AsOfJoin as_of_join = 39;
CommonInlineUserDefinedDataSource common_inline_user_defined_data_source = 40;
WithRelations with_relations = 41;
+ Transpose transpose = 42;
// NA functions
NAFill fill_na = 90;
@@ -889,6 +890,18 @@ message Unpivot {
}
}
+// Transpose a DataFrame, switching rows to columns.
+// Transforms the DataFrame such that the values in the specified index column
+// become the new columns of the DataFrame.
+message Transpose {
+ // (Required) The input relation.
+ Relation input = 1;
+
+ // (Optional) A list of columns that will be treated as the indices.
+ // Only single column is supported now.
+ repeated Expression index_columns = 2;
+}
+
message ToSchema {
// (Required) The input relation.
Relation input = 1;
diff --git a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/common/UdfUtils.scala b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/common/UdfUtils.scala
index d77b4b820c090..2dba8fc3b7778 100644
--- a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/common/UdfUtils.scala
+++ b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/common/UdfUtils.scala
@@ -28,6 +28,9 @@ import org.apache.spark.sql.streaming.GroupState
* mapPartitions etc. This class is shared between the client and the server so that when the
* methods are used in client UDFs, the server will be able to find them when actually executing
* the UDFs.
+ *
+ * DO NOT REMOVE/CHANGE THIS OBJECT OR ANY OF ITS METHODS, THEY ARE NEEDED FOR BACKWARDS
+ * COMPATIBILITY WITH OLDER CLIENTS!
*/
@SerialVersionUID(8464839273647598302L)
private[sql] object UdfUtils extends Serializable {
@@ -137,8 +140,6 @@ private[sql] object UdfUtils extends Serializable {
// ----------------------------------------------------------------------------------------------
// Scala Functions wrappers for java UDFs.
- //
- // DO NOT REMOVE THESE, THEY ARE NEEDED FOR BACKWARDS COMPATIBILITY WITH OLDER CLIENTS!
// ----------------------------------------------------------------------------------------------
// (1 to 22).foreach { i =>
// val extTypeArgs = (0 to i).map(_ => "_").mkString(", ")
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_descFilePath.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_descFilePath.explain
index 6eb4805b4fcc4..61d81bad95c65 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_descFilePath.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_descFilePath.explain
@@ -1,2 +1,2 @@
-Project [from_protobuf(bytes#0, StorageLevel, Some([B)) AS from_protobuf(bytes)#0]
+Project [from_protobuf(bytes#0, StorageLevel, Some([B)) AS from_protobuf(bytes, StorageLevel, X'0AFC010A0C636F6D6D6F6E2E70726F746F120D737061726B2E636F6E6E65637422B0010A0C53746F726167654C6576656C12190A087573655F6469736B18012001280852077573654469736B121D0A0A7573655F6D656D6F727918022001280852097573654D656D6F727912200A0C7573655F6F66665F68656170180320012808520A7573654F66664865617012220A0C646573657269616C697A6564180420012808520C646573657269616C697A656412200A0B7265706C69636174696F6E180520012805520B7265706C69636174696F6E42220A1E6F72672E6170616368652E737061726B2E636F6E6E6563742E70726F746F5001620670726F746F33', NULL)#0]
+- LocalRelation , [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_descFilePath_options.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_descFilePath_options.explain
index c4a47b1aef07b..066dba527a09a 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_descFilePath_options.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_descFilePath_options.explain
@@ -1,2 +1,2 @@
-Project [from_protobuf(bytes#0, StorageLevel, Some([B), (recursive.fields.max.depth,2)) AS from_protobuf(bytes)#0]
+Project [from_protobuf(bytes#0, StorageLevel, Some([B), (recursive.fields.max.depth,2)) AS from_protobuf(bytes, StorageLevel, X'0AFC010A0C636F6D6D6F6E2E70726F746F120D737061726B2E636F6E6E65637422B0010A0C53746F726167654C6576656C12190A087573655F6469736B18012001280852077573654469736B121D0A0A7573655F6D656D6F727918022001280852097573654D656D6F727912200A0C7573655F6F66665F68656170180320012808520A7573654F66664865617012220A0C646573657269616C697A6564180420012808520C646573657269616C697A656412200A0B7265706C69636174696F6E180520012805520B7265706C69636174696F6E42220A1E6F72672E6170616368652E737061726B2E636F6E6E6563742E70726F746F5001620670726F746F33', map(recursive.fields.max.depth, 2))#0]
+- LocalRelation , [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json.explain
index 1219f11d4696e..8d1d122d156ff 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json.explain
@@ -1,2 +1,2 @@
-Project [from_json(StructField(id,LongType,true), StructField(a,IntegerType,true), StructField(b,DoubleType,true), g#0, Some(America/Los_Angeles)) AS from_json(g)#0]
+Project [from_json(StructField(id,LongType,true), StructField(a,IntegerType,true), StructField(b,DoubleType,true), g#0, Some(America/Los_Angeles), false) AS from_json(g)#0]
+- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json_orphaned.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json_orphaned.explain
index 1219f11d4696e..8d1d122d156ff 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json_orphaned.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json_orphaned.explain
@@ -1,2 +1,2 @@
-Project [from_json(StructField(id,LongType,true), StructField(a,IntegerType,true), StructField(b,DoubleType,true), g#0, Some(America/Los_Angeles)) AS from_json(g)#0]
+Project [from_json(StructField(id,LongType,true), StructField(a,IntegerType,true), StructField(b,DoubleType,true), g#0, Some(America/Los_Angeles), false) AS from_json(g)#0]
+- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json_with_json_schema.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json_with_json_schema.explain
index 1219f11d4696e..8d1d122d156ff 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json_with_json_schema.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/function_from_json_with_json_schema.explain
@@ -1,2 +1,2 @@
-Project [from_json(StructField(id,LongType,true), StructField(a,IntegerType,true), StructField(b,DoubleType,true), g#0, Some(America/Los_Angeles)) AS from_json(g)#0]
+Project [from_json(StructField(id,LongType,true), StructField(a,IntegerType,true), StructField(b,DoubleType,true), g#0, Some(America/Los_Angeles), false) AS from_json(g)#0]
+- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences.explain
index 5c88a1f7b3abd..f4532e70675ae 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences.explain
@@ -1,2 +1,2 @@
-Project [sentences(g#0, , ) AS sentences(g, , )#0]
+Project [static_invoke(ExpressionImplUtils.getSentences(g#0, , )) AS sentences(g, , )#0]
+- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences_with_language.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences_with_language.explain
new file mode 100644
index 0000000000000..37bcbf9a319b5
--- /dev/null
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences_with_language.explain
@@ -0,0 +1,2 @@
+Project [static_invoke(ExpressionImplUtils.getSentences(g#0, en, )) AS sentences(g, en, )#0]
++- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences_with_language_and_country.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences_with_language_and_country.explain
new file mode 100644
index 0000000000000..8a8d54cfa0d10
--- /dev/null
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences_with_language_and_country.explain
@@ -0,0 +1,2 @@
+Project [static_invoke(ExpressionImplUtils.getSentences(g#0, en, US)) AS sentences(g, en, US)#0]
++- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences_with_locale.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences_with_locale.explain
deleted file mode 100644
index 7819f9b542340..0000000000000
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/function_sentences_with_locale.explain
+++ /dev/null
@@ -1,2 +0,0 @@
-Project [sentences(g#0, en, US) AS sentences(g, en, US)#0]
-+- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/melt_no_values.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/melt_no_values.explain
index f61fc30a3a529..053937d84ec8f 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/melt_no_values.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/melt_no_values.explain
@@ -1,2 +1,2 @@
-Expand [[id#0L, a#0, b, b#0]], [id#0L, a#0, #0, value#0]
+Expand [[id#0L, a#0, b, b#0]], [id#0L, a#0, name#0, value#0]
+- LocalRelation , [id#0L, a#0, b#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/melt_values.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/melt_values.explain
index b5742d976dee9..5a953f792cd35 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/melt_values.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/melt_values.explain
@@ -1,2 +1,2 @@
-Expand [[a#0, id, id#0L]], [a#0, #0, value#0L]
+Expand [[a#0, id, id#0L]], [a#0, name#0, value#0L]
+- LocalRelation , [id#0L, a#0, b#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName.explain
index e7f70fa2c1a9e..3533406c0bf0a 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName.explain
@@ -1,2 +1,2 @@
-Project [to_protobuf(bytes#0, org.apache.spark.connect.proto.StorageLevel, None) AS to_protobuf(bytes)#0]
+Project [to_protobuf(bytes#0, org.apache.spark.connect.proto.StorageLevel, None) AS to_protobuf(bytes, org.apache.spark.connect.proto.StorageLevel, NULL, NULL)#0]
+- LocalRelation , [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath.explain
index 7c688cc446947..f6a33a20a5dcd 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath.explain
@@ -1,2 +1,2 @@
-Project [to_protobuf(bytes#0, StorageLevel, Some([B)) AS to_protobuf(bytes)#0]
+Project [to_protobuf(bytes#0, StorageLevel, Some([B)) AS to_protobuf(bytes, StorageLevel, X'0AFC010A0C636F6D6D6F6E2E70726F746F120D737061726B2E636F6E6E65637422B0010A0C53746F726167654C6576656C12190A087573655F6469736B18012001280852077573654469736B121D0A0A7573655F6D656D6F727918022001280852097573654D656D6F727912200A0C7573655F6F66665F68656170180320012808520A7573654F66664865617012220A0C646573657269616C697A6564180420012808520C646573657269616C697A656412200A0B7265706C69636174696F6E180520012805520B7265706C69636174696F6E42220A1E6F72672E6170616368652E737061726B2E636F6E6E6563742E70726F746F5001620670726F746F33', NULL)#0]
+- LocalRelation , [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath_options.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath_options.explain
index 9f05bb03c9c6d..393529a15670d 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath_options.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_descFilePath_options.explain
@@ -1,2 +1,2 @@
-Project [to_protobuf(bytes#0, StorageLevel, Some([B), (recursive.fields.max.depth,2)) AS to_protobuf(bytes)#0]
+Project [to_protobuf(bytes#0, StorageLevel, Some([B), (recursive.fields.max.depth,2)) AS to_protobuf(bytes, StorageLevel, X'0AFC010A0C636F6D6D6F6E2E70726F746F120D737061726B2E636F6E6E65637422B0010A0C53746F726167654C6576656C12190A087573655F6469736B18012001280852077573654469736B121D0A0A7573655F6D656D6F727918022001280852097573654D656D6F727912200A0C7573655F6F66665F68656170180320012808520A7573654F66664865617012220A0C646573657269616C697A6564180420012808520C646573657269616C697A656412200A0B7265706C69636174696F6E180520012805520B7265706C69636174696F6E42220A1E6F72672E6170616368652E737061726B2E636F6E6E6563742E70726F746F5001620670726F746F33', map(recursive.fields.max.depth, 2))#0]
+- LocalRelation , [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_options.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_options.explain
index a5d8851a7d1f3..e0c7e1625fe5c 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_options.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/to_protobuf_messageClassName_options.explain
@@ -1,2 +1,2 @@
-Project [to_protobuf(bytes#0, org.apache.spark.connect.proto.StorageLevel, None, (recursive.fields.max.depth,2)) AS to_protobuf(bytes)#0]
+Project [to_protobuf(bytes#0, org.apache.spark.connect.proto.StorageLevel, None, (recursive.fields.max.depth,2)) AS to_protobuf(bytes, org.apache.spark.connect.proto.StorageLevel, NULL, map(recursive.fields.max.depth, 2))#0]
+- LocalRelation , [id#0L, bytes#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/unpivot_no_values.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/unpivot_no_values.explain
index 8d1749ee74c5a..2b2ba19d0c3db 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/unpivot_no_values.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/unpivot_no_values.explain
@@ -1,2 +1,2 @@
-Expand [[id#0L, a, cast(a#0 as double)], [id#0L, b, b#0]], [id#0L, #0, value#0]
+Expand [[id#0L, a, cast(a#0 as double)], [id#0L, b, b#0]], [id#0L, name#0, value#0]
+- LocalRelation , [id#0L, a#0, b#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/explain-results/unpivot_values.explain b/sql/connect/common/src/test/resources/query-tests/explain-results/unpivot_values.explain
index f61fc30a3a529..053937d84ec8f 100644
--- a/sql/connect/common/src/test/resources/query-tests/explain-results/unpivot_values.explain
+++ b/sql/connect/common/src/test/resources/query-tests/explain-results/unpivot_values.explain
@@ -1,2 +1,2 @@
-Expand [[id#0L, a#0, b, b#0]], [id#0L, a#0, #0, value#0]
+Expand [[id#0L, a#0, b, b#0]], [id#0L, a#0, name#0, value#0]
+- LocalRelation , [id#0L, a#0, b#0]
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.json
new file mode 100644
index 0000000000000..869e074ccd604
--- /dev/null
+++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.json
@@ -0,0 +1,29 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "sentences",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "g"
+ }
+ }, {
+ "literal": {
+ "string": "en"
+ }
+ }]
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.proto.bin
new file mode 100644
index 0000000000000..7514b380a1c82
Binary files /dev/null and b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.proto.bin differ
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_locale.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.json
similarity index 100%
rename from sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_locale.json
rename to sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.json
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_locale.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.proto.bin
similarity index 100%
rename from sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_locale.proto.bin
rename to sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.proto.bin
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/melt_no_values.json b/sql/connect/common/src/test/resources/query-tests/queries/melt_no_values.json
index 12db0a5abe368..a17da06b925b9 100644
--- a/sql/connect/common/src/test/resources/query-tests/queries/melt_no_values.json
+++ b/sql/connect/common/src/test/resources/query-tests/queries/melt_no_values.json
@@ -20,6 +20,7 @@
"unparsedIdentifier": "a"
}
}],
+ "variableColumnName": "name",
"valueColumnName": "value"
}
}
\ No newline at end of file
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/melt_no_values.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/melt_no_values.proto.bin
index 23a6aa1289a99..eebb7ad6df8e2 100644
Binary files a/sql/connect/common/src/test/resources/query-tests/queries/melt_no_values.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/melt_no_values.proto.bin differ
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/melt_values.json b/sql/connect/common/src/test/resources/query-tests/queries/melt_values.json
index e2a004f46e781..a8142ee3a8461 100644
--- a/sql/connect/common/src/test/resources/query-tests/queries/melt_values.json
+++ b/sql/connect/common/src/test/resources/query-tests/queries/melt_values.json
@@ -23,6 +23,7 @@
}
}]
},
+ "variableColumnName": "name",
"valueColumnName": "value"
}
}
\ No newline at end of file
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/melt_values.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/melt_values.proto.bin
index e021e1110def5..35829fc62dae9 100644
Binary files a/sql/connect/common/src/test/resources/query-tests/queries/melt_values.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/melt_values.proto.bin differ
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/transpose_index_column.json b/sql/connect/common/src/test/resources/query-tests/queries/transpose_index_column.json
new file mode 100644
index 0000000000000..19a2086c8d7df
--- /dev/null
+++ b/sql/connect/common/src/test/resources/query-tests/queries/transpose_index_column.json
@@ -0,0 +1,20 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "transpose": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,a:int,b:double\u003e"
+ }
+ },
+ "indexColumns": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/transpose_index_column.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/transpose_index_column.proto.bin
new file mode 100644
index 0000000000000..8590932d95cb4
Binary files /dev/null and b/sql/connect/common/src/test/resources/query-tests/queries/transpose_index_column.proto.bin differ
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/transpose_no_index_column.json b/sql/connect/common/src/test/resources/query-tests/queries/transpose_no_index_column.json
new file mode 100644
index 0000000000000..82b0448c373e1
--- /dev/null
+++ b/sql/connect/common/src/test/resources/query-tests/queries/transpose_no_index_column.json
@@ -0,0 +1,15 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "transpose": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,a:int,b:double\u003e"
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/transpose_no_index_column.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/transpose_no_index_column.proto.bin
new file mode 100644
index 0000000000000..c1ea985a64a4b
Binary files /dev/null and b/sql/connect/common/src/test/resources/query-tests/queries/transpose_no_index_column.proto.bin differ
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/unpivot_no_values.json b/sql/connect/common/src/test/resources/query-tests/queries/unpivot_no_values.json
index 9f550c0319147..96b76443b6790 100644
--- a/sql/connect/common/src/test/resources/query-tests/queries/unpivot_no_values.json
+++ b/sql/connect/common/src/test/resources/query-tests/queries/unpivot_no_values.json
@@ -16,6 +16,7 @@
"unparsedIdentifier": "id"
}
}],
+ "variableColumnName": "name",
"valueColumnName": "value"
}
}
\ No newline at end of file
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/unpivot_no_values.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/unpivot_no_values.proto.bin
index ac3bad8bd04ed..b700190a9f667 100644
Binary files a/sql/connect/common/src/test/resources/query-tests/queries/unpivot_no_values.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/unpivot_no_values.proto.bin differ
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/unpivot_values.json b/sql/connect/common/src/test/resources/query-tests/queries/unpivot_values.json
index 92bc19d195c6e..6c31afb04e741 100644
--- a/sql/connect/common/src/test/resources/query-tests/queries/unpivot_values.json
+++ b/sql/connect/common/src/test/resources/query-tests/queries/unpivot_values.json
@@ -27,6 +27,7 @@
}
}]
},
+ "variableColumnName": "name",
"valueColumnName": "value"
}
}
\ No newline at end of file
diff --git a/sql/connect/common/src/test/resources/query-tests/queries/unpivot_values.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/unpivot_values.proto.bin
index 7f717cb23517b..a1cd388fd8a46 100644
Binary files a/sql/connect/common/src/test/resources/query-tests/queries/unpivot_values.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/unpivot_values.proto.bin differ
diff --git a/sql/connect/server/pom.xml b/sql/connect/server/pom.xml
index 2e7c3f81aa1dc..3350c4261e9da 100644
--- a/sql/connect/server/pom.xml
+++ b/sql/connect/server/pom.xml
@@ -111,7 +111,7 @@
org.apache.spark
spark-protobuf_${scala.binary.version}
${project.version}
- provided
+ test