"
diff --git a/common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala b/common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
index 85d460f618a79..af76056dfe928 100644
--- a/common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
+++ b/common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
@@ -19,13 +19,10 @@ package org.apache.spark
import java.net.URL
-import scala.jdk.CollectionConverters._
-
import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.core.`type`.TypeReference
import com.fasterxml.jackson.databind.json.JsonMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
-import org.apache.commons.text.StringSubstitutor
import org.apache.spark.annotation.DeveloperApi
@@ -44,13 +41,18 @@ class ErrorClassesJsonReader(jsonFileURLs: Seq[URL]) {
def getErrorMessage(errorClass: String, messageParameters: Map[String, Any]): String = {
val messageTemplate = getMessageTemplate(errorClass)
+ getErrorMessage(errorClass, messageTemplate, messageParameters)
+ }
+
+ def getErrorMessage(
+ errorClass: String,
+ messageTemplate: String,
+ messageParameters: Map[String, Any]): String = {
val sanitizedParameters = messageParameters.map {
case (key, null) => key -> "null"
case (key, value) => key -> value
}
- val sub = new StringSubstitutor(sanitizedParameters.asJava)
- sub.setEnableUndefinedVariableException(true)
- sub.setDisableSubstitutionInValues(true)
+ val sub = new StringSubstitutor(sanitizedParameters)
val errorMessage = try {
sub.replace(ErrorClassesJsonReader.TEMPLATE_REGEX.replaceAllIn(
messageTemplate, "\\$\\{$1\\}"))
@@ -76,10 +78,30 @@ class ErrorClassesJsonReader(jsonFileURLs: Seq[URL]) {
def getMessageParameters(errorClass: String): Seq[String] = {
val messageTemplate = getMessageTemplate(errorClass)
+ getMessageParametersFromTemplate(messageTemplate)
+ }
+
+ def getMessageParametersFromTemplate(messageTemplate: String): Seq[String] = {
val matches = ErrorClassesJsonReader.TEMPLATE_REGEX.findAllIn(messageTemplate).toSeq
matches.map(m => m.stripSuffix(">").stripPrefix("<"))
}
+ def getBreakingChangeInfo(errorClass: String): Option[BreakingChangeInfo] = {
+ val errorClasses = errorClass.split('.')
+ errorClasses match {
+ case Array(mainClass) =>
+ errorInfoMap.get(mainClass).flatMap(_.breakingChangeInfo)
+ case Array(mainClass, subClass) =>
+ errorInfoMap.get(mainClass).flatMap{
+ errorInfo =>
+ errorInfo.subClass.flatMap(_.get(subClass))
+ .flatMap(_.breakingChangeInfo)
+ .orElse(errorInfo.breakingChangeInfo)
+ }
+ case _ => None
+ }
+ }
+
def getMessageTemplate(errorClass: String): String = {
val errorClasses = errorClass.split("\\.")
assert(errorClasses.length == 1 || errorClasses.length == 2)
@@ -133,7 +155,7 @@ private object ErrorClassesJsonReader {
val map = mapper.readValue(url, new TypeReference[Map[String, ErrorInfo]]() {})
val errorClassWithDots = map.collectFirst {
case (errorClass, _) if errorClass.contains('.') => errorClass
- case (_, ErrorInfo(_, Some(map), _)) if map.keys.exists(_.contains('.')) =>
+ case (_, ErrorInfo(_, Some(map), _, _)) if map.keys.exists(_.contains('.')) =>
map.keys.collectFirst { case s if s.contains('.') => s }.get
}
if (errorClassWithDots.isEmpty) {
@@ -152,14 +174,17 @@ private object ErrorClassesJsonReader {
* @param subClass SubClass associated with this class.
* @param message Message format with optional placeholders (e.g. <parm>).
* The error message is constructed by concatenating the lines with newlines.
+ * @param breakingChangeInfo Additional metadata if the error is due to a breaking change.
*/
private case class ErrorInfo(
message: Seq[String],
subClass: Option[Map[String, ErrorSubInfo]],
- sqlState: Option[String]) {
+ sqlState: Option[String],
+ breakingChangeInfo: Option[BreakingChangeInfo] = None) {
// For compatibility with multi-line error messages
@JsonIgnore
- val messageTemplate: String = message.mkString("\n")
+ val messageTemplate: String = message.mkString("\n") +
+ breakingChangeInfo.map(_.migrationMessage.mkString(" ", "\n", "")).getOrElse("")
}
/**
@@ -167,11 +192,69 @@ private case class ErrorInfo(
*
* @param message Message format with optional placeholders (e.g. <parm>).
* The error message is constructed by concatenating the lines with newlines.
+ * @param breakingChangeInfo Additional metadata if the error is due to a breaking change.
*/
-private case class ErrorSubInfo(message: Seq[String]) {
+private case class ErrorSubInfo(
+ message: Seq[String],
+ breakingChangeInfo: Option[BreakingChangeInfo] = None) {
// For compatibility with multi-line error messages
@JsonIgnore
- val messageTemplate: String = message.mkString("\n")
+ val messageTemplate: String = message.mkString("\n") +
+ breakingChangeInfo.map(_.migrationMessage.mkString(" ", "\n", "")).getOrElse("")
+}
+
+/**
+ * Additional information if the error was caused by a breaking change.
+ *
+ * @param migrationMessage A message explaining how the user can migrate their job to work
+ * with the breaking change.
+ * @param mitigationConfig A spark config flag that can be used to mitigate the
+ * breaking change.
+ * @param needsAudit If true, the breaking change should be inspected manually.
+ * If false, the spark job should be retried by setting the
+ * mitigationConfig.
+ */
+class BreakingChangeInfo(
+ val migrationMessage: Seq[String],
+ val mitigationConfig: Option[MitigationConfig] = None,
+ val needsAudit: Boolean = true) {
+ override def equals(other: Any): Boolean = other match {
+ case that: BreakingChangeInfo =>
+ migrationMessage == that.migrationMessage &&
+ mitigationConfig == that.mitigationConfig &&
+ needsAudit == that.needsAudit
+ case _ => false
+ }
+
+ override def hashCode(): Int = {
+ val prime = 31
+ var result = 1
+ result = prime * result + migrationMessage.hashCode()
+ result = prime * result + mitigationConfig.hashCode()
+ result = prime * result + needsAudit.hashCode()
+ result
+ }
+}
+
+/**
+ * A spark config flag that can be used to mitigate a breaking change.
+ * @param key The spark config key.
+ * @param value The spark config value that mitigates the breaking change.
+ */
+class MitigationConfig(val key: String, val value: String) {
+ override def equals(other: Any): Boolean = other match {
+ case that: MitigationConfig =>
+ key == that.key && value == that.value
+ case _ => false
+ }
+
+ override def hashCode(): Int = {
+ val prime = 31
+ var result = 1
+ result = prime * result + key.hashCode()
+ result = prime * result + value.hashCode()
+ result
+ }
}
/**
diff --git a/common/utils/src/main/scala/org/apache/spark/SparkException.scala b/common/utils/src/main/scala/org/apache/spark/SparkException.scala
index 00989fd29095c..f438c62253475 100644
--- a/common/utils/src/main/scala/org/apache/spark/SparkException.scala
+++ b/common/utils/src/main/scala/org/apache/spark/SparkException.scala
@@ -131,6 +131,14 @@ object SparkException {
messageParameters: java.util.Map[String, String]): Map[String, String] = {
messageParameters.asScala.toMap
}
+
+ def mustOverrideOneMethodError(methodName: String): RuntimeException = {
+ val msg = s"You must override one `$methodName`. It's preferred to not override the " +
+ "deprecated one."
+ new SparkRuntimeException(
+ "INTERNAL_ERROR",
+ Map("message" -> msg))
+ }
}
/**
@@ -145,8 +153,8 @@ private[spark] class SparkDriverExecutionException(cause: Throwable)
* Exception thrown when the main user code is run as a child process (e.g. pyspark) and we want
* the parent SparkSubmit process to exit with the same exit code.
*/
-private[spark] case class SparkUserAppException(exitCode: Int)
- extends SparkException(s"User application exited with $exitCode")
+private[spark] case class SparkUserAppException(exitCode: Int, cause: Throwable = null)
+ extends SparkException(s"User application exited with $exitCode", cause)
/**
* Exception thrown when the relative executor to access is dead.
diff --git a/common/utils/src/main/scala/org/apache/spark/SparkThrowableHelper.scala b/common/utils/src/main/scala/org/apache/spark/SparkThrowableHelper.scala
index b6c2b176de62b..6a0388095d815 100644
--- a/common/utils/src/main/scala/org/apache/spark/SparkThrowableHelper.scala
+++ b/common/utils/src/main/scala/org/apache/spark/SparkThrowableHelper.scala
@@ -37,6 +37,11 @@ private[spark] object SparkThrowableHelper {
// of this ticket: https://issues.apache.org/jira/browse/SPARK-47429
Seq(SparkClassUtils.getSparkClassLoader.getResource("error/error-conditions.json")))
+ // This method is only used to get default message template in SparkThrowable interface.
+ def getMessageTemplate(errorClass: String): String = {
+ errorReader.getMessageTemplate(errorClass)
+ }
+
def getMessage(
errorClass: String,
messageParameters: Map[String, String]): String = {
@@ -55,6 +60,26 @@ private[spark] object SparkThrowableHelper {
context: String): String = {
val displayMessage = errorReader.getErrorMessage(errorClass, messageParameters)
val sqlState = getSqlState(errorClass)
+ formatErrorMessage(errorClass, displayMessage, sqlState, context)
+ }
+
+ def getMessage(
+ errorClass: String,
+ sqlState: String,
+ messageTemplate: String,
+ messageParameters: Map[String, String]): String = {
+ val displayMessage = errorReader.getErrorMessage(
+ errorClass,
+ messageTemplate,
+ messageParameters)
+ formatErrorMessage(errorClass, displayMessage, sqlState, "")
+ }
+
+ def formatErrorMessage(
+ errorClass: String,
+ displayMessage: String,
+ sqlState: String,
+ context: String): String = {
val displaySqlState = if (sqlState == null) "" else s" SQLSTATE: $sqlState"
val displayQueryContext = (if (context.isEmpty) "" else "\n") + context
val prefix = if (errorClass.startsWith("_LEGACY_ERROR_")) "" else s"[$errorClass] "
@@ -73,6 +98,14 @@ private[spark] object SparkThrowableHelper {
errorReader.getMessageParameters(errorClass)
}
+ def getBreakingChangeInfo(errorClass: String): Option[BreakingChangeInfo] = {
+ if (errorClass == null) {
+ None
+ } else {
+ errorReader.getBreakingChangeInfo(errorClass)
+ }
+ }
+
def isInternalError(errorClass: String): Boolean = {
errorClass != null && errorClass.startsWith("INTERNAL_ERROR")
}
@@ -98,7 +131,27 @@ private[spark] object SparkThrowableHelper {
g.writeStartObject()
g.writeStringField("errorClass", errorClass)
if (format == STANDARD) {
- g.writeStringField("messageTemplate", errorReader.getMessageTemplate(errorClass))
+ val messageTemplate = e.getDefaultMessageTemplate
+ // This is required to properly handle null values when the messageTemplate
+ // is not available, ensuring correct JSON serialization of the field.
+ if (messageTemplate != null) {
+ g.writeStringField("messageTemplate", messageTemplate)
+ } else {
+ g.writeNullField("messageTemplate")
+ }
+ errorReader.getBreakingChangeInfo(errorClass).foreach { breakingChangeInfo =>
+ g.writeObjectFieldStart("breakingChangeInfo")
+ g.writeStringField("migrationMessage",
+ breakingChangeInfo.migrationMessage.mkString("\n"))
+ breakingChangeInfo.mitigationConfig.foreach { mitigationConfig =>
+ g.writeObjectFieldStart("mitigationConfig")
+ g.writeStringField("key", mitigationConfig.key)
+ g.writeStringField("value", mitigationConfig.value)
+ g.writeEndObject()
+ }
+ g.writeBooleanField("needsAudit", breakingChangeInfo.needsAudit)
+ g.writeEndObject()
+ }
}
val sqlState = e.getSqlState
if (sqlState != null) g.writeStringField("sqlState", sqlState)
diff --git a/common/utils/src/main/scala/org/apache/spark/StringSubstitutor.scala b/common/utils/src/main/scala/org/apache/spark/StringSubstitutor.scala
new file mode 100644
index 0000000000000..6513c7dc7e8ec
--- /dev/null
+++ b/common/utils/src/main/scala/org/apache/spark/StringSubstitutor.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark
+
+import scala.collection.mutable.{ListBuffer, StringBuilder}
+
+class StringSubstitutor(
+ resolver: Map[String, Any],
+ prefix: String = "${",
+ suffix: String = "}",
+ escape: Char = '$',
+ valueDelimiter: String = ":-",
+ preserveEscapes: Boolean = true,
+ enableSubstitutionInVariables: Boolean = false,
+ enableUndefinedVariableException: Boolean = true) {
+
+ private val prefixLen = prefix.length
+ private val suffixLen = suffix.length
+
+ def replace(source: String): String = {
+ if (source == null) return null
+ val buf = new StringBuilder(source)
+ if (!substitute(buf, 0, buf.length, ListBuffer.empty[String])) {
+ source
+ } else {
+ buf.toString
+ }
+ }
+
+ private def substitute(
+ buf: StringBuilder,
+ offset: Int,
+ length: Int,
+ priorVariables: ListBuffer[String]): Boolean = {
+ var altered = false
+ var currentPos = offset
+ val endPos = offset + length
+
+ while (currentPos < endPos) {
+ // Find prefix
+ val prefixPos = buf.indexOf(prefix, currentPos)
+ if (prefixPos < 0 || prefixPos >= endPos) {
+ return altered
+ }
+
+ if (prefixPos > 0 && buf.charAt(prefixPos - 1) == escape) {
+ if (preserveEscapes) {
+ currentPos = prefixPos + prefixLen
+ } else {
+ buf.deleteCharAt(prefixPos - 1)
+ altered = true
+ currentPos = prefixPos - 1 + prefixLen // adjust position
+ }
+ } else {
+ // Find suffix
+ val suffixPos = buf.indexOf(suffix, prefixPos + prefixLen)
+ if (suffixPos < 0) {
+ currentPos = prefixPos + prefixLen
+ } else {
+ // Extract variable name and default
+ val varFull = buf.substring(prefixPos + prefixLen, suffixPos)
+ val index = varFull.indexOf(valueDelimiter)
+ val (varName, defaultValue) = if (index >= 0) {
+ (varFull.substring(0, index), Some(varFull.substring(index + valueDelimiter.length)))
+ } else {
+ (varFull, None)
+ }
+
+ // Check for cycle
+ if (priorVariables.contains(varName)) {
+ throw new IllegalStateException(
+ s"Infinite loop in property interpolation of $varName")
+ }
+
+ resolver.get(varName) match {
+ case Some(value) =>
+ var replacement = value.toString
+ if (enableSubstitutionInVariables) {
+ val newPrior = priorVariables.clone() += varName
+ val tempBuf = new StringBuilder(replacement)
+ if (substitute(tempBuf, 0, tempBuf.length, newPrior)) {
+ replacement = tempBuf.toString
+ }
+ }
+ buf.replace(prefixPos, suffixPos + suffixLen, replacement)
+ // Adjust positions and recurse on the changed part
+ val changeInLength = replacement.length - (suffixPos + suffixLen - prefixPos)
+ val newLength = length + changeInLength
+ substitute(buf, prefixPos + replacement.length, newLength, priorVariables)
+ return true // restart scan after change
+
+ case None =>
+ defaultValue match {
+ case Some(value) =>
+ buf.replace(prefixPos, suffixPos + suffixLen, value)
+ val changeInLength = value.length - (suffixPos + suffixLen - prefixPos)
+ val newLength = length + changeInLength
+ substitute(buf, prefixPos + value.length, length, priorVariables)
+ return true
+ case None =>
+ if (enableUndefinedVariableException) {
+ throw new IllegalArgumentException(s"Undefined variable: $varName")
+ }
+ }
+ }
+ currentPos = suffixPos + suffixLen
+ }
+ }
+ }
+ altered
+ }
+}
diff --git a/common/utils/src/main/java/org/apache/spark/api/java/function/package.scala b/common/utils/src/main/scala/org/apache/spark/api/java/function/package.scala
similarity index 100%
rename from common/utils/src/main/java/org/apache/spark/api/java/function/package.scala
rename to common/utils/src/main/scala/org/apache/spark/api/java/function/package.scala
diff --git a/common/utils/src/main/scala/org/apache/spark/internal/LogKey.scala b/common/utils/src/main/scala/org/apache/spark/internal/LogKey.scala
deleted file mode 100644
index b0fae3fd9443b..0000000000000
--- a/common/utils/src/main/scala/org/apache/spark/internal/LogKey.scala
+++ /dev/null
@@ -1,915 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.internal
-
-import java.util.Locale
-
-/**
- * All structured logging `keys` used in `MDC` must be extends `LogKey`
- *
- *
- * `LogKey`s serve as identifiers for mapped diagnostic contexts (MDC) within logs.
- * Follow these guidelines when adding a new LogKey:
- *
- * -
- * Define all structured logging keys in `LogKey.scala`, and sort them alphabetically for
- * ease of search.
- *
- * -
- * Use `UPPER_SNAKE_CASE` for key names.
- *
- * -
- * Key names should be both simple and broad, yet include specific identifiers like `STAGE_ID`,
- * `TASK_ID`, and `JOB_ID` when needed for clarity. For instance, use `MAX_ATTEMPTS` as a
- * general key instead of creating separate keys for each scenario such as
- * `EXECUTOR_STATE_SYNC_MAX_ATTEMPTS` and `MAX_TASK_FAILURES`.
- * This balances simplicity with the detail needed for effective logging.
- *
- * -
- * Use abbreviations in names if they are widely understood,
- * such as `APP_ID` for APPLICATION_ID, and `K8S` for KUBERNETES.
- *
- * -
- * For time-related keys, use milliseconds as the unit of time.
- *
- *
- */
-trait LogKey {
- private lazy val _name: String = getClass.getSimpleName.stripSuffix("$").toLowerCase(Locale.ROOT)
- def name: String = _name
-}
-
-/**
- * Various keys used for mapped diagnostic contexts(MDC) in logging. All structured logging keys
- * should be defined here for standardization.
- */
-private[spark] object LogKeys {
- case object ACCUMULATOR_ID extends LogKey
- case object ACL_ENABLED extends LogKey
- case object ACTUAL_NUM_FILES extends LogKey
- case object ACTUAL_PARTITION_COLUMN extends LogKey
- case object ADDED_JARS extends LogKey
- case object ADMIN_ACLS extends LogKey
- case object ADMIN_ACL_GROUPS extends LogKey
- case object ADVISORY_TARGET_SIZE extends LogKey
- case object AGGREGATE_FUNCTIONS extends LogKey
- case object ALIGNED_FROM_TIME extends LogKey
- case object ALIGNED_TO_TIME extends LogKey
- case object ALPHA extends LogKey
- case object ANALYSIS_ERROR extends LogKey
- case object APP_ATTEMPT_ID extends LogKey
- case object APP_ATTEMPT_SHUFFLE_MERGE_ID extends LogKey
- case object APP_DESC extends LogKey
- case object APP_EXECUTOR_ID extends LogKey
- case object APP_ID extends LogKey
- case object APP_NAME extends LogKey
- case object APP_STATE extends LogKey
- case object ARCHIVE_NAME extends LogKey
- case object ARGS extends LogKey
- case object ARTIFACTS extends LogKey
- case object ARTIFACT_ID extends LogKey
- case object ATTRIBUTE_MAP extends LogKey
- case object AUTH_ENABLED extends LogKey
- case object AVG_BATCH_PROC_TIME extends LogKey
- case object BACKUP_FILE extends LogKey
- case object BARRIER_EPOCH extends LogKey
- case object BARRIER_ID extends LogKey
- case object BATCH_ID extends LogKey
- case object BATCH_NAME extends LogKey
- case object BATCH_TIMES extends LogKey
- case object BATCH_TIMESTAMP extends LogKey
- case object BATCH_WRITE extends LogKey
- case object BIND_ADDRESS extends LogKey
- case object BLOCK_GENERATOR_STATUS extends LogKey
- case object BLOCK_ID extends LogKey
- case object BLOCK_IDS extends LogKey
- case object BLOCK_MANAGER_ID extends LogKey
- case object BLOCK_MANAGER_IDS extends LogKey
- case object BLOCK_TYPE extends LogKey
- case object BOOT extends LogKey
- case object BOOTSTRAP_TIME extends LogKey
- case object BOOT_TIME extends LogKey
- case object BROADCAST extends LogKey
- case object BROADCAST_ID extends LogKey
- case object BROADCAST_OUTPUT_STATUS_SIZE extends LogKey
- case object BUCKET extends LogKey
- case object BYTECODE_SIZE extends LogKey
- case object BYTE_BUFFER extends LogKey
- case object BYTE_SIZE extends LogKey
- case object CACHED_TABLE_PARTITION_METADATA_SIZE extends LogKey
- case object CACHE_AUTO_REMOVED_SIZE extends LogKey
- case object CACHE_SIZE extends LogKey
- case object CACHE_UNTIL_HIGHEST_CONSUMED_SIZE extends LogKey
- case object CACHE_UNTIL_LAST_PRODUCED_SIZE extends LogKey
- case object CALL_SITE_LONG_FORM extends LogKey
- case object CALL_SITE_SHORT_FORM extends LogKey
- case object CANCEL_FUTURE_JOBS extends LogKey
- case object CATALOG_NAME extends LogKey
- case object CATEGORICAL_FEATURES extends LogKey
- case object CHECKPOINT_FILE extends LogKey
- case object CHECKPOINT_INTERVAL extends LogKey
- case object CHECKPOINT_LOCATION extends LogKey
- case object CHECKPOINT_PATH extends LogKey
- case object CHECKPOINT_ROOT extends LogKey
- case object CHECKPOINT_TIME extends LogKey
- case object CHOSEN_WATERMARK extends LogKey
- case object CLASSIFIER extends LogKey
- case object CLASS_LOADER extends LogKey
- case object CLASS_NAME extends LogKey
- case object CLASS_PATH extends LogKey
- case object CLASS_PATHS extends LogKey
- case object CLAUSES extends LogKey
- case object CLEANUP_LOCAL_DIRS extends LogKey
- case object CLUSTER_CENTROIDS extends LogKey
- case object CLUSTER_ID extends LogKey
- case object CLUSTER_LABEL extends LogKey
- case object CLUSTER_LEVEL extends LogKey
- case object CLUSTER_WEIGHT extends LogKey
- case object CODE extends LogKey
- case object CODEC_LEVEL extends LogKey
- case object CODEC_NAME extends LogKey
- case object CODEGEN_STAGE_ID extends LogKey
- case object COLUMN_DATA_TYPE_SOURCE extends LogKey
- case object COLUMN_DATA_TYPE_TARGET extends LogKey
- case object COLUMN_DEFAULT_VALUE extends LogKey
- case object COLUMN_NAME extends LogKey
- case object COMMAND extends LogKey
- case object COMMAND_OUTPUT extends LogKey
- case object COMMITTED_VERSION extends LogKey
- case object COMPACT_INTERVAL extends LogKey
- case object COMPONENT extends LogKey
- case object COMPUTE extends LogKey
- case object CONFIG extends LogKey
- case object CONFIG2 extends LogKey
- case object CONFIG3 extends LogKey
- case object CONFIG4 extends LogKey
- case object CONFIG5 extends LogKey
- case object CONFIG_DEPRECATION_MESSAGE extends LogKey
- case object CONFIG_KEY_UPDATED extends LogKey
- case object CONFIG_VERSION extends LogKey
- case object CONSUMER extends LogKey
- case object CONTAINER extends LogKey
- case object CONTAINER_ID extends LogKey
- case object CONTAINER_STATE extends LogKey
- case object CONTEXT extends LogKey
- case object COST extends LogKey
- case object COUNT extends LogKey
- case object CREATED_POOL_NAME extends LogKey
- case object CREATION_SITE extends LogKey
- case object CREDENTIALS_RENEWAL_INTERVAL_RATIO extends LogKey
- case object CROSS_VALIDATION_METRIC extends LogKey
- case object CROSS_VALIDATION_METRICS extends LogKey
- case object CSV_HEADER_COLUMN_NAME extends LogKey
- case object CSV_HEADER_COLUMN_NAMES extends LogKey
- case object CSV_HEADER_LENGTH extends LogKey
- case object CSV_SCHEMA_FIELD_NAME extends LogKey
- case object CSV_SCHEMA_FIELD_NAMES extends LogKey
- case object CSV_SOURCE extends LogKey
- case object CURRENT_BATCH_ID extends LogKey
- case object CURRENT_DISK_SIZE extends LogKey
- case object CURRENT_FILE extends LogKey
- case object CURRENT_MEMORY_SIZE extends LogKey
- case object CURRENT_PATH extends LogKey
- case object CURRENT_TIME extends LogKey
- case object DATA extends LogKey
- case object DATABASE_NAME extends LogKey
- case object DATAFRAME_CACHE_ENTRY extends LogKey
- case object DATAFRAME_ID extends LogKey
- case object DATA_FILE extends LogKey
- case object DATA_SOURCE extends LogKey
- case object DATA_SOURCES extends LogKey
- case object DEFAULT_COMPACT_INTERVAL extends LogKey
- case object DEFAULT_ISOLATION_LEVEL extends LogKey
- case object DEFAULT_NAME extends LogKey
- case object DEFAULT_VALUE extends LogKey
- case object DELAY extends LogKey
- case object DELEGATE extends LogKey
- case object DELTA extends LogKey
- case object DEPRECATED_KEY extends LogKey
- case object DERIVATIVE extends LogKey
- case object DESCRIPTION extends LogKey
- case object DESIRED_NUM_PARTITIONS extends LogKey
- case object DESIRED_TREE_DEPTH extends LogKey
- case object DESTINATION_PATH extends LogKey
- case object DFS_FILE extends LogKey
- case object DIFF_DELTA extends LogKey
- case object DIVISIBLE_CLUSTER_INDICES_SIZE extends LogKey
- case object DRIVER_ID extends LogKey
- case object DRIVER_MEMORY_SIZE extends LogKey
- case object DRIVER_STATE extends LogKey
- case object DROPPED_PARTITIONS extends LogKey
- case object DSTREAM extends LogKey
- case object DURATION extends LogKey
- case object EARLIEST_LOADED_VERSION extends LogKey
- case object EFFECTIVE_STORAGE_LEVEL extends LogKey
- case object ELAPSED_TIME extends LogKey
- case object ENCODING extends LogKey
- case object ENDPOINT_NAME extends LogKey
- case object END_INDEX extends LogKey
- case object END_POINT extends LogKey
- case object END_VERSION extends LogKey
- case object ENGINE extends LogKey
- case object EPOCH extends LogKey
- case object ERROR extends LogKey
- case object ESTIMATOR_PARAM_MAP extends LogKey
- case object EVALUATED_FILTERS extends LogKey
- case object EVENT extends LogKey
- case object EVENT_LOG_DESTINATION extends LogKey
- case object EVENT_LOOP extends LogKey
- case object EVENT_NAME extends LogKey
- case object EVENT_QUEUE extends LogKey
- case object EXCEPTION extends LogKey
- case object EXECUTE_INFO extends LogKey
- case object EXECUTE_KEY extends LogKey
- case object EXECUTION_MEMORY_SIZE extends LogKey
- case object EXECUTION_PLAN_LEAVES extends LogKey
- case object EXECUTOR_BACKEND extends LogKey
- case object EXECUTOR_ENVS extends LogKey
- case object EXECUTOR_ENV_REGEX extends LogKey
- case object EXECUTOR_ID extends LogKey
- case object EXECUTOR_IDS extends LogKey
- case object EXECUTOR_LAUNCH_COMMANDS extends LogKey
- case object EXECUTOR_MEMORY_OFFHEAP extends LogKey
- case object EXECUTOR_MEMORY_OVERHEAD_SIZE extends LogKey
- case object EXECUTOR_MEMORY_SIZE extends LogKey
- case object EXECUTOR_RESOURCES extends LogKey
- case object EXECUTOR_SHUFFLE_INFO extends LogKey
- case object EXECUTOR_STATE extends LogKey
- case object EXECUTOR_TIMEOUT extends LogKey
- case object EXECUTOR_USER_CLASS_PATH_FIRST extends LogKey
- case object EXEC_AMOUNT extends LogKey
- case object EXISTING_FILE extends LogKey
- case object EXISTING_PATH extends LogKey
- case object EXIT_CODE extends LogKey
- case object EXPECTED_NUM_FILES extends LogKey
- case object EXPECTED_PARTITION_COLUMN extends LogKey
- case object EXPIRY_TIMESTAMP extends LogKey
- case object EXPR extends LogKey
- case object EXPR_TERMS extends LogKey
- case object EXTENDED_EXPLAIN_GENERATOR extends LogKey
- case object FAILED_STAGE extends LogKey
- case object FAILED_STAGE_NAME extends LogKey
- case object FAILURES extends LogKey
- case object FALLBACK_VERSION extends LogKey
- case object FEATURE_COLUMN extends LogKey
- case object FEATURE_DIMENSION extends LogKey
- case object FEATURE_NAME extends LogKey
- case object FETCH_SIZE extends LogKey
- case object FIELD_NAME extends LogKey
- case object FIELD_TYPE extends LogKey
- case object FILES extends LogKey
- case object FILE_ABSOLUTE_PATH extends LogKey
- case object FILE_END_OFFSET extends LogKey
- case object FILE_FORMAT extends LogKey
- case object FILE_FORMAT2 extends LogKey
- case object FILE_LENGTH_XATTR extends LogKey
- case object FILE_MODIFICATION_TIME extends LogKey
- case object FILE_NAME extends LogKey
- case object FILE_NAME2 extends LogKey
- case object FILE_NAME3 extends LogKey
- case object FILE_NAMES extends LogKey
- case object FILE_START_OFFSET extends LogKey
- case object FILE_SYSTEM extends LogKey
- case object FILE_VERSION extends LogKey
- case object FILTER extends LogKey
- case object FINAL_CONTEXT extends LogKey
- case object FINAL_OUTPUT_PATH extends LogKey
- case object FINAL_PATH extends LogKey
- case object FINISH_TIME extends LogKey
- case object FINISH_TRIGGER_DURATION extends LogKey
- case object FLOW_NAME extends LogKey
- case object FREE_MEMORY_SIZE extends LogKey
- case object FROM_OFFSET extends LogKey
- case object FROM_TIME extends LogKey
- case object FS_DATA_OUTPUT_STREAM extends LogKey
- case object FUNCTION_NAME extends LogKey
- case object FUNCTION_PARAM extends LogKey
- case object GLOBAL_INIT_FILE extends LogKey
- case object GLOBAL_WATERMARK extends LogKey
- case object GROUP_BY_EXPRS extends LogKey
- case object GROUP_ID extends LogKey
- case object HADOOP_VERSION extends LogKey
- case object HASH_JOIN_KEYS extends LogKey
- case object HASH_MAP_SIZE extends LogKey
- case object HEARTBEAT extends LogKey
- case object HEARTBEAT_INTERVAL extends LogKey
- case object HISTORY_DIR extends LogKey
- case object HIVE_CLIENT_VERSION extends LogKey
- case object HIVE_METASTORE_VERSION extends LogKey
- case object HIVE_OPERATION_STATE extends LogKey
- case object HIVE_OPERATION_TYPE extends LogKey
- case object HOST extends LogKey
- case object HOSTS extends LogKey
- case object HOST_LOCAL_BLOCKS_SIZE extends LogKey
- case object HOST_PORT extends LogKey
- case object HOST_PORT2 extends LogKey
- case object HUGE_METHOD_LIMIT extends LogKey
- case object HYBRID_STORE_DISK_BACKEND extends LogKey
- case object IDENTIFIER extends LogKey
- case object INCOMPATIBLE_TYPES extends LogKey
- case object INDEX extends LogKey
- case object INDEX_FILE extends LogKey
- case object INDEX_NAME extends LogKey
- case object INFERENCE_MODE extends LogKey
- case object INIT extends LogKey
- case object INITIAL_CAPACITY extends LogKey
- case object INITIAL_HEARTBEAT_INTERVAL extends LogKey
- case object INIT_MODE extends LogKey
- case object INIT_TIME extends LogKey
- case object INPUT extends LogKey
- case object INPUT_SPLIT extends LogKey
- case object INTEGRAL extends LogKey
- case object INTERRUPT_THREAD extends LogKey
- case object INTERVAL extends LogKey
- case object INVALID_PARAMS extends LogKey
- case object ISOLATION_LEVEL extends LogKey
- case object ISSUE_DATE extends LogKey
- case object IS_NETWORK_REQUEST_DONE extends LogKey
- case object JAR_ENTRY extends LogKey
- case object JAR_MESSAGE extends LogKey
- case object JAR_URL extends LogKey
- case object JAVA_VERSION extends LogKey
- case object JAVA_VM_NAME extends LogKey
- case object JOB_ID extends LogKey
- case object JOIN_CONDITION extends LogKey
- case object JOIN_CONDITION_SUB_EXPR extends LogKey
- case object JOIN_TYPE extends LogKey
- case object K8S_CONTEXT extends LogKey
- case object KEY extends LogKey
- case object KEY2 extends LogKey
- case object KEYTAB extends LogKey
- case object KEYTAB_FILE extends LogKey
- case object KILL_EXECUTORS extends LogKey
- case object KINESIS_REASON extends LogKey
- case object LABEL_COLUMN extends LogKey
- case object LARGEST_CLUSTER_INDEX extends LogKey
- case object LAST_ACCESS_TIME extends LogKey
- case object LAST_COMMITTED_CHECKPOINT_ID extends LogKey
- case object LAST_COMMIT_BASED_CHECKPOINT_ID extends LogKey
- case object LAST_VALID_TIME extends LogKey
- case object LATEST_BATCH_ID extends LogKey
- case object LATEST_COMMITTED_BATCH_ID extends LogKey
- case object LATEST_SHUFFLE_MERGE_ID extends LogKey
- case object LEARNING_RATE extends LogKey
- case object LEFT_EXPR extends LogKey
- case object LEFT_LOGICAL_PLAN_STATS_SIZE_IN_BYTES extends LogKey
- case object LINE extends LogKey
- case object LINEAGE extends LogKey
- case object LINE_NUM extends LogKey
- case object LISTENER extends LogKey
- case object LOADED_CHECKPOINT_ID extends LogKey
- case object LOADED_VERSION extends LogKey
- case object LOAD_FACTOR extends LogKey
- case object LOAD_TIME extends LogKey
- case object LOCALE extends LogKey
- case object LOCAL_BLOCKS_SIZE extends LogKey
- case object LOCAL_SCRATCH_DIR extends LogKey
- case object LOCATION extends LogKey
- case object LOGICAL_PLAN extends LogKey
- case object LOGICAL_PLAN_COLUMNS extends LogKey
- case object LOGICAL_PLAN_LEAVES extends LogKey
- case object LOG_ID extends LogKey
- case object LOG_LEVEL extends LogKey
- case object LOG_OFFSET extends LogKey
- case object LOG_TYPE extends LogKey
- case object LOSSES extends LogKey
- case object LOWER_BOUND extends LogKey
- case object MALFORMATTED_STRING extends LogKey
- case object MAP_ID extends LogKey
- case object MASTER_URL extends LogKey
- case object MAX_ATTEMPTS extends LogKey
- case object MAX_CACHE_UNTIL_HIGHEST_CONSUMED_SIZE extends LogKey
- case object MAX_CACHE_UNTIL_LAST_PRODUCED_SIZE extends LogKey
- case object MAX_CAPACITY extends LogKey
- case object MAX_CATEGORIES extends LogKey
- case object MAX_EXECUTOR_FAILURES extends LogKey
- case object MAX_FILE_VERSION extends LogKey
- case object MAX_JVM_METHOD_PARAMS_LENGTH extends LogKey
- case object MAX_MEMORY_SIZE extends LogKey
- case object MAX_METHOD_CODE_SIZE extends LogKey
- case object MAX_NUM_BINS extends LogKey
- case object MAX_NUM_CHUNKS extends LogKey
- case object MAX_NUM_FILES extends LogKey
- case object MAX_NUM_LOG_POLICY extends LogKey
- case object MAX_NUM_PARTITIONS extends LogKey
- case object MAX_NUM_POSSIBLE_BINS extends LogKey
- case object MAX_NUM_ROWS_IN_MEMORY_BUFFER extends LogKey
- case object MAX_SEEN_VERSION extends LogKey
- case object MAX_SERVICE_NAME_LENGTH extends LogKey
- case object MAX_SIZE extends LogKey
- case object MAX_SLOTS extends LogKey
- case object MAX_SPLIT_BYTES extends LogKey
- case object MAX_TABLE_PARTITION_METADATA_SIZE extends LogKey
- case object MEMORY_CONSUMER extends LogKey
- case object MEMORY_POOL_NAME extends LogKey
- case object MEMORY_SIZE extends LogKey
- case object MEMORY_THRESHOLD_SIZE extends LogKey
- case object MERGE_DIR_NAME extends LogKey
- case object MESSAGE extends LogKey
- case object METADATA extends LogKey
- case object METADATA_DIRECTORY extends LogKey
- case object METADATA_JSON extends LogKey
- case object META_FILE extends LogKey
- case object METHOD_NAME extends LogKey
- case object METHOD_PARAM_TYPES extends LogKey
- case object METRICS_JSON extends LogKey
- case object METRIC_NAME extends LogKey
- case object MINI_BATCH_FRACTION extends LogKey
- case object MIN_COMPACTION_BATCH_ID extends LogKey
- case object MIN_NUM_FREQUENT_PATTERN extends LogKey
- case object MIN_POINT_PER_CLUSTER extends LogKey
- case object MIN_RATE extends LogKey
- case object MIN_SEEN_VERSION extends LogKey
- case object MIN_SHARE extends LogKey
- case object MIN_SIZE extends LogKey
- case object MIN_TIME extends LogKey
- case object MIN_VERSIONS_TO_DELETE extends LogKey
- case object MIN_VERSION_NUM extends LogKey
- case object MISSING_PARENT_STAGES extends LogKey
- case object MODEL_WEIGHTS extends LogKey
- case object MODIFY_ACLS extends LogKey
- case object MODIFY_ACLS_GROUPS extends LogKey
- case object MODULE_NAME extends LogKey
- case object NAME extends LogKey
- case object NAMESPACE extends LogKey
- case object NETWORK_IF extends LogKey
- case object NEW_FEATURE_COLUMN_NAME extends LogKey
- case object NEW_LABEL_COLUMN_NAME extends LogKey
- case object NEW_PATH extends LogKey
- case object NEW_RDD_ID extends LogKey
- case object NEW_STATE extends LogKey
- case object NEW_VALUE extends LogKey
- case object NEXT_RENEWAL_TIME extends LogKey
- case object NODES extends LogKey
- case object NODE_LOCATION extends LogKey
- case object NON_BUILT_IN_CONNECTORS extends LogKey
- case object NORM extends LogKey
- case object NUM_ADDED_PARTITIONS extends LogKey
- case object NUM_APPS extends LogKey
- case object NUM_ATTEMPT extends LogKey
- case object NUM_BATCHES extends LogKey
- case object NUM_BIN extends LogKey
- case object NUM_BLOCKS extends LogKey
- case object NUM_BLOCK_IDS extends LogKey
- case object NUM_BROADCAST_BLOCK extends LogKey
- case object NUM_BYTES extends LogKey
- case object NUM_BYTES_CURRENT extends LogKey
- case object NUM_BYTES_EVICTED extends LogKey
- case object NUM_BYTES_MAX extends LogKey
- case object NUM_BYTES_TO_FREE extends LogKey
- case object NUM_BYTES_TO_WARN extends LogKey
- case object NUM_BYTES_USED extends LogKey
- case object NUM_CATEGORIES extends LogKey
- case object NUM_CHECKSUM_FILE extends LogKey
- case object NUM_CHUNKS extends LogKey
- case object NUM_CLASSES extends LogKey
- case object NUM_COEFFICIENTS extends LogKey
- case object NUM_COLUMNS extends LogKey
- case object NUM_CONCURRENT_WRITER extends LogKey
- case object NUM_CORES extends LogKey
- case object NUM_DATA_FILE extends LogKey
- case object NUM_DATA_FILES extends LogKey
- case object NUM_DECOMMISSIONED extends LogKey
- case object NUM_DRIVERS extends LogKey
- case object NUM_DROPPED_PARTITIONS extends LogKey
- case object NUM_EFFECTIVE_RULE_OF_RUNS extends LogKey
- case object NUM_ELEMENTS_SPILL_THRESHOLD extends LogKey
- case object NUM_EVENTS extends LogKey
- case object NUM_EXAMPLES extends LogKey
- case object NUM_EXECUTORS extends LogKey
- case object NUM_EXECUTORS_EXITED extends LogKey
- case object NUM_EXECUTORS_KILLED extends LogKey
- case object NUM_EXECUTOR_CORES extends LogKey
- case object NUM_EXECUTOR_CORES_REMAINING extends LogKey
- case object NUM_EXECUTOR_CORES_TOTAL extends LogKey
- case object NUM_EXECUTOR_DESIRED extends LogKey
- case object NUM_EXECUTOR_LAUNCH extends LogKey
- case object NUM_EXECUTOR_TARGET extends LogKey
- case object NUM_FAILURES extends LogKey
- case object NUM_FEATURES extends LogKey
- case object NUM_FILES extends LogKey
- case object NUM_FILES_COPIED extends LogKey
- case object NUM_FILES_FAILED_TO_DELETE extends LogKey
- case object NUM_FILES_REUSED extends LogKey
- case object NUM_FREQUENT_ITEMS extends LogKey
- case object NUM_HOST_LOCAL_BLOCKS extends LogKey
- case object NUM_INDEX_FILE extends LogKey
- case object NUM_INDEX_FILES extends LogKey
- case object NUM_ITERATIONS extends LogKey
- case object NUM_KAFKA_PULLS extends LogKey
- case object NUM_KAFKA_RECORDS_PULLED extends LogKey
- case object NUM_LAGGING_STORES extends LogKey
- case object NUM_LEADING_SINGULAR_VALUES extends LogKey
- case object NUM_LEFT_PARTITION_VALUES extends LogKey
- case object NUM_LOADED_ENTRIES extends LogKey
- case object NUM_LOCAL_BLOCKS extends LogKey
- case object NUM_LOCAL_DIRS extends LogKey
- case object NUM_LOCAL_FREQUENT_PATTERN extends LogKey
- case object NUM_MERGERS extends LogKey
- case object NUM_MERGER_LOCATIONS extends LogKey
- case object NUM_META_FILES extends LogKey
- case object NUM_NODES extends LogKey
- case object NUM_PARTITIONS extends LogKey
- case object NUM_PARTITIONS2 extends LogKey
- case object NUM_PATHS extends LogKey
- case object NUM_PEERS extends LogKey
- case object NUM_PEERS_REPLICATED_TO extends LogKey
- case object NUM_PEERS_TO_REPLICATE_TO extends LogKey
- case object NUM_PENDING_LAUNCH_TASKS extends LogKey
- case object NUM_POD extends LogKey
- case object NUM_POD_SHARED_SLOT extends LogKey
- case object NUM_POD_TARGET extends LogKey
- case object NUM_POINT extends LogKey
- case object NUM_PREFIXES extends LogKey
- case object NUM_PRUNED extends LogKey
- case object NUM_PUSH_MERGED_LOCAL_BLOCKS extends LogKey
- case object NUM_RECEIVERS extends LogKey
- case object NUM_RECORDS_READ extends LogKey
- case object NUM_RELEASED_LOCKS extends LogKey
- case object NUM_REMAINED extends LogKey
- case object NUM_REMOTE_BLOCKS extends LogKey
- case object NUM_REMOVED_WORKERS extends LogKey
- case object NUM_REPLICAS extends LogKey
- case object NUM_REQUESTS extends LogKey
- case object NUM_REQUEST_SYNC_TASK extends LogKey
- case object NUM_RESOURCE_SLOTS extends LogKey
- case object NUM_RETRIES extends LogKey
- case object NUM_RETRY extends LogKey
- case object NUM_RIGHT_PARTITION_VALUES extends LogKey
- case object NUM_ROWS extends LogKey
- case object NUM_RULE_OF_RUNS extends LogKey
- case object NUM_SEQUENCES extends LogKey
- case object NUM_SKIPPED extends LogKey
- case object NUM_SLOTS extends LogKey
- case object NUM_SPILLS extends LogKey
- case object NUM_SPILL_WRITERS extends LogKey
- case object NUM_SUB_DIRS extends LogKey
- case object NUM_SUCCESSFUL_TASKS extends LogKey
- case object NUM_TASKS extends LogKey
- case object NUM_TASK_CPUS extends LogKey
- case object NUM_TRAIN_WORD extends LogKey
- case object NUM_UNFINISHED_DECOMMISSIONED extends LogKey
- case object NUM_VERSIONS_RETAIN extends LogKey
- case object NUM_WEIGHTED_EXAMPLES extends LogKey
- case object NUM_WORKERS extends LogKey
- case object OBJECT_AGG_SORT_BASED_FALLBACK_THRESHOLD extends LogKey
- case object OBJECT_ID extends LogKey
- case object OFFSET extends LogKey
- case object OFFSETS extends LogKey
- case object OFFSET_SEQUENCE_METADATA extends LogKey
- case object OLD_BLOCK_MANAGER_ID extends LogKey
- case object OLD_GENERATION_GC extends LogKey
- case object OLD_VALUE extends LogKey
- case object OPEN_COST_IN_BYTES extends LogKey
- case object OPERATION_HANDLE extends LogKey
- case object OPERATION_HANDLE_ID extends LogKey
- case object OPERATION_ID extends LogKey
- case object OPTIMIZED_PLAN_COLUMNS extends LogKey
- case object OPTIMIZER_CLASS_NAME extends LogKey
- case object OPTIONS extends LogKey
- case object OP_ID extends LogKey
- case object OP_TYPE extends LogKey
- case object ORIGINAL_DISK_SIZE extends LogKey
- case object ORIGINAL_MEMORY_SIZE extends LogKey
- case object OS_ARCH extends LogKey
- case object OS_NAME extends LogKey
- case object OS_VERSION extends LogKey
- case object OUTPUT extends LogKey
- case object OUTPUT_BUFFER extends LogKey
- case object OVERHEAD_MEMORY_SIZE extends LogKey
- case object PAGE_SIZE extends LogKey
- case object PARENT_STAGES extends LogKey
- case object PARSE_MODE extends LogKey
- case object PARTITIONED_FILE_READER extends LogKey
- case object PARTITIONER extends LogKey
- case object PARTITION_ID extends LogKey
- case object PARTITION_IDS extends LogKey
- case object PARTITION_SIZE extends LogKey
- case object PARTITION_SPECIFICATION extends LogKey
- case object PARTITION_SPECS extends LogKey
- case object PATH extends LogKey
- case object PATHS extends LogKey
- case object PEER extends LogKey
- case object PENDING_TIMES extends LogKey
- case object PERCENT extends LogKey
- case object PIPELINE_STAGE_UID extends LogKey
- case object PLUGIN_NAME extends LogKey
- case object POD_ID extends LogKey
- case object POD_NAME extends LogKey
- case object POD_NAMESPACE extends LogKey
- case object POD_PHASE extends LogKey
- case object POD_STATE extends LogKey
- case object POINT_OF_CENTER extends LogKey
- case object POLICY extends LogKey
- case object POOL_NAME extends LogKey
- case object PORT extends LogKey
- case object PORT2 extends LogKey
- case object POST_SCAN_FILTERS extends LogKey
- case object PREDICATE extends LogKey
- case object PREDICATES extends LogKey
- case object PREFERRED_SERVICE_NAME extends LogKey
- case object PREFIX extends LogKey
- case object PRETTY_ID_STRING extends LogKey
- case object PRINCIPAL extends LogKey
- case object PROCESS extends LogKey
- case object PROCESSING_TIME extends LogKey
- case object PRODUCER_ID extends LogKey
- case object PROPERTY_NAME extends LogKey
- case object PROPORTIONAL extends LogKey
- case object PROTOCOL_VERSION extends LogKey
- case object PROVIDER extends LogKey
- case object PUSHED_FILTERS extends LogKey
- case object PUSH_MERGED_LOCAL_BLOCKS_SIZE extends LogKey
- case object PVC_METADATA_NAME extends LogKey
- case object PYTHON_EXEC extends LogKey
- case object PYTHON_PACKAGES extends LogKey
- case object PYTHON_VERSION extends LogKey
- case object PYTHON_WORKER_CHANNEL_IS_BLOCKING_MODE extends LogKey
- case object PYTHON_WORKER_CHANNEL_IS_CONNECTED extends LogKey
- case object PYTHON_WORKER_HAS_INPUTS extends LogKey
- case object PYTHON_WORKER_IDLE_TIMEOUT extends LogKey
- case object PYTHON_WORKER_IS_ALIVE extends LogKey
- case object PYTHON_WORKER_MODULE extends LogKey
- case object PYTHON_WORKER_RESPONSE extends LogKey
- case object PYTHON_WORKER_SELECTION_KEY_INTERESTS extends LogKey
- case object PYTHON_WORKER_SELECTION_KEY_IS_VALID extends LogKey
- case object PYTHON_WORKER_SELECTOR_IS_OPEN extends LogKey
- case object QUANTILES extends LogKey
- case object QUERY_CACHE_VALUE extends LogKey
- case object QUERY_HINT extends LogKey
- case object QUERY_ID extends LogKey
- case object QUERY_PLAN extends LogKey
- case object QUERY_PLAN_COMPARISON extends LogKey
- case object QUERY_PLAN_LENGTH_ACTUAL extends LogKey
- case object QUERY_PLAN_LENGTH_MAX extends LogKey
- case object QUERY_RUN_ID extends LogKey
- case object RANGE extends LogKey
- case object RATE_LIMIT extends LogKey
- case object RATIO extends LogKey
- case object RDD extends LogKey
- case object RDD_CHECKPOINT_DIR extends LogKey
- case object RDD_DEBUG_STRING extends LogKey
- case object RDD_DESCRIPTION extends LogKey
- case object RDD_ID extends LogKey
- case object READ_LIMIT extends LogKey
- case object REASON extends LogKey
- case object REATTACHABLE extends LogKey
- case object RECEIVED_BLOCK_INFO extends LogKey
- case object RECEIVED_BLOCK_TRACKER_LOG_EVENT extends LogKey
- case object RECEIVER_ID extends LogKey
- case object RECEIVER_IDS extends LogKey
- case object RECORDS extends LogKey
- case object RECOVERY_STATE extends LogKey
- case object RECURSIVE_DEPTH extends LogKey
- case object REDACTED_STATEMENT extends LogKey
- case object REDUCE_ID extends LogKey
- case object REGEX extends LogKey
- case object REGISTERED_EXECUTOR_FILE extends LogKey
- case object REGISTER_MERGE_RESULTS extends LogKey
- case object RELATION_NAME extends LogKey
- case object RELATION_OUTPUT extends LogKey
- case object RELATIVE_TOLERANCE extends LogKey
- case object RELEASED_LOCKS extends LogKey
- case object REMAINING_PARTITIONS extends LogKey
- case object REMOTE_ADDRESS extends LogKey
- case object REMOTE_BLOCKS_SIZE extends LogKey
- case object REMOVE_FROM_MASTER extends LogKey
- case object REPORT_DETAILS extends LogKey
- case object REQUESTER_SIZE extends LogKey
- case object REQUEST_EXECUTORS extends LogKey
- case object REQUEST_ID extends LogKey
- case object RESOURCE extends LogKey
- case object RESOURCE_NAME extends LogKey
- case object RESOURCE_PROFILE_ID extends LogKey
- case object RESOURCE_PROFILE_IDS extends LogKey
- case object RESOURCE_PROFILE_TO_TOTAL_EXECS extends LogKey
- case object RESPONSE_BODY_SIZE extends LogKey
- case object RESTART_TIME extends LogKey
- case object RESULT extends LogKey
- case object RESULT_SIZE_BYTES extends LogKey
- case object RESULT_SIZE_BYTES_MAX extends LogKey
- case object RETRY_INTERVAL extends LogKey
- case object RETRY_WAIT_TIME extends LogKey
- case object RIGHT_EXPR extends LogKey
- case object RIGHT_LOGICAL_PLAN_STATS_SIZE_IN_BYTES extends LogKey
- case object RMSE extends LogKey
- case object ROCKS_DB_FILE_MAPPING extends LogKey
- case object ROCKS_DB_LOG_LEVEL extends LogKey
- case object ROCKS_DB_LOG_MESSAGE extends LogKey
- case object RPC_ADDRESS extends LogKey
- case object RPC_ENDPOINT_REF extends LogKey
- case object RPC_MESSAGE_CAPACITY extends LogKey
- case object RPC_SSL_ENABLED extends LogKey
- case object RULE_EXECUTOR_NAME extends LogKey
- case object RULE_NAME extends LogKey
- case object RUN_ID extends LogKey
- case object RUN_ID_STRING extends LogKey
- case object SCALA_VERSION extends LogKey
- case object SCALING_DOWN_RATIO extends LogKey
- case object SCALING_UP_RATIO extends LogKey
- case object SCHEDULER_POOL_NAME extends LogKey
- case object SCHEDULING_MODE extends LogKey
- case object SCHEMA extends LogKey
- case object SCHEMA2 extends LogKey
- case object SERVER_NAME extends LogKey
- case object SERVICE_NAME extends LogKey
- case object SERVLET_CONTEXT_HANDLER_PATH extends LogKey
- case object SESSION_HANDLE extends LogKey
- case object SESSION_HOLD_INFO extends LogKey
- case object SESSION_ID extends LogKey
- case object SESSION_KEY extends LogKey
- case object SET_CLIENT_INFO_REQUEST extends LogKey
- case object SHARD_ID extends LogKey
- case object SHORTER_SERVICE_NAME extends LogKey
- case object SHORT_USER_NAME extends LogKey
- case object SHUFFLE_BLOCK_INFO extends LogKey
- case object SHUFFLE_DB_BACKEND_KEY extends LogKey
- case object SHUFFLE_DB_BACKEND_NAME extends LogKey
- case object SHUFFLE_ID extends LogKey
- case object SHUFFLE_IDS extends LogKey
- case object SHUFFLE_MERGE_ID extends LogKey
- case object SHUFFLE_MERGE_RECOVERY_FILE extends LogKey
- case object SHUFFLE_SERVICE_CONF_OVERLAY_URL extends LogKey
- case object SHUFFLE_SERVICE_METRICS_NAMESPACE extends LogKey
- case object SHUFFLE_SERVICE_NAME extends LogKey
- case object SIGMAS_LENGTH extends LogKey
- case object SIGNAL extends LogKey
- case object SINK extends LogKey
- case object SIZE extends LogKey
- case object SLEEP_TIME extends LogKey
- case object SLIDE_DURATION extends LogKey
- case object SMALLEST_CLUSTER_INDEX extends LogKey
- case object SNAPSHOT_EVENT extends LogKey
- case object SNAPSHOT_EVENT_TIME_DELTA extends LogKey
- case object SNAPSHOT_EVENT_VERSION_DELTA extends LogKey
- case object SNAPSHOT_VERSION extends LogKey
- case object SOCKET_ADDRESS extends LogKey
- case object SOURCE extends LogKey
- case object SOURCE_PATH extends LogKey
- case object SPARK_BRANCH extends LogKey
- case object SPARK_BUILD_DATE extends LogKey
- case object SPARK_BUILD_USER extends LogKey
- case object SPARK_DATA_STREAM extends LogKey
- case object SPARK_PLAN_ID extends LogKey
- case object SPARK_REPO_URL extends LogKey
- case object SPARK_REVISION extends LogKey
- case object SPARK_VERSION extends LogKey
- case object SPILL_TIMES extends LogKey
- case object SQL_TEXT extends LogKey
- case object SRC_PATH extends LogKey
- case object STAGE extends LogKey
- case object STAGES extends LogKey
- case object STAGE_ATTEMPT extends LogKey
- case object STAGE_ATTEMPT_ID extends LogKey
- case object STAGE_ID extends LogKey
- case object STAGE_NAME extends LogKey
- case object START_INDEX extends LogKey
- case object START_TIME extends LogKey
- case object STATEMENT_ID extends LogKey
- case object STATE_NAME extends LogKey
- case object STATE_STORE_COORDINATOR extends LogKey
- case object STATE_STORE_ID extends LogKey
- case object STATE_STORE_PROVIDER extends LogKey
- case object STATE_STORE_PROVIDER_ID extends LogKey
- case object STATE_STORE_PROVIDER_IDS extends LogKey
- case object STATE_STORE_VERSION extends LogKey
- case object STATS extends LogKey
- case object STATUS extends LogKey
- case object STDERR extends LogKey
- case object STOP_SITE_SHORT_FORM extends LogKey
- case object STORAGE_LEVEL extends LogKey
- case object STORAGE_LEVEL_DESERIALIZED extends LogKey
- case object STORAGE_LEVEL_REPLICATION extends LogKey
- case object STORAGE_MEMORY_SIZE extends LogKey
- case object STORE_ID extends LogKey
- case object STRATEGY extends LogKey
- case object STREAMING_CONTEXT extends LogKey
- case object STREAMING_DATA_SOURCE_DESCRIPTION extends LogKey
- case object STREAMING_DATA_SOURCE_NAME extends LogKey
- case object STREAMING_OFFSETS_END extends LogKey
- case object STREAMING_OFFSETS_START extends LogKey
- case object STREAMING_QUERY_PROGRESS extends LogKey
- case object STREAMING_SOURCE extends LogKey
- case object STREAMING_TABLE extends LogKey
- case object STREAMING_WRITE extends LogKey
- case object STREAM_CHUNK_ID extends LogKey
- case object STREAM_ID extends LogKey
- case object STREAM_NAME extends LogKey
- case object SUBMISSION_ID extends LogKey
- case object SUBSAMPLING_RATE extends LogKey
- case object SUB_QUERY extends LogKey
- case object TABLE_NAME extends LogKey
- case object TABLE_TYPE extends LogKey
- case object TABLE_TYPES extends LogKey
- case object TAG extends LogKey
- case object TARGET_NUM_EXECUTOR extends LogKey
- case object TARGET_NUM_EXECUTOR_DELTA extends LogKey
- case object TARGET_PATH extends LogKey
- case object TARGET_SIZE extends LogKey
- case object TASK_ATTEMPT_ID extends LogKey
- case object TASK_ID extends LogKey
- case object TASK_INDEX extends LogKey
- case object TASK_LOCALITY extends LogKey
- case object TASK_NAME extends LogKey
- case object TASK_REQUIREMENTS extends LogKey
- case object TASK_RESOURCES extends LogKey
- case object TASK_RESOURCE_ASSIGNMENTS extends LogKey
- case object TASK_SET_MANAGER extends LogKey
- case object TASK_SET_NAME extends LogKey
- case object TASK_STATE extends LogKey
- case object TEMP_FILE extends LogKey
- case object TEMP_OUTPUT_PATH extends LogKey
- case object TEMP_PATH extends LogKey
- case object TEST_SIZE extends LogKey
- case object THREAD extends LogKey
- case object THREAD_ID extends LogKey
- case object THREAD_NAME extends LogKey
- case object THREAD_POOL_KEEPALIVE_TIME extends LogKey
- case object THREAD_POOL_SIZE extends LogKey
- case object THREAD_POOL_WAIT_QUEUE_SIZE extends LogKey
- case object THRESHOLD extends LogKey
- case object THRESH_TIME extends LogKey
- case object TIME extends LogKey
- case object TIMEOUT extends LogKey
- case object TIMER extends LogKey
- case object TIMESTAMP extends LogKey
- case object TIME_UNITS extends LogKey
- case object TIP extends LogKey
- case object TOKEN extends LogKey
- case object TOKEN_KIND extends LogKey
- case object TOKEN_REGEX extends LogKey
- case object TOKEN_RENEWER extends LogKey
- case object TOPIC extends LogKey
- case object TOPIC_PARTITION extends LogKey
- case object TOPIC_PARTITIONS extends LogKey
- case object TOPIC_PARTITION_OFFSET extends LogKey
- case object TOPIC_PARTITION_OFFSET_RANGE extends LogKey
- case object TOTAL extends LogKey
- case object TOTAL_EFFECTIVE_TIME extends LogKey
- case object TOTAL_SIZE extends LogKey
- case object TOTAL_TIME extends LogKey
- case object TOTAL_TIME_READ extends LogKey
- case object TO_TIME extends LogKey
- case object TRAINING_SIZE extends LogKey
- case object TRAIN_VALIDATION_SPLIT_METRIC extends LogKey
- case object TRAIN_VALIDATION_SPLIT_METRICS extends LogKey
- case object TRANSFER_TYPE extends LogKey
- case object TREE_NODE extends LogKey
- case object TRIGGER_INTERVAL extends LogKey
- case object UI_ACLS extends LogKey
- case object UI_FILTER extends LogKey
- case object UI_FILTER_PARAMS extends LogKey
- case object UI_PROXY_BASE extends LogKey
- case object UNKNOWN_PARAM extends LogKey
- case object UNSUPPORTED_EXPR extends LogKey
- case object UNSUPPORTED_HINT_REASON extends LogKey
- case object UNTIL_OFFSET extends LogKey
- case object UPPER_BOUND extends LogKey
- case object URI extends LogKey
- case object URIS extends LogKey
- case object URL extends LogKey
- case object URL2 extends LogKey
- case object URLS extends LogKey
- case object USER_ID extends LogKey
- case object USER_NAME extends LogKey
- case object UUID extends LogKey
- case object VALUE extends LogKey
- case object VERSIONS_TO_DELETE extends LogKey
- case object VERSION_NUM extends LogKey
- case object VIEW_ACLS extends LogKey
- case object VIEW_ACLS_GROUPS extends LogKey
- case object VIRTUAL_CORES extends LogKey
- case object VOCAB_SIZE extends LogKey
- case object WAIT_RESULT_TIME extends LogKey
- case object WAIT_SEND_TIME extends LogKey
- case object WATERMARK_CONSTRAINT extends LogKey
- case object WEB_URL extends LogKey
- case object WEIGHT extends LogKey
- case object WORKER extends LogKey
- case object WORKER_HOST extends LogKey
- case object WORKER_ID extends LogKey
- case object WORKER_PORT extends LogKey
- case object WORKER_URL extends LogKey
- case object WRITE_AHEAD_LOG_INFO extends LogKey
- case object WRITE_AHEAD_LOG_RECORD_HANDLE extends LogKey
- case object WRITE_JOB_UUID extends LogKey
- case object XML_SCHEDULING_MODE extends LogKey
- case object XSD_PATH extends LogKey
- case object YARN_RESOURCE extends LogKey
- case object YOUNG_GENERATION_GC extends LogKey
- case object ZERO_TIME extends LogKey
-}
diff --git a/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala b/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala
index cc5d0281829d0..810bdabebb38a 100644
--- a/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala
+++ b/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala
@@ -17,6 +17,8 @@
package org.apache.spark.internal
+import java.util.Locale
+
import scala.concurrent.duration._
import scala.jdk.CollectionConverters._
@@ -29,7 +31,6 @@ import org.slf4j.{Logger, LoggerFactory}
import org.slf4j.event.{Level => Slf4jLevel}
import org.apache.spark.internal.Logging.SparkShellLoggingFilter
-import org.apache.spark.internal.LogKeys
import org.apache.spark.util.SparkClassUtils
/**
@@ -61,29 +62,16 @@ import org.apache.spark.util.SparkClassUtils
*
*
* If you want to output logs in `scala code` through the structured log framework,
- * you can define `custom LogKey` and use it in `scala` code as follows:
+ * you can define `custom LogKey` in `java` and use it in `scala` code as follows:
*
*
- * // To add a `custom LogKey`, implement `LogKey`
- * case object CUSTOM_LOG_KEY extends LogKey
- * import org.apache.spark.internal.MDC;
+ * // Add a `CustomLogKeys`, implement `LogKey`
+ * public enum CustomLogKeys implements LogKey {
+ * CUSTOM_LOG_KEY
+ * }
* logInfo(log"${MDC(CUSTOM_LOG_KEY, "key")}")
*/
-/**
- * Mapped Diagnostic Context (MDC) that will be used in log messages.
- * The values of the MDC will be inline in the log message, while the key-value pairs will be
- * part of the ThreadContext.
- */
-case class MDC(key: LogKey, value: Any) {
- require(!value.isInstanceOf[MessageWithContext],
- "the class of value cannot be MessageWithContext")
-}
-
-object MDC {
- def of(key: LogKey, value: Any): MDC = MDC(key, value)
-}
-
/**
* Wrapper class for log messages that include a logging context.
* This is used as the return type of the string interpolator `LogStringContext`.
@@ -155,7 +143,7 @@ trait Logging {
val value = if (mdc.value != null) mdc.value.toString else null
sb.append(value)
if (Logging.isStructuredLoggingEnabled) {
- context.put(mdc.key.name, value)
+ context.put(mdc.key.name.toLowerCase(Locale.ROOT), value)
}
if (processedParts.hasNext) {
@@ -182,6 +170,12 @@ trait Logging {
}
}
+ protected def MDC(key: LogKey, value: Any): MDC = {
+ require(!value.isInstanceOf[MessageWithContext],
+ "the class of value cannot be MessageWithContext")
+ new MDC(key, value)
+ }
+
// Log methods that take only a String
protected def logInfo(msg: => String): Unit = {
if (log.isInfoEnabled) log.info(msg)
@@ -411,7 +405,6 @@ private[spark] object Logging {
@volatile private var initialized = false
@volatile private var defaultRootLevel: Level = null
@volatile private var defaultSparkLog4jConfig = false
- @volatile private var structuredLoggingEnabled = false
@volatile private[spark] var sparkShellThresholdLevel: Level = null
@volatile private[spark] var setLogLevelPrinted: Boolean = false
@@ -484,21 +477,21 @@ private[spark] object Logging {
* Enable Structured logging framework.
*/
private[spark] def enableStructuredLogging(): Unit = {
- structuredLoggingEnabled = true
+ SparkLoggerFactory.enableStructuredLogging()
}
/**
* Disable Structured logging framework.
*/
private[spark] def disableStructuredLogging(): Unit = {
- structuredLoggingEnabled = false
+ SparkLoggerFactory.disableStructuredLogging()
}
/**
* Return true if Structured logging framework is enabled.
*/
private[spark] def isStructuredLoggingEnabled: Boolean = {
- structuredLoggingEnabled
+ SparkLoggerFactory.isStructuredLoggingEnabled
}
private[spark] class SparkShellLoggingFilter extends AbstractFilter {
diff --git a/common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala b/common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
index 5ea3c9afa9c10..db6953031fd2c 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
@@ -24,7 +24,6 @@ import java.lang.reflect.{Field, Modifier}
import scala.collection.mutable.{Map, Queue, Set, Stack}
import scala.jdk.CollectionConverters._
-import org.apache.commons.lang3.ClassUtils
import org.apache.xbean.asm9.{ClassReader, ClassVisitor, Handle, MethodVisitor, Type}
import org.apache.xbean.asm9.Opcodes._
import org.apache.xbean.asm9.tree.{ClassNode, MethodNode}
@@ -619,7 +618,7 @@ private[spark] object IndylambdaScalaClosures extends Logging {
def getSerializationProxy(maybeClosure: AnyRef): Option[SerializedLambda] = {
def isClosureCandidate(cls: Class[_]): Boolean = {
// TODO: maybe lift this restriction to support other functional interfaces in the future
- val implementedInterfaces = ClassUtils.getAllInterfaces(cls).asScala
+ val implementedInterfaces = SparkClassUtils.getAllInterfaces(cls)
implementedInterfaces.exists(_.getName.startsWith("scala.Function"))
}
diff --git a/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala b/common/utils/src/main/scala/org/apache/spark/util/CompletionIterator.scala
similarity index 100%
rename from core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
rename to common/utils/src/main/scala/org/apache/spark/util/CompletionIterator.scala
diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
index d54a2f2ed9cea..5e923ad35a3ab 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
@@ -22,7 +22,6 @@ import java.net.URI
import java.text.ParseException
import java.util.UUID
-import org.apache.commons.lang3.StringUtils
import org.apache.ivy.Ivy
import org.apache.ivy.core.LogOptions
import org.apache.ivy.core.module.descriptor.{Artifact, DefaultDependencyDescriptor, DefaultExcludeRule, DefaultModuleDescriptor, ExcludeRule}
@@ -36,7 +35,7 @@ import org.apache.ivy.plugins.repository.file.FileRepository
import org.apache.ivy.plugins.resolver.{ChainResolver, FileSystemResolver, IBiblioResolver}
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.util.ArrayImplicits._
/** Provides utility functions to be used inside SparkSubmit. */
@@ -567,7 +566,7 @@ private[spark] object MavenUtils extends Logging {
}
private def isInvalidQueryString(tokens: Array[String]): Boolean = {
- tokens.length != 2 || StringUtils.isBlank(tokens(0)) || StringUtils.isBlank(tokens(1))
+ tokens.length != 2 || SparkStringUtils.isBlank(tokens(0)) || SparkStringUtils.isBlank(tokens(1))
}
/**
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkClassUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkClassUtils.scala
index 307006315a3c4..3f22719240c06 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkClassUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkClassUtils.scala
@@ -18,6 +18,7 @@ package org.apache.spark.util
import java.util.Random
+import scala.collection.mutable.LinkedHashSet
import scala.util.Try
private[spark] trait SparkClassUtils {
@@ -136,6 +137,33 @@ private[spark] trait SparkClassUtils {
}
}
}
+
+ /**
+ * Gets a list of all interfaces implemented by the given class and its superclasses.
+ */
+ def getAllInterfaces(cls: Class[_]): List[Class[_]] = {
+ if (cls == null) {
+ return null
+ }
+ val interfacesFound = LinkedHashSet[Class[_]]()
+ getAllInterfacesHelper(cls, interfacesFound)
+ interfacesFound.toList
+ }
+
+ private def getAllInterfacesHelper(
+ clazz: Class[_],
+ interfacesFound: LinkedHashSet[Class[_]]): Unit = {
+ var currentClass = clazz
+ while (currentClass != null) {
+ val interfaces = currentClass.getInterfaces
+ for (i <- interfaces) {
+ if (interfacesFound.add(i)) {
+ getAllInterfacesHelper(i, interfacesFound)
+ }
+ }
+ currentClass = currentClass.getSuperclass
+ }
+ }
}
private[spark] object SparkClassUtils extends SparkClassUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkCollectionUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkCollectionUtils.scala
index eaab50117abdc..9c255daa0522d 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkCollectionUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkCollectionUtils.scala
@@ -16,7 +16,10 @@
*/
package org.apache.spark.util
+import java.util.Arrays
+
import scala.collection.immutable
+import scala.reflect.ClassTag
private[spark] trait SparkCollectionUtils {
/**
@@ -32,6 +35,36 @@ private[spark] trait SparkCollectionUtils {
}
builder.result()
}
+
+ def isEmpty[K, V](map: java.util.Map[K, V]): Boolean = {
+ map == null || map.isEmpty()
+ }
+
+ def isNotEmpty[K, V](map: java.util.Map[K, V]): Boolean = !isEmpty(map)
+
+ def createArray[K: ClassTag](size: Int, defaultValue: K): Array[K] = {
+ val arr = Array.ofDim[K](size)
+ val classTag = implicitly[ClassTag[K]]
+ classTag.runtimeClass match {
+ case c if c == classOf[Boolean] =>
+ Arrays.fill(arr.asInstanceOf[Array[Boolean]], defaultValue.asInstanceOf[Boolean])
+ case c if c == classOf[Byte] =>
+ Arrays.fill(arr.asInstanceOf[Array[Byte]], defaultValue.asInstanceOf[Byte])
+ case c if c == classOf[Short] =>
+ Arrays.fill(arr.asInstanceOf[Array[Short]], defaultValue.asInstanceOf[Short])
+ case c if c == classOf[Int] =>
+ Arrays.fill(arr.asInstanceOf[Array[Int]], defaultValue.asInstanceOf[Int])
+ case c if c == classOf[Long] =>
+ Arrays.fill(arr.asInstanceOf[Array[Long]], defaultValue.asInstanceOf[Long])
+ case c if c == classOf[Float] =>
+ Arrays.fill(arr.asInstanceOf[Array[Float]], defaultValue.asInstanceOf[Float])
+ case c if c == classOf[Double] =>
+ Arrays.fill(arr.asInstanceOf[Array[Double]], defaultValue.asInstanceOf[Double])
+ case _ =>
+ Arrays.fill(arr.asInstanceOf[Array[AnyRef]], defaultValue.asInstanceOf[AnyRef])
+ }
+ arr
+ }
}
private[spark] object SparkCollectionUtils extends SparkCollectionUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkEnvUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkEnvUtils.scala
index b54e6ee5d7309..01e3f52de41f3 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkEnvUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkEnvUtils.scala
@@ -16,16 +16,14 @@
*/
package org.apache.spark.util
+import org.apache.spark.network.util.JavaUtils
+
private[spark] trait SparkEnvUtils {
/**
* Indicates whether Spark is currently running unit tests.
*/
- def isTesting: Boolean = {
- // Scala's `sys.env` creates a ton of garbage by constructing Scala immutable maps, so
- // we directly use the Java APIs instead.
- System.getenv("SPARK_TESTING") != null || System.getProperty("spark.testing") != null
- }
+ def isTesting: Boolean = JavaUtils.isTesting
}
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala
index 872c89e5a29a2..2f16c90ad7149 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala
@@ -16,12 +16,14 @@
*/
package org.apache.spark.util
-import java.io.{Closeable, IOException, PrintWriter}
-import java.nio.charset.StandardCharsets.UTF_8
+import java.io.{Closeable, IOException}
+import scala.annotation.tailrec
+import scala.collection.mutable
import scala.util.control.NonFatal
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
+import org.apache.spark.network.util.JavaUtils
private[spark] trait SparkErrorUtils extends Logging {
/**
@@ -45,7 +47,11 @@ private[spark] trait SparkErrorUtils extends Logging {
def tryWithResource[R <: Closeable, T](createResource: => R)(f: R => T): T = {
val resource = createResource
- try f.apply(resource) finally resource.close()
+ try {
+ f.apply(resource)
+ } finally {
+ closeQuietly(resource)
+ }
}
/**
@@ -59,7 +65,7 @@ private[spark] trait SparkErrorUtils extends Logging {
initialize(resource)
} catch {
case e: Throwable =>
- resource.close()
+ closeQuietly(resource)
throw e
}
}
@@ -97,13 +103,54 @@ private[spark] trait SparkErrorUtils extends Logging {
}
}
- def stackTraceToString(t: Throwable): String = {
- val out = new java.io.ByteArrayOutputStream
- SparkErrorUtils.tryWithResource(new PrintWriter(out)) { writer =>
- t.printStackTrace(writer)
- writer.flush()
+ def stackTraceToString(t: Throwable): String = JavaUtils.stackTraceToString(t)
+
+ /**
+ * Walks the [[Throwable]] to obtain its root cause.
+ *
+ * This method walks through the exception chain until the last element,
+ * the root cause of the chain, using `getCause()`, and
+ * returns that exception.
+ *
+ * This method handles recursive cause chains that might
+ * otherwise cause infinite loops. The cause chain is processed until
+ * the end, or until the next item in the chain is already
+ * processed. If we detect a loop, then return the element before the loop.
+ *
+ * @param throwable the throwable to get the root cause for, may be null
+ * @return the root cause of the [[Throwable]], `null` if null throwable input
+ */
+ def getRootCause(throwable: Throwable): Throwable = {
+ @tailrec
+ def findRoot(
+ current: Throwable,
+ visited: mutable.Set[Throwable] = mutable.Set.empty): Throwable = {
+ if (current == null) null
+ else {
+ visited += current
+ val cause = current.getCause
+ if (cause == null) {
+ current
+ } else if (visited.contains(cause)) {
+ current
+ } else {
+ findRoot(cause, visited)
+ }
+ }
+ }
+
+ findRoot(throwable)
+ }
+
+ /** Try to close by ignoring all exceptions. This is different from JavaUtils.closeQuietly. */
+ def closeQuietly(closeable: Closeable): Unit = {
+ if (closeable != null) {
+ try {
+ closeable.close()
+ } catch {
+ case _: Exception =>
+ }
}
- new String(out.toByteArray, UTF_8)
}
}
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala
index 22f03df1b2697..3f1f9c1f9df7f 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala
@@ -17,10 +17,11 @@
package org.apache.spark.util
import java.io.File
-import java.net.{URI, URISyntaxException}
-import java.nio.file.Files
+import java.net.{URI, URISyntaxException, URL}
+import java.nio.file.{Files, Path, StandardCopyOption}
+import java.nio.file.attribute.FileTime
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.network.util.JavaUtils
private[spark] trait SparkFileUtils extends Logging {
@@ -49,6 +50,13 @@ private[spark] trait SparkFileUtils extends Logging {
new File(path).getCanonicalFile().toURI()
}
+ /**
+ * Size of files recursively.
+ */
+ def sizeOf(f: File): Long = {
+ JavaUtils.sizeOf(f)
+ }
+
/**
* Lists files recursively.
*/
@@ -65,6 +73,20 @@ private[spark] trait SparkFileUtils extends Logging {
result.toArray
}
+ /**
+ * Lists regular files recursively.
+ */
+ def listFiles(f: File): java.util.Set[File] = {
+ JavaUtils.listFiles(f)
+ }
+
+ /**
+ * Lists regular paths recursively.
+ */
+ def listPaths(f: File): java.util.Set[Path] = {
+ JavaUtils.listPaths(f)
+ }
+
/**
* Create a directory given the abstract pathname
* @return true, if the directory is successfully created; otherwise, return false.
@@ -112,6 +134,21 @@ private[spark] trait SparkFileUtils extends Logging {
createDirectory(root, namePrefix)
}
+ def createParentDirs(file: File): Unit = {
+ if (file == null) {
+ throw new IllegalArgumentException("Input should not be null.")
+ }
+ val parent = file.getParentFile()
+ if (parent != null) {
+ Files.createDirectories(parent.toPath())
+ }
+ }
+
+ /** Delete recursively while keeping the given directory itself. */
+ def cleanDirectory(dir: File): Unit = {
+ JavaUtils.cleanDirectory(dir)
+ }
+
/**
* Delete a file or directory and its contents recursively.
* Don't follow directories if they are symlinks.
@@ -120,6 +157,87 @@ private[spark] trait SparkFileUtils extends Logging {
def deleteRecursively(file: File): Unit = {
JavaUtils.deleteRecursively(file)
}
+
+ /** Delete a file or directory and its contents recursively without throwing exceptions. */
+ def deleteQuietly(file: File): Unit = {
+ JavaUtils.deleteQuietly(file)
+ }
+
+ /** Registers the file or directory for deletion when the JVM exists. */
+ def forceDeleteOnExit(file: File): Unit = {
+ JavaUtils.forceDeleteOnExit(file)
+ }
+
+ def getFile(names: String*): File = {
+ require(names != null && names.forall(_ != null))
+ names.tail.foldLeft(Path.of(names.head)) { (path, part) =>
+ path.resolve(part)
+ }.toFile
+ }
+
+ def getFile(parent: File, names: String*): File = {
+ require(parent != null && names != null && names.forall(_ != null))
+ names.foldLeft(parent.toPath) { (path, part) =>
+ path.resolve(part)
+ }.toFile
+ }
+
+ /** Move src to dst simply. File attribute times are not copied. */
+ def moveFile(src: File, dst: File): Unit = {
+ JavaUtils.moveFile(src, dst)
+ }
+
+ /** Move src to dst simply. File attribute times are not copied. */
+ def moveDirectory(src: File, dst: File): Unit = {
+ JavaUtils.moveDirectory(src, dst)
+ }
+
+ /** Copy src to the target directory simply. File attribute times are not copied. */
+ def copyDirectory(src: File, dir: File): Unit = {
+ JavaUtils.copyDirectory(src, dir)
+ }
+
+ /** Copy file to the target directory simply. File attribute times are not copied. */
+ def copyFileToDirectory(file: File, dir: File): Unit = {
+ if (file == null || dir == null || !file.exists() || (dir.exists() && !dir.isDirectory())) {
+ throw new IllegalArgumentException(s"Invalid input file $file or directory $dir")
+ }
+ Files.createDirectories(dir.toPath())
+ val newFile = new File(dir, file.getName())
+ Files.copy(file.toPath(), newFile.toPath(), StandardCopyOption.REPLACE_EXISTING)
+ }
+
+ def copyFile(src: File, dst: File): Unit = {
+ if (src == null || dst == null || !src.exists() || (dst.exists() && dst.isDirectory())) {
+ throw new IllegalArgumentException(s"Invalid input file $src or directory $dst")
+ }
+ Files.copy(src.toPath(), dst.toPath(), StandardCopyOption.REPLACE_EXISTING)
+ }
+
+ def copyURLToFile(url: URL, file: File): Unit = {
+ JavaUtils.copyURLToFile(url, file)
+ }
+
+ /** Return true if the content of the files are equal or they both don't exist */
+ def contentEquals(file1: File, file2: File): Boolean = {
+ JavaUtils.contentEquals(file1, file2)
+ }
+
+ def touch(file: File): Unit = {
+ if (file == null) {
+ throw new IllegalArgumentException("Invalid input file: null")
+ }
+ val path = file.toPath
+ if (Files.exists(path)) {
+ Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis()))
+ } else {
+ val parent = path.getParent()
+ if (parent != null && !Files.exists(parent)) {
+ Files.createDirectories(parent)
+ }
+ Files.createFile(path)
+ }
+ }
}
private[spark] object SparkFileUtils extends SparkFileUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkStreamUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkStreamUtils.scala
index b9148901f1ac6..a2556770ed357 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkStreamUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkStreamUtils.scala
@@ -19,6 +19,7 @@ package org.apache.spark.util
import java.io.{FileInputStream, FileOutputStream, InputStream, OutputStream}
import java.nio.channels.{FileChannel, WritableByteChannel}
+import org.apache.spark.network.util.JavaUtils
import org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally
private[spark] trait SparkStreamUtils {
@@ -104,6 +105,14 @@ private[spark] trait SparkStreamUtils {
""".stripMargin)
}
}
+
+ def toString(in: InputStream): String = {
+ JavaUtils.toString(in)
+ }
+
+ def readFully(in: InputStream, arr: Array[Byte], off: Int, len: Int): Unit = {
+ JavaUtils.readFully(in, arr, off, len)
+ }
}
private [spark] object SparkStreamUtils extends SparkStreamUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkStringUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkStringUtils.scala
index 9f58288ee266c..53f46a5d9c8ed 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkStringUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkStringUtils.scala
@@ -16,11 +16,131 @@
*/
package org.apache.spark.util
+import java.util.HexFormat
+import java.util.concurrent.atomic.AtomicBoolean
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.util.ArrayImplicits._
+
private[spark] trait SparkStringUtils {
+ private final lazy val SPACE_DELIMITED_UPPERCASE_HEX =
+ HexFormat.of().withDelimiter(" ").withUpperCase()
+
+ /**
+ * Returns a pretty string of the byte array which prints each byte as a hex digit and add
+ * spaces between them. For example, [1A C0].
+ */
+ def getHexString(bytes: Array[Byte]): String = {
+ s"[${SPACE_DELIMITED_UPPERCASE_HEX.formatHex(bytes)}]"
+ }
+
+ def fromHexString(hex: String): Array[Byte] = {
+ SPACE_DELIMITED_UPPERCASE_HEX.parseHex(hex.stripPrefix("[").stripSuffix("]"))
+ }
+
+ def isEmpty(str: String): Boolean = str == null || str.length() == 0
+
+ def isNotEmpty(str: String): Boolean = !isEmpty(str)
+
+ def isBlank(str: String): Boolean = str == null || str.isBlank
+
+ def isNotBlank(str: String): Boolean = !isBlank(str)
+
+ def abbreviate(str: String, abbrevMarker: String, len: Int): String = {
+ if (str == null || abbrevMarker == null) {
+ null
+ } else if (str.length() <= len || str.length() <= abbrevMarker.length()) {
+ str
+ } else {
+ str.substring(0, len - abbrevMarker.length()) + abbrevMarker
+ }
+ }
+
+ def abbreviate(str: String, len: Int): String = abbreviate(str, "...", len)
+
+ def sideBySide(left: String, right: String): Seq[String] = {
+ sideBySide(left.split("\n").toImmutableArraySeq, right.split("\n").toImmutableArraySeq)
+ }
+
+ def sideBySide(left: Seq[String], right: Seq[String]): Seq[String] = {
+ val maxLeftSize = left.map(_.length).max
+ val leftPadded = left ++ Seq.fill(math.max(right.size - left.size, 0))("")
+ val rightPadded = right ++ Seq.fill(math.max(left.size - right.size, 0))("")
+
+ leftPadded.zip(rightPadded).map { case (l, r) =>
+ (if (l == r) " " else "!") + l + " ".repeat((maxLeftSize - l.length) + 3) + r
+ }
+ }
+
def stringToSeq(str: String): Seq[String] = {
import org.apache.spark.util.ArrayImplicits._
str.split(",").map(_.trim()).filter(_.nonEmpty).toImmutableArraySeq
}
+
+ /** Try to strip prefix and suffix with the given string 's' */
+ def strip(str: String, s: String): String =
+ if (str == null || s == null) str else str.stripPrefix(s).stripSuffix(s)
+
+ def leftPad(str: String, width: Int): String =
+ if (str == null || str.length >= width) str else String.format(s"%${width}s", str)
+
+ def rightPad(str: String, width: Int): String =
+ if (str == null || str.length >= width) str else String.format(s"%-${width}s", str)
+
+ def rightPad(str: String, width: Int, s: String): String =
+ if (str == null || str.length >= width) {
+ str
+ } else {
+ val tmp = str + s.repeat((width - str.length)/s.length)
+ tmp + s.substring(0, width - tmp.length)
+ }
}
-private[spark] object SparkStringUtils extends SparkStringUtils
+private[spark] object SparkStringUtils extends SparkStringUtils with Logging {
+
+ /** Whether we have warned about plan string truncation yet. */
+ private val truncationWarningPrinted = new AtomicBoolean(false)
+
+ /**
+ * Format a sequence with semantics similar to calling .mkString(). Any elements beyond
+ * `maxFields` will be dropped and replaced by a "... N more fields" placeholder.
+ *
+ * @return
+ * the trimmed and formatted string.
+ */
+ def truncatedString[T](
+ seq: Seq[T],
+ start: String,
+ sep: String,
+ end: String,
+ maxFields: Int,
+ customToString: Option[T => String] = None): String = {
+ if (seq.length > maxFields) {
+ if (truncationWarningPrinted.compareAndSet(false, true)) {
+ logWarning(
+ "Truncated the string representation of a plan since it was too large. This " +
+ s"behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.")
+ }
+ val numFields = math.max(0, maxFields)
+ val restNum = seq.length - numFields
+ val ending = (if (numFields == 0) "" else sep) +
+ (if (restNum == 0) "" else s"... $restNum more fields") + end
+ if (customToString.isDefined) {
+ seq.take(numFields).map(customToString.get).mkString(start, sep, ending)
+ } else {
+ seq.take(numFields).mkString(start, sep, ending)
+ }
+ } else {
+ if (customToString.isDefined) {
+ seq.map(customToString.get).mkString(start, sep, end)
+ } else {
+ seq.mkString(start, sep, end)
+ }
+ }
+ }
+
+ /** Shorthand for calling truncatedString() without start or end strings. */
+ def truncatedString[T](seq: Seq[T], sep: String, maxFields: Int): String = {
+ truncatedString(seq, "", sep, "", maxFields)
+ }
+}
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkSystemUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkSystemUtils.scala
new file mode 100644
index 0000000000000..eccf556c424eb
--- /dev/null
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkSystemUtils.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.util
+
+import org.apache.spark.network.util.JavaUtils
+
+private[spark] trait SparkSystemUtils {
+ /**
+ * The `os.name` system property.
+ */
+ val osName = JavaUtils.osName
+
+ /**
+ * The `os.version` system property.
+ */
+ val osVersion = JavaUtils.osVersion
+
+ /**
+ * The `os.arch` system property.
+ */
+ val osArch = JavaUtils.osArch
+
+ /**
+ * The `java.version` system property.
+ */
+ val javaVersion = JavaUtils.javaVersion
+
+ /**
+ * Whether the underlying operating system is Windows.
+ */
+ val isWindows = JavaUtils.isWindows
+
+ /**
+ * Whether the underlying operating system is Mac OS X.
+ */
+ val isMac = JavaUtils.isMac
+
+ /**
+ * Whether the underlying operating system is Mac OS X and processor is Apple Silicon.
+ */
+ val isMacOnAppleSilicon = JavaUtils.isMacOnAppleSilicon
+
+ /**
+ * Whether the underlying operating system is Linux.
+ */
+ val isLinux = JavaUtils.isLinux
+
+ /**
+ * Whether the underlying operating system is UNIX.
+ */
+ val isUnix = JavaUtils.isUnix
+}
+
+object SparkSystemUtils extends SparkSystemUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
index 34b40a5a8890c..83028ad06a8f3 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
@@ -66,8 +66,6 @@ private[spark] trait SparkTestUtils {
assert(result.exists(), "Compiled file not found: " + result.getAbsolutePath())
val out = new File(destDir, fileName)
- // renameTo cannot handle in and out files in different filesystems
- // use google's Files.move instead
Files.move(result.toPath, out.toPath)
assert(out.exists(), "Destination file not moved: " + out.getAbsolutePath())
diff --git a/core/src/main/scala/org/apache/spark/util/VersionUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/VersionUtils.scala
similarity index 100%
rename from core/src/main/scala/org/apache/spark/util/VersionUtils.scala
rename to common/utils/src/main/scala/org/apache/spark/util/VersionUtils.scala
diff --git a/common/utils/src/test/scala/org/apache/spark/util/IvyTestUtils.scala b/common/utils/src/test/scala/org/apache/spark/util/IvyTestUtils.scala
index 140de836622f4..e240603ee2527 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/IvyTestUtils.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/IvyTestUtils.scala
@@ -23,10 +23,10 @@ import java.util.jar.Attributes.Name
import scala.collection.mutable.ArrayBuffer
-import org.apache.commons.io.FileUtils
import org.apache.ivy.core.settings.IvySettings
import org.apache.spark.util.MavenUtils.MavenCoordinate
+import org.apache.spark.util.SparkFileUtils.deleteRecursively
private[spark] object IvyTestUtils {
@@ -95,7 +95,7 @@ private[spark] object IvyTestUtils {
className: String,
packageName: String): Seq[(String, File)] = {
val rFilesDir = new File(dir, "R" + File.separator + "pkg")
- new File(rFilesDir, "R").mkdirs()
+ SparkFileUtils.createDirectory(new File(rFilesDir, "R"))
val contents =
s"""myfunc <- function(x) {
| SparkR:::callJStatic("$packageName.$className", "myFunc", x)
@@ -150,20 +150,20 @@ private[spark] object IvyTestUtils {
useIvyLayout: Boolean): File = {
if (useIvyLayout) {
val ivyXmlPath = pathFromCoordinate(artifact, tempPath, "ivy", true)
- ivyXmlPath.mkdirs()
+ SparkFileUtils.createDirectory(ivyXmlPath)
createIvyDescriptor(ivyXmlPath, artifact, dependencies)
} else {
val pomPath = pathFromCoordinate(artifact, tempPath, "pom", useIvyLayout)
- pomPath.mkdirs()
+ SparkFileUtils.createDirectory(pomPath)
createPom(pomPath, artifact, dependencies)
}
}
/** Helper method to write artifact information in the pom. */
private def pomArtifactWriter(artifact: MavenCoordinate, tabCount: Int = 1): String = {
- var result = "\n" + " " * tabCount + s"${artifact.groupId}"
- result += "\n" + " " * tabCount + s"${artifact.artifactId}"
- result += "\n" + " " * tabCount + s"${artifact.version}"
+ var result = "\n" + " ".repeat(tabCount) + s"${artifact.groupId}"
+ result += "\n" + " ".repeat(tabCount) + s"${artifact.artifactId}"
+ result += "\n" + " ".repeat(tabCount) + s"${artifact.version}"
result
}
@@ -293,13 +293,13 @@ private[spark] object IvyTestUtils {
// Where the root of the repository exists, and what Ivy will search in
val tempPath = tempDir.getOrElse(SparkFileUtils.createTempDir())
// Create directory if it doesn't exist
- tempPath.mkdirs()
+ SparkFileUtils.createDirectory(tempPath)
// Where to create temporary class files and such
val root = new File(tempPath, tempPath.hashCode().toString)
- root.mkdirs()
+ SparkFileUtils.createDirectory(root)
try {
val jarPath = pathFromCoordinate(artifact, tempPath, "jar", useIvyLayout)
- jarPath.mkdirs()
+ SparkFileUtils.createDirectory(jarPath)
val className = "MyLib"
val javaClass = createJavaClass(root, className, artifact.groupId)
@@ -319,7 +319,7 @@ private[spark] object IvyTestUtils {
val descriptor = createDescriptor(tempPath, artifact, dependencies, useIvyLayout)
assert(descriptor.exists(), "Problem creating Pom file")
} finally {
- FileUtils.deleteDirectory(root)
+ deleteRecursively(root)
}
tempPath
}
@@ -377,13 +377,13 @@ private[spark] object IvyTestUtils {
if (repo.toString.contains(".m2") || repo.toString.contains(".ivy2") ||
repo.toString.contains(".ivy2.5.2")) {
val groupDir = getBaseGroupDirectory(artifact, useIvyLayout)
- FileUtils.deleteDirectory(new File(repo, groupDir + File.separator + artifact.artifactId))
+ deleteRecursively(new File(repo, groupDir + File.separator + artifact.artifactId))
deps.foreach { _.foreach { dep =>
- FileUtils.deleteDirectory(new File(repo, getBaseGroupDirectory(dep, useIvyLayout)))
+ deleteRecursively(new File(repo, getBaseGroupDirectory(dep, useIvyLayout)))
}
}
} else {
- FileUtils.deleteDirectory(repo)
+ deleteRecursively(repo)
}
purgeLocalIvyCache(artifact, deps, ivySettings)
}
@@ -395,9 +395,9 @@ private[spark] object IvyTestUtils {
dependencies: Option[Seq[MavenCoordinate]],
ivySettings: IvySettings): Unit = {
// delete the artifact from the cache as well if it already exists
- FileUtils.deleteDirectory(new File(ivySettings.getDefaultCache, artifact.groupId))
+ deleteRecursively(new File(ivySettings.getDefaultCache, artifact.groupId))
dependencies.foreach { _.foreach { dep =>
- FileUtils.deleteDirectory(new File(ivySettings.getDefaultCache, dep.groupId))
+ deleteRecursively(new File(ivySettings.getDefaultCache, dep.groupId))
}
}
}
diff --git a/common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala b/common/utils/src/test/scala/org/apache/spark/util/LogKeysSuite.scala
similarity index 65%
rename from common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala
rename to common/utils/src/test/scala/org/apache/spark/util/LogKeysSuite.scala
index 17e360f510a24..25a908a8b96e2 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/LogKeysSuite.scala
@@ -19,25 +19,23 @@ package org.apache.spark.util
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path}
-import java.util.{ArrayList => JList}
import scala.jdk.CollectionConverters._
-import scala.reflect.runtime.universe._
-import org.apache.commons.io.FileUtils
import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
import org.apache.spark.internal.{Logging, LogKeys}
+import org.apache.spark.util.ArrayImplicits._
// scalastyle:off line.size.limit
/**
- * To re-generate the file `LogKey.scala`, run:
+ * To re-generate the file `LogKeys.java`, run:
* {{{
- * SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "common-utils/testOnly org.apache.spark.util.LogKeySuite"
+ * SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "common-utils/testOnly org.apache.spark.util.LogKeysSuite"
* }}}
*/
// scalastyle:on line.size.limit
-class LogKeySuite
+class LogKeysSuite
extends AnyFunSuite // scalastyle:ignore funsuite
with Logging {
@@ -54,37 +52,33 @@ class LogKeySuite
private val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
- private val logKeyFilePath = getWorkspaceFilePath("common", "utils", "src", "main", "scala",
- "org", "apache", "spark", "internal", "LogKey.scala")
+ private val logKeyFilePath = getWorkspaceFilePath("common", "utils", "src", "main", "java",
+ "org", "apache", "spark", "internal", "LogKeys.java")
- // regenerate the file `LogKey.scala` with its members sorted alphabetically
+ // regenerate the file `LogKeys.java` with its members sorted alphabetically
private def regenerateLogKeyFile(
originalKeys: Seq[String], sortedKeys: Seq[String]): Unit = {
if (originalKeys != sortedKeys) {
val logKeyFile = logKeyFilePath.toFile
logInfo(s"Regenerating the file $logKeyFile")
- val originalContents = FileUtils.readLines(logKeyFile, StandardCharsets.UTF_8)
- val sortedContents = new JList[String]()
- var firstMatch = false
- originalContents.asScala.foreach { line =>
- if (line.trim.startsWith("case object ") && line.trim.endsWith(" extends LogKey")) {
- if (!firstMatch) {
- sortedKeys.foreach { key =>
- sortedContents.add(s" case object $key extends LogKey")
- }
- firstMatch = true
- }
- } else {
- sortedContents.add(line)
- }
- }
+ val originalContents = Files.readAllLines(logKeyFile.toPath, StandardCharsets.UTF_8).asScala
+
+ val beforeFirstIndex = originalContents
+ .indexWhere(_.contains("public enum LogKeys implements LogKey"))
+
+ val content =
+ s"""${originalContents.take(beforeFirstIndex + 1).mkString("\n")}
+ |${sortedKeys.map { key => s" $key" }.mkString(",\n")}
+ |}
+ |""".stripMargin
+
Files.delete(logKeyFile.toPath)
- FileUtils.writeLines(logKeyFile, StandardCharsets.UTF_8.name(), sortedContents)
+ Files.writeString(logKeyFile.toPath, content, StandardCharsets.UTF_8)
}
}
test("The members of LogKeys are correctly sorted") {
- val originalKeys = getAllLogKeys.reverse
+ val originalKeys = LogKeys.values.map(_.name).toImmutableArraySeq
val sortedKeys = originalKeys.sorted
if (regenerateGoldenFiles) {
regenerateLogKeyFile(originalKeys, sortedKeys)
@@ -93,11 +87,4 @@ class LogKeySuite
"The members of LogKeys must be sorted alphabetically")
}
}
-
- private def getAllLogKeys: Seq[String] = {
- val logKeysType = typeOf[LogKeys.type]
- val classSymbol = logKeysType.typeSymbol.asClass
- val members = classSymbol.typeSignature.members
- members.filter(m => m.isTerm && !m.isMethod).map(_.name.toString).toSeq
- }
}
diff --git a/common/utils/src/test/scala/org/apache/spark/util/MDCSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/MDCSuite.scala
index 9615eb2263636..f086b6d18dafc 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/MDCSuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/MDCSuite.scala
@@ -22,7 +22,7 @@ import scala.jdk.CollectionConverters._
import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{EXIT_CODE, OFFSET, RANGE}
class MDCSuite
diff --git a/common/utils/src/test/scala/org/apache/spark/util/SparkErrorUtilsSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/SparkErrorUtilsSuite.scala
new file mode 100644
index 0000000000000..791003b9cddce
--- /dev/null
+++ b/common/utils/src/test/scala/org/apache/spark/util/SparkErrorUtilsSuite.scala
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.io.{Closeable, IOException}
+
+import scala.annotation.nowarn
+
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
+
+class SparkErrorUtilsSuite
+ extends AnyFunSuite // scalastyle:ignore funsuite
+ with BeforeAndAfterEach {
+
+ private var withoutCause: Throwable = _
+ private var withCause: Throwable = _
+ private var jdkNoCause: Throwable = _
+ private var nested: NestableException = _
+ private var cyclicCause: ExceptionWithCause = _
+
+ override def beforeEach(): Unit = {
+ withoutCause = createExceptionWithoutCause
+ nested = new NestableException(withoutCause)
+ withCause = new ExceptionWithCause(nested)
+ jdkNoCause = new NullPointerException
+ val exceptionA = new ExceptionWithCause(null.asInstanceOf[Throwable])
+ val exceptionB = new ExceptionWithCause(exceptionA)
+ exceptionA.setCauseValue(exceptionB)
+ cyclicCause = new ExceptionWithCause(exceptionA)
+ }
+
+ override def afterEach(): Unit = {
+ withoutCause = null
+ nested = null
+ withCause = null
+ jdkNoCause = null
+ cyclicCause = null
+ }
+
+ test("getRootCause") {
+ assert(SparkErrorUtils.getRootCause(null) == null)
+ assert(SparkErrorUtils.getRootCause(withoutCause) == withoutCause)
+ assert(SparkErrorUtils.getRootCause(nested) == withoutCause)
+ assert(SparkErrorUtils.getRootCause(withCause) == withoutCause)
+ assert(SparkErrorUtils.getRootCause(jdkNoCause) == jdkNoCause)
+ assert(SparkErrorUtils.getRootCause(cyclicCause) == cyclicCause.getCause.getCause)
+ }
+
+ test("tryWithResource / tryInitializeResource") {
+ val closeable = new Closeable {
+ override def close(): Unit = {
+ throw new IOException("Catch me if you can")
+ }
+ }
+ val e1 = intercept[IOException] {
+ SparkErrorUtils.tryWithResource(closeable)(_ => throw new IOException("You got me!"))
+ }
+ assert(e1.getMessage === "You got me!")
+ val e2 = intercept[IOException] {
+ SparkErrorUtils.tryInitializeResource(closeable)(_ => throw new IOException("You got me!"))
+ }
+ assert(e2.getMessage === "You got me!")
+ }
+
+ private def createExceptionWithoutCause: Throwable =
+ try throw new ExceptionWithoutCause
+ catch {
+ case t: Throwable => t
+ }
+
+ private final class ExceptionWithoutCause extends Exception {
+ @nowarn def getTargetException(): Unit = {}
+ }
+
+ private final class NestableException extends Exception {
+ def this(t: Throwable) = {
+ this()
+ initCause(t)
+ }
+ }
+
+ private final class ExceptionWithCause(message: String = null) extends Exception(message) {
+
+ private var _cause: Throwable = _
+
+ def this(message: String, cause: Throwable) = {
+ this(message)
+ this._cause = cause
+ }
+
+ def this(cause: Throwable) = {
+ this(null.asInstanceOf[String])
+ this._cause = cause
+ }
+
+ override def getCause: Throwable = synchronized {
+ _cause
+ }
+
+ def setCauseValue(cause: Throwable): Unit = {
+ this._cause = cause
+ }
+ }
+}
diff --git a/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala
index 0026b696f0695..20f6d2347eabe 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala
@@ -26,7 +26,7 @@ import org.apache.logging.log4j.Level
import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
-import org.apache.spark.internal.{LogEntry, Logging, LogKey, LogKeys, MDC, MessageWithContext}
+import org.apache.spark.internal.{LogEntry, Logging, LogKeys, MessageWithContext}
trait LoggingSuiteBase
extends AnyFunSuite // scalastyle:ignore funsuite
@@ -491,8 +491,3 @@ class StructuredLoggingSuite extends LoggingSuiteBase {
assert(mdcPattern.r.findFirstIn(logOutputWithoutMDCSet).isEmpty)
}
}
-
-object CustomLogKeys {
- // Custom `LogKey` must be `extends LogKey`
- case object CUSTOM_LOG_KEY extends LogKey
-}
diff --git a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
similarity index 96%
rename from core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
rename to common/utils/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
index ff68dd150973d..f04b55b1a6523 100644
--- a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
@@ -17,9 +17,9 @@
package org.apache.spark.util
-import org.apache.spark.SparkFunSuite
+import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
-class VersionUtilsSuite extends SparkFunSuite {
+class VersionUtilsSuite extends AnyFunSuite { // scalastyle:ignore funsuite
import org.apache.spark.util.VersionUtils._
diff --git a/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java b/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
index 9c4487dbd3f75..1bd008a5c9149 100644
--- a/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
+++ b/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
@@ -229,7 +229,7 @@ public void appendTimestampNtz(long microsSinceEpoch) {
public void appendFloat(float f) {
checkCapacity(1 + 4);
writeBuffer[writePos++] = primitiveHeader(FLOAT);
- writeLong(writeBuffer, writePos, Float.floatToIntBits(f), 8);
+ writeLong(writeBuffer, writePos, Float.floatToIntBits(f), 4);
writePos += 4;
}
diff --git a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
index 40ea064cbf115..795d46ec2062b 100644
--- a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
+++ b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
@@ -19,6 +19,7 @@
import org.apache.spark.QueryContext;
import org.apache.spark.SparkRuntimeException;
+import org.apache.spark.network.util.JavaUtils;
import scala.collection.immutable.Map$;
import java.math.BigDecimal;
@@ -140,7 +141,7 @@ public class VariantUtil {
// Both variant value and variant metadata need to be no longer than 128MiB.
// Note: to make tests more reliable, we set the max size to 16Mib to avoid OOM in tests.
public static final int SIZE_LIMIT =
- System.getenv("SPARK_TESTING") != null ? U24_MAX + 1 : 128 * 1024 * 1024;
+ JavaUtils.isTesting() ? U24_MAX + 1 : 128 * 1024 * 1024;
public static final int MAX_DECIMAL4_PRECISION = 9;
public static final int MAX_DECIMAL8_PRECISION = 18;
diff --git a/connect-examples/server-library-example/README.md b/connect-examples/server-library-example/README.md
deleted file mode 100644
index adf4830d58ff5..0000000000000
--- a/connect-examples/server-library-example/README.md
+++ /dev/null
@@ -1,133 +0,0 @@
-# Spark Server Library Example - Custom Datasource Handler
-
-This example demonstrates a modular maven-based project architecture with separate client, server
-and common components. It leverages the extensibility of Spark Connect to create a server library
-that may be attached to the server to extend the functionality of the Spark Connect server as a whole. Below is a detailed overview of the setup and functionality.
-
-## Project Structure
-
-```
-├── common/ # Shared protobuf/utilities/classes
-├── client/ # Sample client implementation
-│ ├── src/ # Source code for client functionality
-│ ├── pom.xml # Maven configuration for the client
-├── server/ # Server-side plugin extension
-│ ├── src/ # Source code for server functionality
-│ ├── pom.xml # Maven configuration for the server
-├── resources/ # Static resources
-├── pom.xml # Parent Maven configuration
-```
-
-## Functionality Overview
-
-To demonstrate the extensibility of Spark Connect, a custom datasource handler, `CustomTable` is
-implemented in the server module. The class handles reading, writing and processing data stored in
-a custom format, here we simply use the `.custom` extension (which itself is a wrapper over `.csv`
-files).
-
-First and foremost, the client and the server must be able to communicate with each other through
-custom messages that 'understand' our custom data format. This is achieved by defining custom
-protobuf messages in the `common` module. The client and server modules both depend on the `common`
-module to access these messages.
-- `common/src/main/protobuf/base.proto`: Defines the base `CustomTable` which is simply represented
-by a path and a name.
-```protobuf
-message CustomTable {
- string path = 1;
- string name = 2;
-}
-```
-- `common/src/main/protobuf/commands.proto`: Defines the custom commands that the client can send
-to the server. These commands are typically operations that the server can perform, such as cloning
-an existing custom table.
-```protobuf
-message CustomCommand {
- oneof command_type {
- CreateTable create_table = 1;
- CloneTable clone_table = 2;
- }
-}
-```
-- `common/src/main/protobuf/relations.proto`: Defines custom `relations`, which are a mechanism through which an optional input dataset is transformed into an
- output dataset such as a Scan.
-```protobuf
-message Scan {
- CustomTable table = 1;
-}
-```
-
-On the client side, the `CustomTable` class mimics the style of Spark's `Dataset` API, allowing the
-user to perform and chain operations on a `CustomTable` object.
-
-On the server side, a similar `CustomTable` class is implemented to handle the core functionality of
-reading, writing and processing data in the custom format. The plugins (`CustomCommandPlugin` and
-`CustomRelationPlugin`) are responsible for processing the custom protobuf messages sent from the client
-(those defined in the `common` module) and delegating the appropriate actions to the `CustomTable`.
-
-
-
-## Build and Run Instructions
-
-1. **Navigate to the sample project from `SPARK_HOME`**:
- ```bash
- cd connect-examples/server-library-example
- ```
-
-2. **Build and package the modules**:
- ```bash
- mvn clean package
- ```
-
-3. **Download the `4.0.0-preview2` release to use as the Spark Connect Server**:
- - Choose a distribution from https://archive.apache.org/dist/spark/spark-4.0.0-preview2/.
- - Example: `curl -L https://archive.apache.org/dist/spark/spark-4.0.0-preview2/spark-4.0.0-preview2-bin-hadoop3.tgz | tar xz`
-
-4. **Copy relevant JARs to the root of the unpacked Spark distribution**:
- ```bash
- cp \
- /connect-examples/server-library-example/common/target/spark-daria_2.13-1.2.3.jar \
- /connect-examples/server-library-example/common/target/spark-server-library-example-common-1.0.0.jar \
- /connect-examples/server-library-example/server/target/spark-server-library-example-server-extension-1.0.0.jar \
- .
- ```
-5. **Start the Spark Connect Server with the relevant JARs**:
- ```bash
- bin/spark-connect-shell \
- --jars spark-server-library-example-server-extension-1.0.0.jar,spark-server-library-example-common-1.0.0.jar,spark-daria_2.13-1.2.3.jar \
- --conf spark.connect.extensions.relation.classes=org.apache.connect.examples.serverlibrary.CustomRelationPlugin \
- --conf spark.connect.extensions.command.classes=org.apache.connect.examples.serverlibrary.CustomCommandPlugin
- ```
-6. **In a different terminal, navigate back to the root of the sample project and start the client**:
- ```bash
- java -cp client/target/spark-server-library-client-package-scala-1.0.0.jar org.apache.connect.examples.serverlibrary.CustomTableExample
- ```
-7. **Notice the printed output in the client terminal as well as the creation of the cloned table**:
-```protobuf
-Explaining plan for custom table: sample_table with path: /spark/connect-examples/server-library-example/client/../resources/dummy_data.custom
-== Parsed Logical Plan ==
-Relation [id#2,name#3] csv
-
-== Analyzed Logical Plan ==
-id: int, name: string
-Relation [id#2,name#3] csv
-
-== Optimized Logical Plan ==
-Relation [id#2,name#3] csv
-
-== Physical Plan ==
-FileScan csv [id#2,name#3] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/Users/venkata.gudesa/spark/connect-examples/server-library-example/resou..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct
-
-Explaining plan for custom table: cloned_table with path: /connect-examples/server-library-example/client/../resources/cloned_data.data
-== Parsed Logical Plan ==
-Relation [id#2,name#3] csv
-
-== Analyzed Logical Plan ==
-id: int, name: string
-Relation [id#2,name#3] csv
-
-== Optimized Logical Plan ==
-Relation [id#2,name#3] csv
-
-== Physical Plan ==
-FileScan csv [id#2,name#3] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/Users/venkata.gudesa/spark/connect-examples/server-library-example/resou..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct
-```
\ No newline at end of file
diff --git a/connect-examples/server-library-example/client/pom.xml b/connect-examples/server-library-example/client/pom.xml
deleted file mode 100644
index 364920a2ec22b..0000000000000
--- a/connect-examples/server-library-example/client/pom.xml
+++ /dev/null
@@ -1,112 +0,0 @@
-
-
-
-
- 4.0.0
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example
- 1.0.0
- ../pom.xml
-
-
- spark-server-library-client-package-scala
- jar
-
-
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example-common
- 1.0.0
-
-
- com.google.protobuf
- protobuf-java
-
-
-
-
-
- org.apache.spark
- spark-connect-common_${scala.binary}
- ${spark.version}
-
-
-
- org.apache.spark
- spark-connect-client-jvm_${scala.binary}
- ${spark.version}
-
-
-
- org.scala-lang
- scala-library
- ${scala.version}
-
-
-
-
-
- src/main/scala
-
-
-
-
- net.alchim31.maven
- scala-maven-plugin
- 4.9.2
-
-
-
- compile
- testCompile
-
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-shade-plugin
- 3.5.1
-
-
- package
-
- shade
-
-
- false
- true
- false
-
-
-
-
-
-
-
-
-
-
-
diff --git a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala b/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala
deleted file mode 100644
index 782a246d92984..0000000000000
--- a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import com.google.protobuf.Any
-import org.apache.spark.connect.proto.Command
-import org.apache.spark.sql.{functions, Column, DataFrame, Dataset, Row, SparkSession}
-
-import org.apache.connect.examples.serverlibrary.proto
-import org.apache.connect.examples.serverlibrary.proto.CreateTable.Column.{DataType => ProtoDataType}
-
-/**
- * Represents a custom table with associated DataFrame and metadata.
- *
- * @param df The underlying DataFrame.
- * @param table The metadata of the custom table.
- */
-class CustomTable private (private val df: Dataset[Row], private val table: proto.CustomTable) {
-
- /**
- * Returns the Spark session associated with the DataFrame.
- */
- private def spark = df.sparkSession
-
- /**
- * Converts the custom table to a DataFrame.
- *
- * @return The underlying DataFrame.
- */
- def toDF: Dataset[Row] = df
-
- /**
- * Prints the execution plan of the custom table.
- */
- def explain(): Unit = {
- println(s"Explaining plan for custom table: ${table.getName} with path: ${table.getPath}")
- df.explain("extended")
- }
-
- /**
- * Clones the custom table to a new location with a new name.
- *
- * @param target The target path for the cloned table.
- * @param newName The new name for the cloned table.
- * @param replace Whether to replace the target location if it exists.
- * @return A new `CustomTable` instance representing the cloned table.
- */
- def clone(target: String, newName: String, replace: Boolean): CustomTable = {
- val cloneTableProto = proto.CloneTable
- .newBuilder()
- .setTable(
- proto.CustomTable
- .newBuilder()
- .setName(table.getName)
- .setPath(table.getPath))
- .setClone(
- proto.CustomTable
- .newBuilder()
- .setName(newName)
- .setPath(target))
- .setReplace(replace)
- .build()
- val customCommand = proto.CustomCommand
- .newBuilder()
- .setCloneTable(cloneTableProto)
- .build()
- // Pack the CustomCommand into Any
- val customCommandAny = Any.pack(customCommand)
- // Set the Any as the extension of a Command
- val commandProto = Command
- .newBuilder()
- .setExtension(customCommandAny)
- .build()
- // Execute the command
- spark.execute(commandProto)
- CustomTable.from(spark, newName, target)
- }
-}
-
-object CustomTable {
- /**
- * Creates a `CustomTable` from the given Spark session, name, and path.
- *
- * @param spark The Spark session.
- * @param name The name of the table.
- * @param path The path of the table.
- * @return A new `CustomTable` instance.
- */
- def from(spark: SparkSession, name: String, path: String): CustomTable = {
- val table = proto.CustomTable
- .newBuilder()
- .setName(name)
- .setPath(path)
- .build()
- val relation = proto.CustomRelation
- .newBuilder()
- .setScan(proto.Scan.newBuilder().setTable(table))
- .build()
- val customRelation = Any.pack(relation)
- val df = spark.newDataFrame(f => f.setExtension(customRelation))
- new CustomTable(df, table)
- }
-
- /**
- * Creates a new `CustomTableBuilder` instance.
- *
- * @param spark The Spark session.
- * @return A new `CustomTableBuilder` instance.
- */
- def create(spark: SparkSession): CustomTableBuilder = new CustomTableBuilder(spark)
-
- /**
- * Enumeration for data types.
- */
- object DataType extends Enumeration {
- type DataType = Value
- val Int, String, Float, Boolean = Value
-
- /**
- * Converts a `DataType` to its corresponding `ProtoDataType`.
- *
- * @param dataType The data type to convert.
- * @return The corresponding `ProtoDataType`.
- */
- def toProto(dataType: DataType): ProtoDataType = {
- dataType match {
- case Int => ProtoDataType.INT
- case String => ProtoDataType.STRING
- case Float => ProtoDataType.FLOAT
- case Boolean => ProtoDataType.BOOLEAN
- }
- }
- }
-}
diff --git a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableBuilder.scala b/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableBuilder.scala
deleted file mode 100644
index a1b8ffdb8dd72..0000000000000
--- a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableBuilder.scala
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import com.google.protobuf.Any
-import org.apache.spark.connect.proto.Command
-import org.apache.spark.sql.SparkSession
-
-import org.apache.connect.examples.serverlibrary.CustomTable
-
-/**
- * Builder class for constructing a `CustomTable` instance.
- *
- * @param spark The Spark session.
- */
-class CustomTableBuilder private[serverlibrary] (spark: SparkSession) {
- import CustomTableBuilder._
-
- private var name: String = _
- private var path: String = _
- private var columns: Seq[Column] = Seq.empty
-
- /**
- * Sets the name of the custom table.
- *
- * @param name The name of the table.
- * @return The current `CustomTableBuilder` instance.
- */
- def name(name: String): CustomTableBuilder = {
- this.name = name
- this
- }
-
- /**
- * Sets the path of the custom table.
- *
- * @param path The path of the table.
- * @return The current `CustomTableBuilder` instance.
- */
- def path(path: String): CustomTableBuilder = {
- this.path = path
- this
- }
-
- /**
- * Adds a column to the custom table.
- *
- * @param name The name of the column.
- * @param dataType The data type of the column.
- * @return The current `CustomTableBuilder` instance.
- */
- def addColumn(name: String, dataType: CustomTable.DataType.Value): CustomTableBuilder = {
- columns = columns :+ Column(name, dataType)
- this
- }
-
- /**
- * Builds the `CustomTable` instance.
- *
- * @return A new `CustomTable` instance.
- * @throws IllegalArgumentException if name, path, or columns are not set.
- */
- def build(): CustomTable = {
- require(name != null, "Name must be set")
- require(path != null, "Path must be set")
- require(columns.nonEmpty, "At least one column must be added")
-
- // Define the table creation proto
- val createTableProtoBuilder = proto.CreateTable
- .newBuilder()
- .setTable(
- proto.CustomTable
- .newBuilder()
- .setPath(path)
- .setName(name)
- .build())
-
- // Add columns to the table creation proto
- columns.foreach { column =>
- createTableProtoBuilder.addColumns(
- proto.CreateTable.Column
- .newBuilder()
- .setName(column.name)
- .setDataType(CustomTable.DataType.toProto(column.dataType))
- .build())
- }
- val createTableProto = createTableProtoBuilder.build() // Build the CreateTable object
-
- // Wrap the CreateTable proto in CustomCommand
- val customCommand = proto.CustomCommand
- .newBuilder()
- .setCreateTable(createTableProto)
- .build()
-
- // Pack the CustomCommand into Any
- val customCommandAny = Any.pack(customCommand)
- // Set the Any as the extension of a Command
- val commandProto = Command
- .newBuilder()
- .setExtension(customCommandAny)
- .build()
-
- // Execute the command
- spark.execute(commandProto)
-
- // After the command is executed, create a client-side representation of the table with the
- // leaf node of the client-side dataset being the Scan node for the custom table.
- CustomTable.from(spark, name, path)
- }
-}
-
-object CustomTableBuilder {
- /**
- * Case class representing a column in the custom table.
- *
- * @param name The name of the column.
- * @param dataType The data type of the column.
- */
- private case class Column(name: String, dataType: CustomTable.DataType.Value)
-}
diff --git a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableExample.scala b/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableExample.scala
deleted file mode 100644
index 8470465cd7a0b..0000000000000
--- a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableExample.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import java.nio.file.{Path, Paths}
-
-import com.google.protobuf.Any
-import org.apache.spark.connect.proto.Command
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.types.{StructType, StructField, StringType, IntegerType}
-
-import org.apache.connect.examples.serverlibrary.proto
-import org.apache.connect.examples.serverlibrary.proto.CreateTable.Column.{DataType => ProtoDataType}
-
-object CustomTableExample {
- def main(args: Array[String]): Unit = {
- val spark = SparkSession.builder().remote("sc://localhost").build()
-
- // Step 1: Create a custom table from existing data
- val tableName = "sample_table"
- val fileName = "dummy_data.data"
-
- val workingDirectory = System.getProperty("user.dir")
- val fullPath = Paths.get(workingDirectory, s"resources/$fileName")
- val customTable = CustomTable
- .create(spark)
- .name(tableName)
- .path(fullPath.toString)
- .addColumn("id", CustomTable.DataType.Int)
- .addColumn("name", CustomTable.DataType.String)
- .build()
-
- // Step 2: Verify
- customTable.explain()
-
- // Step 3: Clone the custom table
- val clonedPath = fullPath.getParent.resolve("cloned_data.data")
- val clonedName = "cloned_table"
- val clonedTable =
- customTable.clone(target = clonedPath.toString, newName = clonedName, replace = true)
-
- // Step 4: Verify
- clonedTable.explain()
- }
-}
diff --git a/connect-examples/server-library-example/common/pom.xml b/connect-examples/server-library-example/common/pom.xml
deleted file mode 100644
index 592c43f26770b..0000000000000
--- a/connect-examples/server-library-example/common/pom.xml
+++ /dev/null
@@ -1,76 +0,0 @@
-
-
-
-
- 4.0.0
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example
- 1.0.0
- ../pom.xml
-
-
- spark-server-library-example-common
- jar
-
-
-
-
- com.google.protobuf
- protobuf-java
- ${protobuf.version}
-
-
- org.scala-lang
- scala-library
- ${scala.version}
-
-
-
-
-
-
-
- kr.motd.maven
- os-maven-plugin
- 1.7.0
- true
-
-
- org.xolstice.maven.plugins
- protobuf-maven-plugin
- 0.6.1
-
- com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
- src/main/protobuf
-
-
-
- generate-sources
-
- compile
-
-
-
-
-
-
-
diff --git a/connect-examples/server-library-example/common/src/main/protobuf/commands.proto b/connect-examples/server-library-example/common/src/main/protobuf/commands.proto
deleted file mode 100644
index 13d9945cfe61d..0000000000000
--- a/connect-examples/server-library-example/common/src/main/protobuf/commands.proto
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = 'proto3';
-
-option java_multiple_files = true;
-option java_package = "org.apache.connect.examples.serverlibrary.proto";
-
-import "base.proto";
-
-message CustomCommand {
- oneof command_type {
- CreateTable create_table = 1;
- CloneTable clone_table = 2;
- }
-}
-
-message CreateTable {
-
- // Column in the schema of the table.
- message Column {
- // (Required) Name of the column.
- string name = 1;
- // (Required) Data type of the column.
- enum DataType {
- DATA_TYPE_UNSPECIFIED = 0; // Default value
- INT = 1; // Integer data type
- STRING = 2; // String data type
- FLOAT = 3; // Float data type
- BOOLEAN = 4; // Boolean data type
- }
- DataType data_type = 2;
- }
- // (Required) Table properties.
- CustomTable table = 1;
- // (Required) List of columns in the schema of the table.
- repeated Column columns = 2;
-}
-
-message CloneTable {
- // (Required) The source table to clone.
- CustomTable table = 1;
- // (Required) Path to the location where the data of the cloned table should be stored.
- CustomTable clone = 2;
- // (Required) Overwrites the target location when true.
- bool replace = 3;
-}
diff --git a/connect-examples/server-library-example/pom.xml b/connect-examples/server-library-example/pom.xml
deleted file mode 100644
index 1723f3b0154fa..0000000000000
--- a/connect-examples/server-library-example/pom.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-
-
-
-
- 4.0.0
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example
- 1.0.0
- pom
-
- common
- server
- client
-
-
- 17
- 17
- UTF-8
- 2.13
- 2.13.15
- 3.25.4
- 4.0.0-preview2
-
-
diff --git a/connect-examples/server-library-example/resources/dummy_data.data b/connect-examples/server-library-example/resources/dummy_data.data
deleted file mode 100644
index 0a6645b757222..0000000000000
--- a/connect-examples/server-library-example/resources/dummy_data.data
+++ /dev/null
@@ -1,6 +0,0 @@
-id,name
-1,John Doe
-2,Jane Smith
-3,Bob Johnson
-4,Alice Williams
-5,Charlie Brown
\ No newline at end of file
diff --git a/connect-examples/server-library-example/server/pom.xml b/connect-examples/server-library-example/server/pom.xml
deleted file mode 100644
index b13a7537f9c13..0000000000000
--- a/connect-examples/server-library-example/server/pom.xml
+++ /dev/null
@@ -1,90 +0,0 @@
-
-
-
-
- 4.0.0
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example
- 1.0.0
- ../pom.xml
-
-
- spark-server-library-example-server-extension
- jar
-
-
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example-common
- 1.0.0
-
-
-
- org.scala-lang
- scala-library
- ${scala.version}
-
-
-
- org.apache.spark
- spark-sql_${scala.binary}
- ${spark.version}
- provided
-
-
-
- org.apache.spark
- spark-connect_${scala.binary}
- ${spark.version}
- provided
-
-
-
-
- com.github.mrpowers
- spark-daria_${scala.binary}
- 1.2.3
-
-
-
-
- src/main/scala
-
-
-
-
- net.alchim31.maven
- scala-maven-plugin
- 4.9.2
-
-
-
- compile
- testCompile
-
-
-
-
-
-
-
diff --git a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomCommandPlugin.scala b/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomCommandPlugin.scala
deleted file mode 100644
index 2253c4b238be4..0000000000000
--- a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomCommandPlugin.scala
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import scala.collection.JavaConverters._
-
-import com.google.protobuf.Any
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.connect.planner.SparkConnectPlanner
-import org.apache.spark.sql.connect.plugin.CommandPlugin
-import org.apache.spark.sql.types.{StringType, IntegerType, FloatType, DoubleType, BooleanType, LongType, StructType, StructField, DataType}
-
-import org.apache.connect.examples.serverlibrary.CustomTable
-import org.apache.connect.examples.serverlibrary.proto
-import org.apache.connect.examples.serverlibrary.proto.CreateTable.Column.{DataType => ProtoDataType}
-
-/**
- * Commands are distinct actions that can be executed. Unlike relations, which focus on the
- * transformation and nesting of output data, commands represent singular operations that perform
- * specific tasks on the data.
- * In this example, the `CustomCommandPlugin` handles operations related to creating and duplicating
- * custom tables.
- */
-class CustomCommandPlugin extends CommandPlugin with CustomPluginBase {
-
- /**
- * Processes the raw byte array containing the command.
- *
- * @param raw The raw byte array of the command.
- * @param planner The SparkConnectPlanner instance.
- * @return True if the command was processed, false otherwise.
- */
- override def process(raw: Array[Byte], planner: SparkConnectPlanner): Boolean = {
- val command = Any.parseFrom(raw)
- if (command.is(classOf[proto.CustomCommand])) {
- processInternal(command.unpack(classOf[proto.CustomCommand]), planner)
- true
- } else {
- false
- }
- }
-
- /**
- * Processes the unpacked CustomCommand.
- *
- * @param command The unpacked CustomCommand.
- * @param planner The SparkConnectPlanner instance.
- */
- private def processInternal(
- command: proto.CustomCommand,
- planner: SparkConnectPlanner): Unit = {
- command.getCommandTypeCase match {
- case proto.CustomCommand.CommandTypeCase.CREATE_TABLE =>
- processCreateTable(planner, command.getCreateTable)
- case proto.CustomCommand.CommandTypeCase.CLONE_TABLE =>
- processCloneTable(planner, command.getCloneTable)
- case _ =>
- throw new IllegalArgumentException(
- s"Unsupported command type: ${command.getCommandTypeCase}")
- }
- }
-
- /**
- * Processes the CreateTable command.
- *
- * @param planner The SparkConnectPlanner instance.
- * @param createTable The CreateTable message.
- */
- private def processCreateTable(
- planner: SparkConnectPlanner,
- createTable: proto.CreateTable): Unit = {
- val tableName = createTable.getTable.getName
- val tablePath = createTable.getTable.getPath
-
- // Convert the list of columns from the protobuf message to a Spark schema
- val schema = StructType(createTable.getColumnsList.asScala.toSeq.map { column =>
- StructField(
- column.getName,
- protoDataTypeToSparkType(column.getDataType),
- nullable = true // Assuming all columns are nullable for simplicity
- )
- })
-
- // Create the table using the CustomTable utility
- CustomTable.createTable(tableName, tablePath, planner.session, schema)
- }
-
- /**
- * Converts a protobuf DataType to a Spark DataType.
- *
- * @param protoType The protobuf DataType.
- * @return The corresponding Spark DataType.
- */
- private def protoDataTypeToSparkType(protoType: ProtoDataType): DataType = {
- protoType match {
- case ProtoDataType.INT => IntegerType
- case ProtoDataType.STRING => StringType
- case ProtoDataType.FLOAT => FloatType
- case ProtoDataType.BOOLEAN => BooleanType
- case _ =>
- throw new IllegalArgumentException(s"Unsupported or unknown data type: ${protoType}")
- }
- }
-
- /**
- * Processes the CloneTable command.
- *
- * @param planner The SparkConnectPlanner instance.
- * @param msg The CloneTable message.
- */
- private def processCloneTable(planner: SparkConnectPlanner, msg: proto.CloneTable): Unit = {
- val sourceTable = getCustomTable(msg.getTable)
- CustomTable.cloneTable(
- sourceTable,
- msg.getClone.getName,
- msg.getClone.getPath,
- msg.getReplace)
- }
-}
diff --git a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomRelationPlugin.scala b/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomRelationPlugin.scala
deleted file mode 100644
index 7b444803a065f..0000000000000
--- a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomRelationPlugin.scala
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import java.util.Optional
-
-import com.google.protobuf.Any
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.connect.planner.SparkConnectPlanner
-import org.apache.spark.sql.connect.plugin.RelationPlugin
-
-import org.apache.connect.examples.serverlibrary.{CustomPluginBase, CustomTable}
-import org.apache.connect.examples.serverlibrary.proto
-
-/**
- * Relations are fundamental to dataset transformations, acting as the mechanism through which an
- * input dataset is transformed into an output dataset. Conceptually, relations can be likened to
- * tables within a database, manipulated to achieve desired outcomes.
- * In this example, the `CustomRelationPlugin` handles the transformation related to scanning
- * custom tables. The scan relation would appear as leaf nodes in a dataset's associated logical
- * plan node when it involves reads from the custom tables.
- */
-class CustomRelationPlugin extends RelationPlugin with CustomPluginBase {
-
- /**
- * Transforms the raw byte array containing the relation into a Spark logical plan.
- *
- * @param raw The raw byte array of the relation.
- * @param planner The SparkConnectPlanner instance.
- * @return An Optional containing the LogicalPlan if the relation was processed, empty otherwise.
- */
- override def transform(
- raw: Array[Byte],
- planner: SparkConnectPlanner): Optional[LogicalPlan] = {
- val rel = Any.parseFrom(raw)
- if (rel.is(classOf[proto.CustomRelation])) {
- val customRelation = rel.unpack(classOf[proto.CustomRelation])
- // Transform the custom relation
- Optional.of(transformInner(customRelation, planner))
- } else {
- Optional.empty()
- }
- }
-
- /**
- * Transforms the unpacked CustomRelation into a Spark logical plan.
- *
- * @param relation The unpacked CustomRelation.
- * @param planner The SparkConnectPlanner instance.
- * @return The corresponding Spark LogicalPlan.
- */
- private def transformInner(
- relation: proto.CustomRelation,
- planner: SparkConnectPlanner): LogicalPlan = {
- relation.getRelationTypeCase match {
- case proto.CustomRelation.RelationTypeCase.SCAN =>
- transformScan(relation.getScan, planner)
- case _ =>
- throw new IllegalArgumentException(
- s"Unsupported relation type: ${relation.getRelationTypeCase}")
- }
- }
-
- /**
- * Transforms the Scan relation into a Spark logical plan.
- *
- * @param scan The Scan message.
- * @param planner The SparkConnectPlanner instance.
- * @return The corresponding Spark LogicalPlan.
- */
- private def transformScan(scan: proto.Scan, planner: SparkConnectPlanner): LogicalPlan = {
- val customTable = getCustomTable(scan.getTable)
- // Convert the custom table to a DataFrame and get its logical plan
- customTable.toDF().queryExecution.analyzed
- }
-}
diff --git a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala b/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala
deleted file mode 100644
index e1630cc25edd4..0000000000000
--- a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import java.util.UUID
-
-import com.github.mrpowers.spark.daria.sql.DariaWriters
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.{Dataset, Row, SparkSession}
-
-/**
- * Represents a custom table with an identifier and a DataFrame.
- *
- * @param identifier The unique identifier for the table.
- * @param df The DataFrame associated with the table.
- */
-class CustomTable private (identifier: CustomTable.Identifier, df: Dataset[Row]) {
-
- /**
- * Returns the DataFrame associated with the table.
- *
- * @return The DataFrame.
- */
- def toDF(): Dataset[Row] = df
-
- /**
- * Writes the DataFrame to disk as a CSV file.
- */
- def flush(): Unit = {
- // Write dataset to disk as a CSV file
- DariaWriters.writeSingleFile(
- df = df,
- format = "csv",
- sc = df.sparkSession.sparkContext,
- tmpFolder = s"./${UUID.randomUUID().toString}",
- filename = identifier.path,
- saveMode = "overwrite")
- }
-}
-
-object CustomTable {
-
- /**
- * Represents the unique identifier for a custom table.
- *
- * @param name The name of the table.
- * @param path The path where the table is stored.
- */
- private case class Identifier(name: String, path: String)
-
- // Collection holding all the CustomTable instances and searchable by the identifier
- private val tablesByIdentifier = scala.collection.mutable.Map[Identifier, CustomTable]()
-
- /**
- * Creates a new custom table.
- *
- * @param name The name of the table.
- * @param path The path where the table is stored.
- * @param spark The SparkSession instance.
- * @param schema The schema of the table.
- * @return The created CustomTable instance.
- */
- private[serverlibrary] def createTable(
- name: String,
- path: String,
- spark: SparkSession,
- schema: StructType): CustomTable = {
- val identifier = Identifier(name, path)
- val df = spark.read
- .option("header", "true")
- .schema(schema)
- .csv(path)
- val table = new CustomTable(identifier, df)
- tablesByIdentifier(identifier) = table
- table
- }
-
- /**
- * Clones an existing custom table.
- *
- * @param sourceTable The source table to clone.
- * @param newName The name of the new table.
- * @param newPath The path where the new table will be stored.
- * @param replace Whether to replace the existing table if it exists.
- * @return The cloned CustomTable instance.
- */
- private[serverlibrary] def cloneTable(
- sourceTable: CustomTable,
- newName: String,
- newPath: String,
- replace: Boolean): CustomTable = {
- val newIdentifier = Identifier(newName, newPath)
- val clonedDf = sourceTable.toDF()
- val clonedTable = new CustomTable(newIdentifier, clonedDf)
- clonedTable.flush()
- tablesByIdentifier(newIdentifier) = clonedTable
- clonedTable
- }
-
- /**
- * Retrieves a custom table based on its identifier.
- *
- * @param name The name of the table.
- * @param path The path where the table is stored.
- * @return The CustomTable instance.
- * @throws IllegalArgumentException if the table is not found.
- */
- def getTable(name: String, path: String): CustomTable = {
- val identifier = Identifier(name, path)
- tablesByIdentifier.get(identifier) match {
- case Some(table) => table
- case None =>
- throw new IllegalArgumentException(s"Table with identifier $identifier not found")
- }
- }
-}
diff --git a/connector/avro/benchmarks/AvroReadBenchmark-jdk21-results.txt b/connector/avro/benchmarks/AvroReadBenchmark-jdk21-results.txt
index e01810e1e970c..6eca519442e12 100644
--- a/connector/avro/benchmarks/AvroReadBenchmark-jdk21-results.txt
+++ b/connector/avro/benchmarks/AvroReadBenchmark-jdk21-results.txt
@@ -2,140 +2,140 @@
SQL Single Numeric Column Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single TINYINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 2045 2062 24 7.7 130.0 1.0X
+Sum 2054 2123 98 7.7 130.6 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single SMALLINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 2033 2040 11 7.7 129.2 1.0X
+Sum 2044 2050 9 7.7 129.9 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single INT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1959 1965 8 8.0 124.6 1.0X
+Sum 2037 2050 18 7.7 129.5 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single BIGINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1983 2019 51 7.9 126.1 1.0X
+Sum 2038 2056 25 7.7 129.6 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single FLOAT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1935 1941 7 8.1 123.1 1.0X
+Sum 2021 2024 4 7.8 128.5 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single DOUBLE Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1960 1976 23 8.0 124.6 1.0X
+Sum 2024 2029 7 7.8 128.7 1.0X
================================================================================================
Int and String Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Int and String Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of columns 3561 3573 16 2.9 339.6 1.0X
+Sum of columns 3401 3402 0 3.1 324.4 1.0X
================================================================================================
Partitioned Table Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Partitioned Table: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Data column 2138 2158 28 7.4 135.9 1.0X
-Partition column 1817 1828 15 8.7 115.5 1.2X
-Both columns 2066 2092 36 7.6 131.4 1.0X
+Data column 2021 2167 207 7.8 128.5 1.0X
+Partition column 1812 1823 16 8.7 115.2 1.1X
+Both columns 2034 2039 8 7.7 129.3 1.0X
================================================================================================
Repeated String Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Repeated String: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 2183 2212 41 4.8 208.2 1.0X
+Sum of string length 2157 2180 32 4.9 205.7 1.0X
================================================================================================
String with Nulls Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (0.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 3278 3296 26 3.2 312.6 1.0X
+Sum of string length 3241 3292 71 3.2 309.1 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (50.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 2148 2152 5 4.9 204.9 1.0X
+Sum of string length 2167 2172 7 4.8 206.7 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (95.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 1128 1129 1 9.3 107.6 1.0X
+Sum of string length 1115 1121 9 9.4 106.3 1.0X
================================================================================================
Select All From Wide Columns
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Wide Column Scan from 1000 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Select of all columns 20327 20405 111 0.0 40653.8 1.0X
+Select of all columns 18712 18825 161 0.0 37423.2 1.0X
================================================================================================
Single Column Scan From Wide Columns
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 100 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 3365 3372 9 0.3 3209.4 1.0X
+Sum of single column 3210 3224 19 0.3 3061.4 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 200 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 6611 6633 32 0.2 6304.7 1.0X
+Sum of single column 6290 6298 11 0.2 5998.6 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 300 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 10030 10036 9 0.1 9565.3 1.0X
+Sum of single column 9353 9359 8 0.1 8920.0 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Filters pushdown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-w/o filters 5455 5470 21 0.2 5455.0 1.0X
-pushdown disabled 5362 5367 5 0.2 5362.0 1.0X
-w/ filters 2367 2384 15 0.4 2367.4 2.3X
+w/o filters 5389 5409 20 0.2 5389.3 1.0X
+pushdown disabled 5262 5277 19 0.2 5262.5 1.0X
+w/ filters 2315 2327 17 0.4 2314.9 2.3X
diff --git a/connector/avro/benchmarks/AvroReadBenchmark-results.txt b/connector/avro/benchmarks/AvroReadBenchmark-results.txt
index 85f3a34340cd7..a52077d0349ef 100644
--- a/connector/avro/benchmarks/AvroReadBenchmark-results.txt
+++ b/connector/avro/benchmarks/AvroReadBenchmark-results.txt
@@ -2,140 +2,140 @@
SQL Single Numeric Column Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single TINYINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1925 1972 66 8.2 122.4 1.0X
+Sum 1972 2013 57 8.0 125.4 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single SMALLINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1958 1971 18 8.0 124.5 1.0X
+Sum 1946 2000 77 8.1 123.7 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single INT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1918 1920 4 8.2 121.9 1.0X
+Sum 1904 1908 5 8.3 121.1 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single BIGINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1877 1888 16 8.4 119.3 1.0X
+Sum 1897 1897 0 8.3 120.6 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single FLOAT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1886 1891 7 8.3 119.9 1.0X
+Sum 1903 1905 3 8.3 121.0 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single DOUBLE Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1870 1871 1 8.4 118.9 1.0X
+Sum 1891 1893 3 8.3 120.2 1.0X
================================================================================================
Int and String Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Int and String Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of columns 3539 3544 7 3.0 337.5 1.0X
+Sum of columns 3386 3389 5 3.1 322.9 1.0X
================================================================================================
Partitioned Table Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Partitioned Table: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Data column 1949 1950 1 8.1 123.9 1.0X
-Partition column 1780 1791 16 8.8 113.2 1.1X
-Both columns 1985 2001 23 7.9 126.2 1.0X
+Data column 2097 2100 4 7.5 133.3 1.0X
+Partition column 1906 1910 6 8.3 121.2 1.1X
+Both columns 2121 2140 27 7.4 134.9 1.0X
================================================================================================
Repeated String Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Repeated String: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 2081 2096 21 5.0 198.4 1.0X
+Sum of string length 2265 2302 52 4.6 216.0 1.0X
================================================================================================
String with Nulls Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (0.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 3094 3096 2 3.4 295.1 1.0X
+Sum of string length 3130 3153 32 3.3 298.5 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (50.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 2161 2185 34 4.9 206.1 1.0X
+Sum of string length 2007 2010 4 5.2 191.4 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (95.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 1106 1110 6 9.5 105.4 1.0X
+Sum of string length 1081 1085 5 9.7 103.1 1.0X
================================================================================================
Select All From Wide Columns
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Wide Column Scan from 1000 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Select of all columns 18000 18000 0 0.0 35999.1 1.0X
+Select of all columns 19411 19525 161 0.0 38822.8 1.0X
================================================================================================
Single Column Scan From Wide Columns
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 100 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 3094 3103 13 0.3 2950.6 1.0X
+Sum of single column 3194 3195 2 0.3 3046.2 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 200 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 6074 6083 13 0.2 5792.7 1.0X
+Sum of single column 6186 6196 14 0.2 5899.7 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 300 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 9105 9122 23 0.1 8683.6 1.0X
+Sum of single column 9421 9428 9 0.1 8984.8 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Filters pushdown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-w/o filters 5123 5132 12 0.2 5122.5 1.0X
-pushdown disabled 5071 5085 20 0.2 5070.8 1.0X
-w/ filters 1967 1984 26 0.5 1967.2 2.6X
+w/o filters 5279 5288 14 0.2 5278.6 1.0X
+pushdown disabled 5157 5168 10 0.2 5156.7 1.0X
+w/ filters 2228 2252 21 0.4 2228.2 2.4X
diff --git a/connector/avro/benchmarks/AvroWriteBenchmark-jdk21-results.txt b/connector/avro/benchmarks/AvroWriteBenchmark-jdk21-results.txt
index 2014407498850..78e9034718f8b 100644
--- a/connector/avro/benchmarks/AvroWriteBenchmark-jdk21-results.txt
+++ b/connector/avro/benchmarks/AvroWriteBenchmark-jdk21-results.txt
@@ -1,56 +1,56 @@
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro writer benchmark: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column 1672 1680 12 9.4 106.3 1.0X
-Output Single Double Column 1572 1572 0 10.0 100.0 1.1X
-Output Int and String Column 3337 3351 20 4.7 212.1 0.5X
-Output Partitions 2984 2998 20 5.3 189.7 0.6X
-Output Buckets 3939 3948 14 4.0 250.4 0.4X
+Output Single Int Column 1482 1497 21 10.6 94.3 1.0X
+Output Single Double Column 1535 1551 23 10.2 97.6 1.0X
+Output Int and String Column 3392 3472 114 4.6 215.6 0.4X
+Output Partitions 3017 3038 29 5.2 191.8 0.5X
+Output Buckets 3915 3959 63 4.0 248.9 0.4X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro compression with different codec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-BZIP2: 117102 117155 74 0.0 1171024.1 1.0X
-DEFLATE: 6412 6438 36 0.0 64121.2 18.3X
-UNCOMPRESSED: 5015 5034 26 0.0 50153.9 23.3X
-SNAPPY: 4540 4542 2 0.0 45403.0 25.8X
-XZ: 41913 42574 934 0.0 419134.4 2.8X
-ZSTANDARD: 4807 4848 58 0.0 48065.1 24.4X
+BZIP2: 116777 117122 488 0.0 1167768.2 1.0X
+DEFLATE: 6324 6341 25 0.0 63236.5 18.5X
+UNCOMPRESSED: 4864 4880 22 0.0 48644.2 24.0X
+SNAPPY: 4491 4554 89 0.0 44910.2 26.0X
+XZ: 42198 42471 386 0.0 421982.0 2.8X
+ZSTANDARD: 4894 4921 37 0.0 48943.6 23.9X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro deflate with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-DEFLATE: deflate.level=1 4691 4719 40 0.0 46909.5 1.0X
-DEFLATE: deflate.level=3 4773 4773 0 0.0 47731.7 1.0X
-DEFLATE: deflate.level=5 6225 6233 11 0.0 62254.0 0.8X
-DEFLATE: deflate.level=7 6308 6323 21 0.0 63075.8 0.7X
-DEFLATE: deflate.level=9 6632 6639 9 0.0 66320.6 0.7X
+DEFLATE: deflate.level=1 4788 4806 26 0.0 47878.4 1.0X
+DEFLATE: deflate.level=3 4758 4762 5 0.0 47583.6 1.0X
+DEFLATE: deflate.level=5 6334 6340 9 0.0 63338.4 0.8X
+DEFLATE: deflate.level=7 6394 6417 33 0.0 63938.5 0.7X
+DEFLATE: deflate.level=9 6682 6689 9 0.0 66823.7 0.7X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro xz with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-XZ: xz.level=1 11561 11582 30 0.0 115608.9 1.0X
-XZ: xz.level=3 21560 21618 82 0.0 215604.1 0.5X
-XZ: xz.level=5 40673 40707 48 0.0 406733.7 0.3X
-XZ: xz.level=7 58646 59202 787 0.0 586455.4 0.2X
-XZ: xz.level=9 130798 131012 302 0.0 1307981.2 0.1X
+XZ: xz.level=1 11426 11457 44 0.0 114258.9 1.0X
+XZ: xz.level=3 21402 21445 60 0.0 214021.3 0.5X
+XZ: xz.level=5 41381 41439 82 0.0 413809.1 0.3X
+XZ: xz.level=7 58841 59364 740 0.0 588409.7 0.2X
+XZ: xz.level=9 134154 135421 1792 0.0 1341536.7 0.1X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro zstandard with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------------------
-ZSTANDARD: zstandard.level=1 4842 4883 58 0.0 48423.9 1.0X
-ZSTANDARD: zstandard.level=1, zstandard.bufferPool.enabled=true 4659 4685 37 0.0 46593.3 1.0X
-ZSTANDARD: zstandard.level=3 4808 4874 93 0.0 48075.0 1.0X
-ZSTANDARD: zstandard.level=3, zstandard.bufferPool.enabled=true 4554 4586 44 0.0 45541.4 1.1X
-ZSTANDARD: zstandard.level=5 5024 5026 2 0.0 50244.9 1.0X
-ZSTANDARD: zstandard.level=5, zstandard.bufferPool.enabled=true 4810 4812 3 0.0 48099.6 1.0X
-ZSTANDARD: zstandard.level=7 5223 5237 20 0.0 52232.5 0.9X
-ZSTANDARD: zstandard.level=7, zstandard.bufferPool.enabled=true 5184 5192 12 0.0 51837.0 0.9X
-ZSTANDARD: zstandard.level=9 6023 6059 52 0.0 60226.6 0.8X
-ZSTANDARD: zstandard.level=9, zstandard.bufferPool.enabled=true 5886 5896 15 0.0 58855.8 0.8X
+ZSTANDARD: zstandard.level=1 4657 4660 5 0.0 46567.0 1.0X
+ZSTANDARD: zstandard.level=1, zstandard.bufferPool.enabled=true 4631 4647 23 0.0 46306.3 1.0X
+ZSTANDARD: zstandard.level=3 4766 4834 97 0.0 47657.0 1.0X
+ZSTANDARD: zstandard.level=3, zstandard.bufferPool.enabled=true 4656 4684 40 0.0 46559.4 1.0X
+ZSTANDARD: zstandard.level=5 4908 4912 5 0.0 49079.9 0.9X
+ZSTANDARD: zstandard.level=5, zstandard.bufferPool.enabled=true 4832 4846 21 0.0 48318.0 1.0X
+ZSTANDARD: zstandard.level=7 5376 5398 32 0.0 53755.4 0.9X
+ZSTANDARD: zstandard.level=7, zstandard.bufferPool.enabled=true 5270 5279 13 0.0 52703.5 0.9X
+ZSTANDARD: zstandard.level=9 6256 6389 188 0.0 62559.6 0.7X
+ZSTANDARD: zstandard.level=9, zstandard.bufferPool.enabled=true 6291 6304 18 0.0 62913.0 0.7X
diff --git a/connector/avro/benchmarks/AvroWriteBenchmark-results.txt b/connector/avro/benchmarks/AvroWriteBenchmark-results.txt
index 4582347ceb11b..4e2ed7cabc262 100644
--- a/connector/avro/benchmarks/AvroWriteBenchmark-results.txt
+++ b/connector/avro/benchmarks/AvroWriteBenchmark-results.txt
@@ -1,56 +1,56 @@
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro writer benchmark: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column 1556 1579 33 10.1 98.9 1.0X
-Output Single Double Column 1591 1601 14 9.9 101.2 1.0X
-Output Int and String Column 3363 3379 22 4.7 213.8 0.5X
-Output Partitions 2937 2952 21 5.4 186.8 0.5X
-Output Buckets 3823 3825 3 4.1 243.0 0.4X
+Output Single Int Column 1630 1632 4 9.7 103.6 1.0X
+Output Single Double Column 1690 1732 60 9.3 107.4 1.0X
+Output Int and String Column 3267 3277 14 4.8 207.7 0.5X
+Output Partitions 3014 3027 18 5.2 191.6 0.5X
+Output Buckets 3786 3838 74 4.2 240.7 0.4X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro compression with different codec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-BZIP2: 132833 133636 1136 0.0 1328332.0 1.0X
-DEFLATE: 6522 6554 45 0.0 65216.8 20.4X
-UNCOMPRESSED: 5131 5150 27 0.0 51311.0 25.9X
-SNAPPY: 4709 4766 80 0.0 47091.8 28.2X
-XZ: 42816 43168 498 0.0 428160.6 3.1X
-ZSTANDARD: 4835 4850 22 0.0 48345.6 27.5X
+BZIP2: 132013 132044 45 0.0 1320126.5 1.0X
+DEFLATE: 6551 6565 20 0.0 65507.6 20.2X
+UNCOMPRESSED: 5077 5095 25 0.0 50773.1 26.0X
+SNAPPY: 4750 4754 6 0.0 47498.4 27.8X
+XZ: 43062 43195 187 0.0 430623.4 3.1X
+ZSTANDARD: 4866 4875 13 0.0 48658.9 27.1X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro deflate with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-DEFLATE: deflate.level=1 4832 4836 6 0.0 48320.1 1.0X
-DEFLATE: deflate.level=3 4798 4818 28 0.0 47979.1 1.0X
-DEFLATE: deflate.level=5 6472 6473 2 0.0 64717.5 0.7X
-DEFLATE: deflate.level=7 6516 6519 5 0.0 65161.0 0.7X
-DEFLATE: deflate.level=9 6877 6879 3 0.0 68768.4 0.7X
+DEFLATE: deflate.level=1 4833 4891 83 0.0 48325.8 1.0X
+DEFLATE: deflate.level=3 4814 4815 2 0.0 48143.0 1.0X
+DEFLATE: deflate.level=5 6409 6503 132 0.0 64094.4 0.8X
+DEFLATE: deflate.level=7 6465 6484 27 0.0 64647.7 0.7X
+DEFLATE: deflate.level=9 6805 6808 4 0.0 68050.2 0.7X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro xz with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-XZ: xz.level=1 11901 11909 11 0.0 119013.0 1.0X
-XZ: xz.level=3 22159 22204 64 0.0 221592.3 0.5X
-XZ: xz.level=5 41324 41591 378 0.0 413243.8 0.3X
-XZ: xz.level=7 59391 59735 487 0.0 593906.4 0.2X
-XZ: xz.level=9 132884 135160 3219 0.0 1328837.1 0.1X
+XZ: xz.level=1 11743 11841 138 0.0 117432.0 1.0X
+XZ: xz.level=3 21455 21455 0 0.0 214552.1 0.5X
+XZ: xz.level=5 40359 40688 465 0.0 403590.5 0.3X
+XZ: xz.level=7 58295 58471 249 0.0 582952.8 0.2X
+XZ: xz.level=9 131053 135036 5632 0.0 1310532.2 0.1X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro zstandard with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------------------
-ZSTANDARD: zstandard.level=1 4707 4753 65 0.0 47073.3 1.0X
-ZSTANDARD: zstandard.level=1, zstandard.bufferPool.enabled=true 4701 4717 23 0.0 47006.8 1.0X
-ZSTANDARD: zstandard.level=3 4810 4857 67 0.0 48096.6 1.0X
-ZSTANDARD: zstandard.level=3, zstandard.bufferPool.enabled=true 4768 4799 44 0.0 47684.0 1.0X
-ZSTANDARD: zstandard.level=5 5093 5134 59 0.0 50928.2 0.9X
-ZSTANDARD: zstandard.level=5, zstandard.bufferPool.enabled=true 5033 5052 27 0.0 50331.4 0.9X
-ZSTANDARD: zstandard.level=7 5538 5554 22 0.0 55381.9 0.8X
-ZSTANDARD: zstandard.level=7, zstandard.bufferPool.enabled=true 5479 5484 7 0.0 54790.8 0.9X
-ZSTANDARD: zstandard.level=9 6233 6347 161 0.0 62331.1 0.8X
-ZSTANDARD: zstandard.level=9, zstandard.bufferPool.enabled=true 6266 6319 74 0.0 62663.4 0.8X
+ZSTANDARD: zstandard.level=1 4732 4823 128 0.0 47324.6 1.0X
+ZSTANDARD: zstandard.level=1, zstandard.bufferPool.enabled=true 4695 4725 43 0.0 46945.7 1.0X
+ZSTANDARD: zstandard.level=3 4807 4849 60 0.0 48066.2 1.0X
+ZSTANDARD: zstandard.level=3, zstandard.bufferPool.enabled=true 4751 4760 13 0.0 47505.5 1.0X
+ZSTANDARD: zstandard.level=5 5075 5092 25 0.0 50749.7 0.9X
+ZSTANDARD: zstandard.level=5, zstandard.bufferPool.enabled=true 5015 5024 13 0.0 50154.6 0.9X
+ZSTANDARD: zstandard.level=7 5348 5372 34 0.0 53484.2 0.9X
+ZSTANDARD: zstandard.level=7, zstandard.bufferPool.enabled=true 5282 5293 14 0.0 52824.0 0.9X
+ZSTANDARD: zstandard.level=9 6270 6275 7 0.0 62703.9 0.8X
+ZSTANDARD: zstandard.level=9, zstandard.bufferPool.enabled=true 6082 6095 19 0.0 60819.1 0.8X
diff --git a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 6f345e069ff78..0b3823ca16160 100644
--- a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -32,7 +32,6 @@ import org.apache.avro.Schema.Type._
import org.apache.avro.file.{DataFileReader, DataFileWriter}
import org.apache.avro.generic.{GenericData, GenericDatumReader, GenericDatumWriter, GenericRecord}
import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed}
-import org.apache.commons.io.FileUtils
import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf, SparkException, SparkRuntimeException, SparkThrowable, SparkUpgradeException}
import org.apache.spark.TestUtils.assertExceptionMsg
@@ -640,7 +639,7 @@ abstract class AvroSuite
private def createDummyCorruptFile(dir: File): Unit = {
Utils.tryWithResource {
- FileUtils.forceMkdir(dir)
+ Files.createDirectories(dir.toPath)
val corruptFile = new File(dir, "corrupt.avro")
new BufferedWriter(new FileWriter(corruptFile))
} { writer =>
@@ -760,12 +759,12 @@ abstract class AvroSuite
spark.conf.set(SQLConf.AVRO_COMPRESSION_CODEC.key, ZSTANDARD.lowerCaseName())
df.write.format("avro").save(zstandardDir)
- val uncompressSize = FileUtils.sizeOfDirectory(new File(uncompressDir))
- val bzip2Size = FileUtils.sizeOfDirectory(new File(bzip2Dir))
- val xzSize = FileUtils.sizeOfDirectory(new File(xzDir))
- val deflateSize = FileUtils.sizeOfDirectory(new File(deflateDir))
- val snappySize = FileUtils.sizeOfDirectory(new File(snappyDir))
- val zstandardSize = FileUtils.sizeOfDirectory(new File(zstandardDir))
+ val uncompressSize = Utils.sizeOf(new File(uncompressDir))
+ val bzip2Size = Utils.sizeOf(new File(bzip2Dir))
+ val xzSize = Utils.sizeOf(new File(xzDir))
+ val deflateSize = Utils.sizeOf(new File(deflateDir))
+ val snappySize = Utils.sizeOf(new File(snappyDir))
+ val zstandardSize = Utils.sizeOf(new File(zstandardDir))
assert(uncompressSize > deflateSize)
assert(snappySize > deflateSize)
@@ -1875,7 +1874,7 @@ abstract class AvroSuite
intercept[FileNotFoundException] {
withTempDir { dir =>
- FileUtils.touch(new File(dir, "test"))
+ Utils.touch(new File(dir, "test"))
withSQLConf(AvroFileFormat.IgnoreFilesWithoutExtensionProperty -> "true") {
spark.read.format("avro").load(dir.toString)
}
@@ -1884,7 +1883,7 @@ abstract class AvroSuite
intercept[FileNotFoundException] {
withTempDir { dir =>
- FileUtils.touch(new File(dir, "test"))
+ Utils.touch(new File(dir, "test"))
spark
.read
@@ -1899,7 +1898,7 @@ abstract class AvroSuite
withTempPath { tempDir =>
val tempEmptyDir = s"$tempDir/sqlOverwrite"
// Create a temp directory for table that will be overwritten
- new File(tempEmptyDir).mkdirs()
+ Utils.createDirectory(tempEmptyDir)
spark.sql(
s"""
|CREATE TEMPORARY VIEW episodes
@@ -3095,6 +3094,26 @@ abstract class AvroSuite
assert(AvroOptions.isValidOption("recursiveFieldMaxDepth"))
}
+ test("SPARK-53973: boolean Avro options reject non-boolean values") {
+ Seq(
+ AvroOptions.STABLE_ID_FOR_UNION_TYPE,
+ AvroOptions.POSITIONAL_FIELD_MATCHING,
+ AvroOptions.IGNORE_EXTENSION
+ ).foreach { opt =>
+ val e = intercept[AnalysisException] {
+ AvroOptions(Map(opt -> "not_bool"))
+ }
+ checkError(
+ exception = e,
+ condition = "STDS_INVALID_OPTION_VALUE.WITH_MESSAGE",
+ parameters = Map(
+ "optionName" -> opt,
+ "message" -> "Cannot cast value 'not_bool' to Boolean."
+ )
+ )
+ }
+ }
+
test("SPARK-46633: read file with empty blocks") {
for (maxPartitionBytes <- Seq(100, 100000, 100000000)) {
withSQLConf(SQLConf.FILES_MAX_PARTITION_BYTES.key -> s"$maxPartitionBytes") {
diff --git a/connector/docker-integration-tests/README.md b/connector/docker-integration-tests/README.md
index 385218097a3cb..0e77a2816c3aa 100644
--- a/connector/docker-integration-tests/README.md
+++ b/connector/docker-integration-tests/README.md
@@ -45,7 +45,7 @@ the container bootstrapping. To run an individual Docker integration test, use t
Besides the default Docker images, the integration tests can be run with custom Docker images. For example,
- ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-free:23.7-slim ./build/sbt -Pdocker-integration-tests "docker-integration-tests/testOnly *OracleIntegrationSuite"
+ ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-free:23.9-slim ./build/sbt -Pdocker-integration-tests "docker-integration-tests/testOnly *OracleIntegrationSuite"
The following environment variables can be used to specify the custom Docker images for different databases:
diff --git a/connector/docker-integration-tests/pom.xml b/connector/docker-integration-tests/pom.xml
index 4fec4a6d82c32..09fabdf64dba8 100644
--- a/connector/docker-integration-tests/pom.xml
+++ b/connector/docker-integration-tests/pom.xml
@@ -39,7 +39,6 @@
com.google.guava
guava
- 33.1.0-jre
test
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
index 25695f144f558..c2d83943e3222 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
@@ -37,7 +37,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class DB2IntegrationSuite extends DockerJDBCIntegrationSuite with SharedJDBCIntegrationTests {
+class DB2IntegrationSuite extends SharedJDBCIntegrationSuite {
override val db = new DB2DatabaseOnDocker
override def dataPreparation(conn: Connection): Unit = {
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
index 1df01bd3bfb62..0062b03575930 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
@@ -37,7 +37,6 @@ import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
import org.scalatest.time.SpanSugar._
import org.apache.spark.internal.LogKeys.{CLASS_NAME, CONTAINER, STATUS}
-import org.apache.spark.internal.MDC
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.util.{DockerUtils, Utils}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala
index 53e45baa99679..4f782bed77922 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala
@@ -33,8 +33,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class MariaDBKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite
- with SharedJDBCIntegrationTests {
+class MariaDBKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
override protected val userName = s"mariadb/$dockerIp"
override protected val keytabFileName = "mariadb.keytab"
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
index f72f3ecca6929..0950fd7330c5a 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
@@ -40,8 +40,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite
- with SharedJDBCIntegrationTests {
+class MsSqlServerIntegrationSuite extends SharedJDBCIntegrationSuite {
override val db = new MsSQLServerDatabaseOnDocker
override def dataPreparation(conn: Connection): Unit = {
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
index 7244c5547abb3..f74a24c6319da 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
@@ -39,7 +39,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite {
+class MySQLIntegrationSuite extends SharedJDBCIntegrationSuite {
override val db = new MySQLDatabaseOnDocker
override def dataPreparation(conn: Connection): Unit = {
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala
index 8656d23a2e6a0..e32563c7e94bd 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala
@@ -21,7 +21,7 @@ import org.apache.spark.internal.Logging
class OracleDatabaseOnDocker extends DatabaseOnDocker with Logging {
lazy override val imageName =
- sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.7-slim")
+ sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.9-slim")
val oracle_password = "Th1s1sThe0racle#Pass"
override val env = Map(
"ORACLE_PWD" -> oracle_password, // oracle images uses this
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
index 33a89f4f20c4c..d44e2d1e748fd 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
@@ -61,9 +61,8 @@ import org.apache.spark.tags.DockerTest
* and with Oracle Express Edition versions 18.4.0 and 21.4.0
*/
@DockerTest
-class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
- with SharedSparkSession
- with SharedJDBCIntegrationTests {
+class OracleIntegrationSuite extends SharedJDBCIntegrationSuite
+ with SharedSparkSession {
import testImplicits._
override val db = new OracleDatabaseOnDocker
@@ -458,7 +457,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
.load()
df1.logicalPlan match {
- case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+ case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
assert(whereClauses === Set(
""""D" < '2018-07-11' or "D" is null""",
@@ -481,7 +480,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
.load()
df2.logicalPlan match {
- case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+ case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
assert(whereClauses === Set(
""""T" < '2018-07-15 20:50:32.5' or "T" is null""",
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
index 9d1186463ca2a..050f74f369034 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
@@ -40,7 +40,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite with SharedJDBCIntegrationTests {
+class PostgresIntegrationSuite extends SharedJDBCIntegrationSuite {
override val db = new PostgresDatabaseOnDocker
override def dataPreparation(conn: Connection): Unit = {
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala
new file mode 100644
index 0000000000000..05338e79998c6
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.sql.Connection
+
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.Row
+
+abstract class SharedJDBCIntegrationSuite extends DockerJDBCIntegrationSuite {
+
+ override def beforeAll(): Unit = runIfTestsEnabled(s"Prepare for ${this.getClass.getName}") {
+ super.beforeAll()
+ var conn: Connection = null
+ eventually(connectionTimeout, interval(1.second)) {
+ conn = getConnection()
+ }
+ try {
+ createSharedTable(conn)
+ } finally {
+ conn.close()
+ }
+ }
+
+ /**
+ * Create a table with the same name that can be used to test common functionality
+ * in
+ * @param conn
+ */
+ def createSharedTable(conn: Connection): Unit = {
+ val batchStmt = conn.createStatement()
+
+ batchStmt.addBatch("CREATE TABLE tbl_shared (x INTEGER)")
+ batchStmt.addBatch("INSERT INTO tbl_shared VALUES(1)")
+
+ batchStmt.executeBatch()
+ batchStmt.close()
+ }
+
+ test("SPARK-52184: Wrap external engine syntax error") {
+ val ex = intercept[SparkException] {
+ spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", "THIS IS NOT VALID SQL").load()
+ }
+
+ // Exception should be detected in analysis phase first when we resolve a schema from
+ // through JDBC by sending SELECT * FROM () [LIMIT 1][WHERE 1=0] query.
+ checkErrorMatchPVals(
+ ex,
+ condition = "JDBC_EXTERNAL_ENGINE_SYNTAX_ERROR.DURING_OUTPUT_SCHEMA_RESOLUTION",
+ parameters = Map(
+ "jdbcQuery" -> "SELECT \\* FROM \\(.*",
+ "externalEngineError" -> "[\\s\\S]*"
+ )
+ )
+ }
+
+ test("SPARK-53386: Parameter `query` should work when ending with semicolons") {
+ val dfSingle = spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", "SELECT x FROM tbl_shared; ")
+ .load()
+ checkAnswer(dfSingle, Seq(Row(1)))
+
+ val dfMultiple = spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", "SELECT x FROM tbl_shared;;;")
+ .load()
+ checkAnswer(dfMultiple, Seq(Row(1)))
+ }
+}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala
index a693920c10032..c959aae8e91e6 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala
@@ -14,9 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.jdbc
+package org.apache.spark.sql.jdbc.querytest
import java.io.File
+import java.nio.file.Files
import java.sql.ResultSet
import scala.collection.mutable.ArrayBuffer
@@ -24,7 +25,7 @@ import scala.util.control.NonFatal
import org.apache.spark.sql.Row
import org.apache.spark.sql.SQLQueryTestHelper
-import org.apache.spark.sql.catalyst.util.fileToString
+import org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite
/**
* This suite builds off of that to allow us to run other DBMS against the SQL test golden files (on
@@ -76,7 +77,7 @@ trait CrossDbmsQueryTestSuite extends DockerJDBCIntegrationSuite with SQLQueryTe
}
protected def runSqlTestCase(testCase: TestCase, listTestCases: Seq[TestCase]): Unit = {
- val input = fileToString(new File(testCase.inputFile))
+ val input = Files.readString(new File(testCase.inputFile).toPath)
val (comments, code) = splitCommentsAndCodes(input)
val queries = getQueries(code, comments, listTestCases)
@@ -143,7 +144,7 @@ trait CrossDbmsQueryTestSuite extends DockerJDBCIntegrationSuite with SQLQueryTe
// Read back the golden files.
var curSegment = 0
val expectedOutputs: Seq[QueryTestOutput] = {
- val goldenOutput = fileToString(new File(testCase.resultFile))
+ val goldenOutput = Files.readString(new File(testCase.resultFile).toPath)
val segments = goldenOutput.split("-- !query.*\n")
outputs.map { output =>
val result =
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala
index 3a1d5e18b7e5a..eabdb033ebc24 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.jdbc
+package org.apache.spark.sql.jdbc.querytest
import java.sql.{Connection, ResultSet, Statement}
import java.util.Locale
@@ -24,6 +24,7 @@ import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.{QueryGeneratorHelper, QueryTest}
import org.apache.spark.sql.Row
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, PostgresDatabaseOnDocker}
import org.apache.spark.tags.DockerTest
/**
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala
index 28320a9e0a949..4b7fc0c3e20cd 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala
@@ -14,11 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.jdbc
+package org.apache.spark.sql.jdbc.querytest
import java.io.File
import java.sql.Connection
+import org.apache.spark.sql.jdbc.PostgresDatabaseOnDocker
import org.apache.spark.tags.DockerTest
/**
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DockerJDBCIntegrationV2Suite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DockerJDBCIntegrationV2Suite.scala
index 60345257f2dc4..2a66845f46ed5 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DockerJDBCIntegrationV2Suite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DockerJDBCIntegrationV2Suite.scala
@@ -19,10 +19,17 @@ package org.apache.spark.sql.jdbc.v2
import java.sql.Connection
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite
abstract class DockerJDBCIntegrationV2Suite extends DockerJDBCIntegrationSuite {
+ override def sparkConf: SparkConf = super.sparkConf
+ // DS V2 relies on ANSI mode to translate expressions, we should always
+ // run JDBC v2 tests with ANSI on.
+ .set(SQLConf.ANSI_ENABLED, true)
+
/**
* Prepare databases and tables for testing.
*/
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala
index 7a58fca17970f..c71f9ae7688f3 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala
@@ -118,11 +118,28 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTes
"CREATE TABLE employee (dept NUMBER(32), name VARCHAR2(32), salary NUMBER(20, 2)," +
" bonus BINARY_DOUBLE)").executeUpdate()
connection.prepareStatement(
- s"""CREATE TABLE pattern_testing_table (
- |pattern_testing_col VARCHAR(50)
- |)
- """.stripMargin
+ """CREATE TABLE pattern_testing_table (
+ |pattern_testing_col VARCHAR(50)
+ |)
+ """.stripMargin
).executeUpdate()
+ connection.prepareStatement(
+ "CREATE TABLE datetime (name VARCHAR(32), date1 DATE, time1 TIMESTAMP)")
+ .executeUpdate()
+ }
+
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation(connection)
+ connection.prepareStatement(
+ "INSERT INTO datetime VALUES ('amy', TO_DATE('2022-05-19', 'YYYY-MM-DD')," +
+ " TO_TIMESTAMP('2022-05-19 00:00:00', 'YYYY-MM-DD HH24:MI:SS'))").executeUpdate()
+ connection.prepareStatement(
+ "INSERT INTO datetime VALUES ('alex', TO_DATE('2022-05-18', 'YYYY-MM-DD')," +
+ " TO_TIMESTAMP('2022-05-18 00:00:00', 'YYYY-MM-DD HH24:MI:SS'))").executeUpdate()
+ // '2022-01-01' is Saturday and is in ISO year 2021.
+ connection.prepareStatement(
+ "INSERT INTO datetime VALUES ('tom', TO_DATE('2022-01-01', 'YYYY-MM-DD')," +
+ " TO_TIMESTAMP('2022-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS'))").executeUpdate()
}
override def testUpdateColumnType(tbl: String): Unit = {
@@ -185,4 +202,143 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTes
checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row("Eason", "Y ")))
}
}
+
+ override def testDatetime(tbl: String): Unit = {
+ val df1 = sql(s"SELECT name FROM $tbl WHERE " +
+ "dayofyear(date1) > 100 AND dayofmonth(date1) > 10 ")
+ checkFilterPushed(df1, false)
+ val rows1 = df1.collect()
+ assert(rows1.length === 2)
+ assert(rows1(0).getString(0) === "amy")
+ assert(rows1(1).getString(0) === "alex")
+
+ val df2 = sql(s"SELECT name FROM $tbl WHERE year(date1) = 2022 AND quarter(date1) = 2")
+ checkFilterPushed(df2, false)
+ val rows2 = df2.collect()
+ assert(rows2.length === 2)
+ assert(rows2(0).getString(0) === "amy")
+ assert(rows2(1).getString(0) === "alex")
+
+ val df3 = sql(s"SELECT name FROM $tbl WHERE month(date1) = 5")
+ checkFilterPushed(df3)
+ val rows3 = df3.collect()
+ assert(rows3.length === 2)
+ assert(rows3(0).getString(0) === "amy")
+ assert(rows3(1).getString(0) === "alex")
+
+ val df4 = sql(s"SELECT name FROM $tbl WHERE hour(time1) = 0 AND minute(time1) = 0")
+ checkFilterPushed(df4)
+ val rows4 = df4.collect()
+ assert(rows4.length === 3)
+ assert(rows4(0).getString(0) === "amy")
+ assert(rows4(1).getString(0) === "alex")
+ assert(rows4(2).getString(0) === "tom")
+
+ val df5 = sql(s"SELECT name FROM $tbl WHERE " +
+ "extract(WEEK from date1) > 10 AND extract(YEAR from date1) = 2022")
+ checkFilterPushed(df5, false)
+ val rows5 = df5.collect()
+ assert(rows5.length === 3)
+ assert(rows5(0).getString(0) === "amy")
+ assert(rows5(1).getString(0) === "alex")
+ assert(rows5(2).getString(0) === "tom")
+
+ val df6 = sql(s"SELECT name FROM $tbl WHERE date_add(date1, 1) = date'2022-05-20' " +
+ "AND datediff(date1, '2022-05-10') > 0")
+ checkFilterPushed(df6, false)
+ val rows6 = df6.collect()
+ assert(rows6.length === 1)
+ assert(rows6(0).getString(0) === "amy")
+
+ val df7 = sql(s"SELECT name FROM $tbl WHERE weekday(date1) = 2")
+ checkFilterPushed(df7, false)
+ val rows7 = df7.collect()
+ assert(rows7.length === 1)
+ assert(rows7(0).getString(0) === "alex")
+
+ withClue("weekofyear") {
+ val woy = sql(s"SELECT weekofyear(date1) FROM $tbl WHERE name = 'tom'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE weekofyear(date1) = $woy")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "tom")
+ }
+
+ withClue("dayofweek") {
+ val dow = sql(s"SELECT dayofweek(date1) FROM $tbl WHERE name = 'alex'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE dayofweek(date1) = $dow")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "alex")
+ }
+
+ withClue("yearofweek") {
+ val yow = sql(s"SELECT extract(YEAROFWEEK from date1) FROM $tbl WHERE name = 'tom'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE extract(YEAROFWEEK from date1) = $yow")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "tom")
+ }
+
+ withClue("dayofyear") {
+ val doy = sql(s"SELECT dayofyear(date1) FROM $tbl WHERE name = 'amy'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE dayofyear(date1) = $doy")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "amy")
+ }
+
+ withClue("dayofmonth") {
+ val dom = sql(s"SELECT dayofmonth(date1) FROM $tbl WHERE name = 'amy'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE dayofmonth(date1) = $dom")
+ checkFilterPushed(df)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "amy")
+ }
+
+ withClue("year") {
+ val year = sql(s"SELECT year(date1) FROM $tbl WHERE name = 'amy'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE year(date1) = $year")
+ checkFilterPushed(df)
+ val rows = df.collect()
+ assert(rows.length === 3)
+ assert(rows(0).getString(0) === "amy")
+ assert(rows5(1).getString(0) === "alex")
+ assert(rows5(2).getString(0) === "tom")
+ }
+
+ withClue("second") {
+ val df = sql(s"SELECT name FROM $tbl WHERE second(time1) = 0 AND month(date1) = 5")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 2)
+ assert(rows(0).getString(0) === "amy")
+ assert(rows(1).getString(0) === "alex")
+ }
+
+ val df9 = sql(s"SELECT name FROM $tbl WHERE " +
+ "dayofyear(date1) > 100 order by dayofyear(date1) limit 1")
+ checkFilterPushed(df9, false)
+ val rows9 = df9.collect()
+ assert(rows9.length === 1)
+ assert(rows9(0).getString(0) === "alex")
+
+ val df10 = sql(s"SELECT name FROM $tbl WHERE trunc(date1, 'week') = date'2022-05-16'")
+ checkFilterPushed(df10)
+ val rows10 = df10.collect()
+ assert(rows10.length === 2)
+ assert(rows10(0).getString(0) === "amy")
+ assert(rows10(1).getString(0) === "alex")
+ }
}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala
index ee4a479960145..fb18dca97994e 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala
@@ -22,20 +22,21 @@ import org.apache.logging.log4j.Level
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{AnalysisException, DataFrame}
import org.apache.spark.sql.catalyst.analysis.{IndexAlreadyExistsException, NoSuchIndexException}
-import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, GlobalLimit, LocalLimit, Offset, Sample, Sort}
+import org.apache.spark.sql.connector.DataSourcePushdownTestUtils
import org.apache.spark.sql.connector.catalog.{Catalogs, Identifier, TableCatalog}
import org.apache.spark.sql.connector.catalog.index.SupportsIndex
import org.apache.spark.sql.connector.expressions.NullOrdering
-import org.apache.spark.sql.connector.expressions.aggregate.GeneralAggregateFunc
import org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD
-import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanRelation, V1ScanWrapper}
import org.apache.spark.sql.jdbc.DockerIntegrationFunSuite
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types._
import org.apache.spark.tags.DockerTest
@DockerTest
-private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFunSuite {
+private[v2] trait V2JDBCTest
+ extends DataSourcePushdownTestUtils
+ with DockerIntegrationFunSuite
+ with SharedSparkSession {
import testImplicits._
val catalogName: String
@@ -468,56 +469,6 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu
def supportsTableSample: Boolean = false
- private def checkSamplePushed(df: DataFrame, pushed: Boolean = true): Unit = {
- val sample = df.queryExecution.optimizedPlan.collect {
- case s: Sample => s
- }
- if (pushed) {
- assert(sample.isEmpty)
- } else {
- assert(sample.nonEmpty)
- }
- }
-
- protected def checkFilterPushed(df: DataFrame, pushed: Boolean = true): Unit = {
- val filter = df.queryExecution.optimizedPlan.collect {
- case f: Filter => f
- }
- if (pushed) {
- assert(filter.isEmpty)
- } else {
- assert(filter.nonEmpty)
- }
- }
-
- protected def checkLimitRemoved(df: DataFrame, pushed: Boolean = true): Unit = {
- val limit = df.queryExecution.optimizedPlan.collect {
- case l: LocalLimit => l
- case g: GlobalLimit => g
- }
- if (pushed) {
- assert(limit.isEmpty)
- } else {
- assert(limit.nonEmpty)
- }
- }
-
- private def checkLimitPushed(df: DataFrame, limit: Option[Int]): Unit = {
- df.queryExecution.optimizedPlan.collect {
- case relation: DataSourceV2ScanRelation => relation.scan match {
- case v1: V1ScanWrapper =>
- assert(v1.pushedDownOperators.limit == limit)
- }
- }
- }
-
- private def checkColumnPruned(df: DataFrame, col: String): Unit = {
- val scan = df.queryExecution.optimizedPlan.collectFirst {
- case s: DataSourceV2ScanRelation => s
- }.get
- assert(scan.schema.names.sameElements(Seq(col)))
- }
-
test("SPARK-48172: Test CONTAINS") {
val df1 = spark.sql(
s"""
@@ -841,39 +792,6 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu
}
}
- private def checkSortRemoved(df: DataFrame, pushed: Boolean = true): Unit = {
- val sorts = df.queryExecution.optimizedPlan.collect {
- case s: Sort => s
- }
-
- if (pushed) {
- assert(sorts.isEmpty)
- } else {
- assert(sorts.nonEmpty)
- }
- }
-
- private def checkOffsetRemoved(df: DataFrame, pushed: Boolean = true): Unit = {
- val offsets = df.queryExecution.optimizedPlan.collect {
- case o: Offset => o
- }
-
- if (pushed) {
- assert(offsets.isEmpty)
- } else {
- assert(offsets.nonEmpty)
- }
- }
-
- private def checkOffsetPushed(df: DataFrame, offset: Option[Int]): Unit = {
- df.queryExecution.optimizedPlan.collect {
- case relation: DataSourceV2ScanRelation => relation.scan match {
- case v1: V1ScanWrapper =>
- assert(v1.pushedDownOperators.offset == offset)
- }
- }
- }
-
gridTest("simple scan")(partitioningEnabledTestCase) { partitioningEnabled =>
val (tableOptions, partitionInfo) = getTableOptions("employee", partitioningEnabled)
val df = sql(s"SELECT name, salary, bonus FROM $catalogAndNamespace." +
@@ -1028,27 +946,6 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu
}
}
- private def checkAggregateRemoved(df: DataFrame): Unit = {
- val aggregates = df.queryExecution.optimizedPlan.collect {
- case agg: Aggregate => agg
- }
- assert(aggregates.isEmpty)
- }
-
- private def checkAggregatePushed(df: DataFrame, funcName: String): Unit = {
- df.queryExecution.optimizedPlan.collect {
- case DataSourceV2ScanRelation(_, scan, _, _, _) =>
- assert(scan.isInstanceOf[V1ScanWrapper])
- val wrapper = scan.asInstanceOf[V1ScanWrapper]
- assert(wrapper.pushedDownOperators.aggregation.isDefined)
- val aggregationExpressions =
- wrapper.pushedDownOperators.aggregation.get.aggregateExpressions()
- assert(aggregationExpressions.length == 1)
- assert(aggregationExpressions(0).isInstanceOf[GeneralAggregateFunc])
- assert(aggregationExpressions(0).asInstanceOf[GeneralAggregateFunc].name() == funcName)
- }
- }
-
protected def caseConvert(tableName: String): String = tableName
Seq(true, false).foreach { isDistinct =>
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MsSqlServerJoinPushdownIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MsSqlServerJoinPushdownIntegrationSuite.scala
new file mode 100644
index 0000000000000..d8f6fa4b1b44e
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MsSqlServerJoinPushdownIntegrationSuite.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2.join
+
+import java.sql.Connection
+import java.util.Locale
+
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, JdbcDialect, MsSQLServerDatabaseOnDocker, MsSqlServerDialect}
+import org.apache.spark.sql.jdbc.v2.JDBCV2JoinPushdownIntegrationSuiteBase
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., 2022-CU15-ubuntu-22.04):
+ * {{{
+ * ENABLE_DOCKER_INTEGRATION_TESTS=1
+ * MSSQLSERVER_DOCKER_IMAGE_NAME=mcr.microsoft.com/mssql/server:2022-CU15-ubuntu-22.04
+ * ./build/sbt -Pdocker-integration-tests "testOnly *v2*MsSqlServerIntegrationSuite"
+ * }}}
+ */
+@DockerTest
+class MsSqlServerJoinPushdownIntegrationSuite
+ extends DockerJDBCIntegrationSuite
+ with JDBCV2JoinPushdownIntegrationSuiteBase {
+ override val db = new MsSQLServerDatabaseOnDocker
+
+ override val url = db.getJdbcUrl(dockerIp, externalPort)
+
+ override val jdbcDialect: JdbcDialect = MsSqlServerDialect()
+
+ override def caseConvert(identifier: String): String = identifier.toUpperCase(Locale.ROOT)
+
+ // This method comes from DockerJDBCIntegrationSuite
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation()
+ }
+}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MySQLJoinPushdownIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MySQLJoinPushdownIntegrationSuite.scala
new file mode 100644
index 0000000000000..1a1536b6a2929
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MySQLJoinPushdownIntegrationSuite.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2.join
+
+import java.sql.Connection
+import java.util.Locale
+
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, JdbcDialect, MySQLDatabaseOnDocker, MySQLDialect}
+import org.apache.spark.sql.jdbc.v2.JDBCV2JoinPushdownIntegrationSuiteBase
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., mysql:9.2.0):
+ * {{{
+ * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:9.2.0
+ * ./build/sbt -Pdocker-integration-tests "testOnly *v2*MySQLIntegrationSuite"
+ * }}}
+ */
+@DockerTest
+class MySQLJoinPushdownIntegrationSuite
+ extends DockerJDBCIntegrationSuite
+ with JDBCV2JoinPushdownIntegrationSuiteBase {
+ override val db = new MySQLDatabaseOnDocker
+
+ override val url = db.getJdbcUrl(dockerIp, externalPort)
+
+ override val jdbcDialect: JdbcDialect = MySQLDialect()
+
+ override def caseConvert(identifier: String): String = identifier.toUpperCase(Locale.ROOT)
+
+ override def remainColumnCase(identifier: String): String = "`" + identifier + "`"
+
+ // This method comes from DockerJDBCIntegrationSuite
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation()
+ }
+}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/OracleJoinPushdownIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/OracleJoinPushdownIntegrationSuite.scala
new file mode 100644
index 0000000000000..ad9e5d002bf50
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/OracleJoinPushdownIntegrationSuite.scala
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2.join
+
+import java.sql.Connection
+import java.util.Locale
+
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, JdbcDialect, OracleDatabaseOnDocker, OracleDialect}
+import org.apache.spark.sql.jdbc.v2.JDBCV2JoinPushdownIntegrationSuiteBase
+import org.apache.spark.sql.types.DataTypes
+import org.apache.spark.tags.DockerTest
+
+/**
+ * The following are the steps to test this:
+ *
+ * 1. Choose to use a prebuilt image or build Oracle database in a container
+ * - The documentation on how to build Oracle RDBMS in a container is at
+ * https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md
+ * - Official Oracle container images can be found at https://container-registry.oracle.com
+ * - Trustable and streamlined Oracle Database Free images can be found on Docker Hub at
+ * https://hub.docker.com/r/gvenzl/oracle-free
+ * see also https://github.com/gvenzl/oci-oracle-free
+ * 2. Run: export ORACLE_DOCKER_IMAGE_NAME=image_you_want_to_use_for_testing
+ * - Example: export ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-free:latest
+ * 3. Run: export ENABLE_DOCKER_INTEGRATION_TESTS=1
+ * 4. Start docker: sudo service docker start
+ * - Optionally, docker pull $ORACLE_DOCKER_IMAGE_NAME
+ * 5. Run Spark integration tests for Oracle with: ./build/sbt -Pdocker-integration-tests
+ * "testOnly org.apache.spark.sql.jdbc.v2.OracleIntegrationSuite"
+ *
+ * A sequence of commands to build the Oracle Database Free container image:
+ * $ git clone https://github.com/oracle/docker-images.git
+ * $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles0
+ * $ ./buildContainerImage.sh -v 23.4.0 -f
+ * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:23.4.0-free
+ *
+ * This procedure has been validated with Oracle Database Free version 23.4.0,
+ * and with Oracle Express Edition versions 18.4.0 and 21.4.0
+ */
+@DockerTest
+class OracleJoinPushdownIntegrationSuite
+ extends DockerJDBCIntegrationSuite
+ with JDBCV2JoinPushdownIntegrationSuiteBase {
+ override def excluded: Seq[String] = Seq(
+ // Following tests are harder to be supported for Oracle because Oracle connector does
+ // casts in predicates. There is a separate test in this suite that is similar to
+ // "Test explain formatted" test from base suite.
+ "Test self join with condition",
+ "Test multi-way self join with conditions",
+ "Test explain formatted"
+ )
+
+ override val namespace: String = "SYSTEM"
+
+ override val db = new OracleDatabaseOnDocker
+
+ override val url = db.getJdbcUrl(dockerIp, externalPort)
+
+ override val jdbcDialect: JdbcDialect = OracleDialect()
+
+ override val integerType = DataTypes.createDecimalType(10, 0)
+
+ override def caseConvert(identifier: String): String = identifier.toUpperCase(Locale.ROOT)
+
+ override def schemaPreparation(): Unit = {}
+
+ // This method comes from DockerJDBCIntegrationSuite
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation()
+ }
+
+ test("Test explain formatted - Oracle compatible") {
+ val sqlQuery =
+ s"""
+ |SELECT * FROM $catalogAndNamespace.$casedJoinTableName1 a
+ |JOIN $catalogAndNamespace.$casedJoinTableName2 b
+ |ON a.id = b.id + 1
+ |JOIN $catalogAndNamespace.$casedJoinTableName3 c
+ |ON b.id = c.id + 1
+ |JOIN $catalogAndNamespace.$casedJoinTableName4 d
+ |ON c.id = d.id + 1
+ |""".stripMargin
+
+ withSQLConf(SQLConf.DATA_SOURCE_V2_JOIN_PUSHDOWN.key -> "true") {
+ val df = sql(sqlQuery)
+
+ // scalastyle:off line.size.limit
+ checkJoinPushed(
+ df,
+ s"""PushedFilters: [CAST(id_3 AS decimal(11,0)) = (id_4 + 1)], PushedJoins:\u0020
+ |[L]: PushedFilters: [CAST(ID_1 AS decimal(11,0)) = (id_3 + 1)]
+ | PushedJoins:
+ | [L]: PushedFilters: [CAST(ID AS decimal(11,0)) = (ID_1 + 1)]
+ | PushedJoins:
+ | [L]: Relation: $catalogAndNamespace.${caseConvert(joinTableName1)}
+ | PushedFilters: [${caseConvert("id")} IS NOT NULL]
+ | [R]: Relation: $catalogAndNamespace.${caseConvert(joinTableName2)}
+ | PushedFilters: [${caseConvert("id")} IS NOT NULL]
+ | [R]: Relation: $catalogAndNamespace.${caseConvert(joinTableName3)}
+ | PushedFilters: [id IS NOT NULL]
+ |[R]: Relation: $catalogAndNamespace.${caseConvert(joinTableName4)}
+ | PushedFilters: [id IS NOT NULL]""".stripMargin
+ )
+ // scalastyle:on line.size.limit
+ }
+ }
+}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/PostgresJoinPushdownIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/PostgresJoinPushdownIntegrationSuite.scala
new file mode 100644
index 0000000000000..e515eb698f60c
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/PostgresJoinPushdownIntegrationSuite.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2.join
+
+import java.sql.Connection
+
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, JdbcDialect, PostgresDatabaseOnDocker, PostgresDialect}
+import org.apache.spark.sql.jdbc.v2.JDBCV2JoinPushdownIntegrationSuiteBase
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., postgres:17.2-alpine)
+ * {{{
+ * ENABLE_DOCKER_INTEGRATION_TESTS=1 POSTGRES_DOCKER_IMAGE_NAME=postgres:17.2-alpine
+ * ./build/sbt -Pdocker-integration-tests "testOnly *v2.PostgresIntegrationSuite"
+ * }}}
+ */
+@DockerTest
+class PostgresJoinPushdownIntegrationSuite
+ extends DockerJDBCIntegrationSuite
+ with JDBCV2JoinPushdownIntegrationSuiteBase {
+ override val db = new PostgresDatabaseOnDocker
+
+ override val url = db.getJdbcUrl(dockerIp, externalPort)
+
+ override val jdbcDialect: JdbcDialect = PostgresDialect()
+
+ // This method comes from DockerJDBCIntegrationSuite
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation()
+ }
+}
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/ConsumerStrategy.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/ConsumerStrategy.scala
index ab41e53d8ffbe..2d51d8d8c35d4 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/ConsumerStrategy.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/ConsumerStrategy.scala
@@ -62,14 +62,15 @@ private[kafka010] sealed trait ConsumerStrategy extends Logging {
.build()
protected def retrieveAllPartitions(admin: Admin, topics: Set[String]): Set[TopicPartition] = {
- admin.describeTopics(topics.asJava).all().get().asScala.filterNot(_._2.isInternal).flatMap {
- case (topic, topicDescription) =>
- topicDescription.partitions().asScala.map { topicPartitionInfo =>
- val partition = topicPartitionInfo.partition()
- logDebug(s"Partition found: $topic:$partition")
- new TopicPartition(topic, partition)
- }
- }.toSet
+ admin.describeTopics(topics.asJava).allTopicNames().get().asScala.filterNot(_._2.isInternal)
+ .flatMap {
+ case (topic, topicDescription) =>
+ topicDescription.partitions().asScala.map { topicPartitionInfo =>
+ val partition = topicPartitionInfo.partition()
+ logDebug(s"Partition found: $topic:$partition")
+ new TopicPartition(topic, partition)
+ }
+ }.toSet
}
}
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala
index cb1c7055483b3..02568aa89eb1d 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala
@@ -20,13 +20,13 @@ package org.apache.spark.sql.kafka010
import java.{util => ju}
import org.apache.spark.TaskContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
import org.apache.spark.sql.connector.metric.CustomTaskMetric
import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory}
-import org.apache.spark.sql.execution.streaming.{MicroBatchExecution, StreamExecution}
+import org.apache.spark.sql.execution.streaming.runtime.{MicroBatchExecution, StreamExecution}
import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer
/** A [[InputPartition]] for reading Kafka data in a batch based streaming query. */
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousStream.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousStream.scala
index 10bdbb1d9d447..041fe074f7e40 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousStream.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousStream.scala
@@ -24,7 +24,7 @@ import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord, Offset
import org.apache.kafka.common.TopicPartition
import org.apache.spark.TaskContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ERROR, OFFSETS, TIP}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
index c79da13017b97..7449e91230334 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import org.apache.kafka.common.TopicPartition
import org.apache.spark.SparkEnv
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ERROR, OFFSETS, TIP}
import org.apache.spark.internal.config.Network.NETWORK_TIMEOUT
import org.apache.spark.sql.SparkSession
@@ -91,6 +91,8 @@ private[kafka010] class KafkaMicroBatchStream(
private var allDataForTriggerAvailableNow: PartitionOffsetMap = _
+ private var isTriggerAvailableNow: Boolean = false
+
/**
* Lazily initialize `initialPartitionOffsets` to make sure that `KafkaConsumer.poll` is only
* called in StreamExecutionThread. Otherwise, interrupting a thread while running
@@ -126,8 +128,14 @@ private[kafka010] class KafkaMicroBatchStream(
val startPartitionOffsets = start.asInstanceOf[KafkaSourceOffset].partitionToOffsets
// Use the pre-fetched list of partition offsets when Trigger.AvailableNow is enabled.
- latestPartitionOffsets = if (allDataForTriggerAvailableNow != null) {
- allDataForTriggerAvailableNow
+ latestPartitionOffsets = if (isTriggerAvailableNow) {
+ if (allDataForTriggerAvailableNow != null) {
+ allDataForTriggerAvailableNow
+ } else {
+ allDataForTriggerAvailableNow =
+ kafkaOffsetReader.fetchLatestOffsets(Some(startPartitionOffsets))
+ allDataForTriggerAvailableNow
+ }
} else {
kafkaOffsetReader.fetchLatestOffsets(Some(startPartitionOffsets))
}
@@ -359,8 +367,7 @@ private[kafka010] class KafkaMicroBatchStream(
}
override def prepareForTriggerAvailableNow(): Unit = {
- allDataForTriggerAvailableNow = kafkaOffsetReader.fetchLatestOffsets(
- Some(getOrCreateInitialPartitionOffsets()))
+ isTriggerAvailableNow = true
}
}
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala
index 86cec6fc041cd..7420c2c1055b2 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala
@@ -30,7 +30,7 @@ import org.apache.kafka.common.{IsolationLevel, TopicPartition}
import org.apache.kafka.common.requests.OffsetFetchResponse
import org.apache.spark.SparkEnv
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NUM_RETRY, OFFSETS, TOPIC_PARTITION_OFFSET}
import org.apache.spark.scheduler.ExecutorCacheTaskLocation
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala
index fb06797d1fe34..b7ac9a171c575 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala
@@ -27,7 +27,7 @@ import org.apache.kafka.clients.consumer.{Consumer, ConsumerConfig, OffsetAndTim
import org.apache.kafka.common.TopicPartition
import org.apache.spark.SparkEnv
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NUM_RETRY, OFFSETS, TOPIC_PARTITION_OFFSET}
import org.apache.spark.scheduler.ExecutorCacheTaskLocation
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
index ff884310f660c..7e98446567640 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.kafka010
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.TOPIC_PARTITIONS
import org.apache.spark.internal.config.Network.NETWORK_TIMEOUT
import org.apache.spark.rdd.RDD
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSink.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSink.scala
index fb473e71d5a75..d3ea5332f4e44 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSink.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSink.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.kafka010
import java.{util => ju}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.BATCH_ID
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.execution.streaming.Sink
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
index 1b52046b14833..62cd20448453b 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
@@ -22,7 +22,7 @@ import java.{util => ju}
import org.apache.kafka.common.TopicPartition
import org.apache.spark.SparkContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ERROR, FROM_OFFSET, OFFSETS, TIP, TOPIC_PARTITIONS, UNTIL_OFFSET}
import org.apache.spark.internal.config.Network.NETWORK_TIMEOUT
import org.apache.spark.scheduler.ExecutorCacheTaskLocation
@@ -112,6 +112,8 @@ private[kafka010] class KafkaSource(
private var allDataForTriggerAvailableNow: PartitionOffsetMap = _
+ private var isTriggerAvailableNow = false
+
/**
* Lazily initialize `initialPartitionOffsets` to make sure that `KafkaConsumer.poll` is only
* called in StreamExecutionThread. Otherwise, interrupting a thread while running
@@ -175,8 +177,13 @@ private[kafka010] class KafkaSource(
val currentOffsets = currentPartitionOffsets.orElse(Some(initialPartitionOffsets))
// Use the pre-fetched list of partition offsets when Trigger.AvailableNow is enabled.
- val latest = if (allDataForTriggerAvailableNow != null) {
- allDataForTriggerAvailableNow
+ val latest = if (isTriggerAvailableNow) {
+ if (allDataForTriggerAvailableNow != null) {
+ allDataForTriggerAvailableNow
+ } else {
+ allDataForTriggerAvailableNow = kafkaReader.fetchLatestOffsets(currentOffsets)
+ allDataForTriggerAvailableNow
+ }
} else {
kafkaReader.fetchLatestOffsets(currentOffsets)
}
@@ -404,7 +411,7 @@ private[kafka010] class KafkaSource(
}
override def prepareForTriggerAvailableNow(): Unit = {
- allDataForTriggerAvailableNow = kafkaReader.fetchLatestOffsets(Some(initialPartitionOffsets))
+ isTriggerAvailableNow = true
}
}
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceInitialOffsetWriter.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceInitialOffsetWriter.scala
index f9b7e23df49b6..cef712fafd888 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceInitialOffsetWriter.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceInitialOffsetWriter.scala
@@ -20,10 +20,10 @@ package org.apache.spark.sql.kafka010
import java.io._
import java.nio.charset.StandardCharsets
-import org.apache.commons.io.IOUtils
-
import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.execution.streaming.{HDFSMetadataLog, SerializedOffset}
+import org.apache.spark.sql.execution.streaming.checkpointing.HDFSMetadataLog
+import org.apache.spark.sql.execution.streaming.runtime.SerializedOffset
+import org.apache.spark.util.Utils
/** A version of [[HDFSMetadataLog]] specialized for saving the initial offsets. */
private[kafka010] class KafkaSourceInitialOffsetWriter(
@@ -43,7 +43,7 @@ private[kafka010] class KafkaSourceInitialOffsetWriter(
override def deserialize(in: InputStream): KafkaSourceOffset = {
in.read() // A zero byte is read to support Spark 2.1.0 (SPARK-19517)
- val content = IOUtils.toString(new InputStreamReader(in, StandardCharsets.UTF_8))
+ val content = Utils.toString(in)
// HDFSMetadataLog guarantees that it never creates a partial file.
require(content.nonEmpty)
if (content(0) == 'v') {
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala
index 8790a77e5d895..d2c0c5d6fba85 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala
@@ -21,7 +21,8 @@ import org.apache.kafka.common.TopicPartition
import org.apache.spark.sql.connector.read.streaming
import org.apache.spark.sql.connector.read.streaming.PartitionOffset
-import org.apache.spark.sql.execution.streaming.{Offset, SerializedOffset}
+import org.apache.spark.sql.execution.streaming.Offset
+import org.apache.spark.sql.execution.streaming.runtime.SerializedOffset
/**
* An [[Offset]] for the [[KafkaSource]]. This one tracks all partitions of subscribed topics and
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
index 82ad75e028afe..ff2f16d26b932 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
@@ -27,7 +27,7 @@ import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer}
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.kafka010.KafkaConfigUpdater
import org.apache.spark.sql.{AnalysisException, DataFrame, SaveMode, SQLContext}
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
index 0f962a29588a0..9bae686a65bf8 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
@@ -23,7 +23,6 @@ import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.spark.{Partition, SparkContext, SparkException, TaskContext}
import org.apache.spark.internal.LogKeys.{FROM_OFFSET, PARTITION_ID, TOPIC}
-import org.apache.spark.internal.MDC
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer
import org.apache.spark.storage.StorageLevel
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala
index 9f68cb6fd0882..2ea1104177a54 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala
@@ -26,7 +26,7 @@ import scala.collection.mutable
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{DATA, KEY}
import org.apache.spark.sql.kafka010.{FETCHED_DATA_CACHE_EVICTOR_THREAD_RUN_INTERVAL, FETCHED_DATA_CACHE_TIMEOUT}
import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.{AvailableOffsetRange, CacheKey, UNKNOWN_OFFSET}
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
index ceb9d96660ae3..2d1125294df27 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
@@ -30,7 +30,7 @@ import org.apache.kafka.common.TopicPartition
import org.apache.spark.{SparkEnv, TaskContext}
import org.apache.spark.deploy.security.HadoopDelegationTokenManager
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.kafka010.{KafkaConfigUpdater, KafkaTokenUtil}
import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_MILLIS
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/CachedKafkaProducer.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/CachedKafkaProducer.scala
index c3457cf8982d9..9b40bcf8ce779 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/CachedKafkaProducer.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/CachedKafkaProducer.scala
@@ -23,7 +23,7 @@ import scala.util.control.NonFatal
import org.apache.kafka.clients.producer.KafkaProducer
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PRODUCER_ID
private[kafka010] class CachedKafkaProducer(
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/InternalKafkaProducerPool.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/InternalKafkaProducerPool.scala
index 79e0a91dd8968..ca9a0059ba1e4 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/InternalKafkaProducerPool.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/InternalKafkaProducerPool.scala
@@ -27,7 +27,7 @@ import scala.jdk.CollectionConverters._
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.spark.{SparkConf, SparkEnv}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PRODUCER_ID
import org.apache.spark.kafka010.{KafkaConfigUpdater, KafkaRedactionUtil}
import org.apache.spark.sql.kafka010.{PRODUCER_CACHE_EVICTOR_THREAD_RUN_INTERVAL, PRODUCER_CACHE_TIMEOUT}
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
index 4b6a5b899fc8e..2dff58ff87875 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
@@ -22,8 +22,8 @@ import java.util.concurrent.atomic.AtomicInteger
import org.apache.spark.SparkContext
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd, SparkListenerTaskStart}
import org.apache.spark.sql.execution.datasources.v2.ContinuousScanExec
-import org.apache.spark.sql.execution.streaming.StreamExecution
import org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution
+import org.apache.spark.sql.execution.streaming.runtime.StreamExecution
import org.apache.spark.sql.streaming.Trigger
import org.apache.spark.sql.test.TestSparkSession
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
index 702bd4f6ebdad..a55b6e0068519 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
@@ -26,7 +26,7 @@ import org.apache.kafka.common.security.auth.SecurityProtocol.SASL_PLAINTEXT
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.security.HadoopDelegationTokenManager
import org.apache.spark.internal.config.{KEYTAB, PRINCIPAL}
-import org.apache.spark.sql.execution.streaming.MemoryStream
+import org.apache.spark.sql.execution.streaming.runtime.MemoryStream
import org.apache.spark.sql.streaming.{OutputMode, StreamTest}
import org.apache.spark.sql.test.SharedSparkSession
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
index e738abf21f597..7eabf05235060 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
@@ -29,22 +29,23 @@ import scala.io.Source
import scala.jdk.CollectionConverters._
import scala.util.Random
-import org.apache.commons.io.FileUtils
import org.apache.kafka.clients.producer.{ProducerRecord, RecordMetadata}
import org.apache.kafka.common.TopicPartition
import org.scalatest.concurrent.PatienceConfiguration.Timeout
import org.scalatest.matchers.should._
import org.scalatest.time.SpanSugar._
-import org.apache.spark.TestUtils
+import org.apache.spark.{SparkException, TestUtils}
import org.apache.spark.sql.{Dataset, ForeachWriter, Row, SparkSession}
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
import org.apache.spark.sql.connector.read.streaming.SparkDataStream
import org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2ScanRelation
import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
import org.apache.spark.sql.execution.streaming._
-import org.apache.spark.sql.execution.streaming.AsyncProgressTrackingMicroBatchExecution.{ASYNC_PROGRESS_TRACKING_CHECKPOINTING_INTERVAL_MS, ASYNC_PROGRESS_TRACKING_ENABLED}
+import org.apache.spark.sql.execution.streaming.checkpointing.OffsetSeq
import org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution
+import org.apache.spark.sql.execution.streaming.runtime.{MicroBatchExecution, StreamExecution, StreamingExecutionRelation}
+import org.apache.spark.sql.execution.streaming.runtime.AsyncProgressTrackingMicroBatchExecution.{ASYNC_PROGRESS_TRACKING_CHECKPOINTING_INTERVAL_MS, ASYNC_PROGRESS_TRACKING_ENABLED}
import org.apache.spark.sql.functions.{count, expr, window}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.kafka010.KafkaSourceProvider._
@@ -1729,7 +1730,7 @@ abstract class KafkaMicroBatchV2SourceSuite extends KafkaMicroBatchSourceSuiteBa
val checkpointDir = Utils.createTempDir().getCanonicalFile
// Copy the checkpoint to a temp dir to prevent changes to the original.
// Not doing this will lead to the test passing on the first run, but fail subsequent runs.
- FileUtils.copyDirectory(new File(resourceUri), checkpointDir)
+ Utils.copyDirectory(new File(resourceUri), checkpointDir)
testStream(query)(
StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
@@ -2071,6 +2072,44 @@ abstract class KafkaSourceSuiteBase extends KafkaSourceTest {
"subscribePattern" -> s"$topicPrefix-.*")
}
+ test("SPARK-53560: no crash looping during uncommitted batch retry in AvailableNow trigger") {
+ val topic = newTopic()
+ testUtils.createTopic(topic, partitions = 1)
+ testUtils.sendMessages(topic, (1 to 7).map(_.toString).toArray, Some(0))
+ def udfFailOn7(x: Int): Int = {
+ if (x == 7) throw new RuntimeException("error for 7")
+ x
+ }
+ val kafka =
+ spark.readStream.format("kafka")
+ .option("kafka.bootstrap.servers", testUtils.brokerAddress)
+ .option("subscribe", topic)
+ .option("startingOffsets", "earliest")
+ .load()
+ .select(expr("CAST(CAST(value AS STRING) AS INT)").as("value"))
+ .as[Int]
+ .map(udfFailOn7)
+
+ withTempDir { dir =>
+ testStream(kafka)(
+ StartStream(Trigger.AvailableNow, checkpointLocation = dir.getAbsolutePath),
+ ExpectFailure[SparkException] { e =>
+ assert(e.getMessage.contains("error for 7"))
+ },
+ AssertOnQuery { q =>
+ testUtils.addPartitions(topic, 2)
+ !q.isActive
+ },
+ StartStream(Trigger.AvailableNow, checkpointLocation = dir.getAbsolutePath),
+ // Getting this error means the query has passed the planning stage, so
+ // verifyEndOffsetForTriggerAvailableNow succeeds.
+ ExpectFailure[SparkException] { e =>
+ assert(e.getMessage.contains("error for 7"))
+ }
+ )
+ }
+ }
+
private def testFromSpecificTimestampsWithNoMatchingStartingOffset(
topic: String,
options: (String, String)*): Unit = {
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
index 82edba59995ec..a444128cb2c0b 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
@@ -32,7 +32,7 @@ import org.scalatest.time.SpanSugar._
import org.apache.spark.{SparkConf, SparkContext, TestUtils}
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, SpecificInternalRow, UnsafeProjection}
-import org.apache.spark.sql.execution.streaming.{MemoryStream, MemoryStreamBase}
+import org.apache.spark.sql.execution.streaming.runtime.{MemoryStream, MemoryStreamBase}
import org.apache.spark.sql.execution.streaming.sources.ContinuousMemoryStream
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala
index 553ab42b9c8b1..18330feac861c 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.sql.kafka010
import java.io.File
-import org.apache.spark.sql.execution.streaming._
+import org.apache.spark.sql.execution.streaming.checkpointing.{OffsetSeq, OffsetSeqLog}
+import org.apache.spark.sql.execution.streaming.runtime._
import org.apache.spark.sql.streaming.OffsetSuite
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.util.Utils
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 60de3705636ec..b31f6af1c794c 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.kafka010
import java.io.{File, IOException}
import java.net.InetSocketAddress
-import java.nio.charset.StandardCharsets
+import java.nio.file.Files
import java.util.{Collections, Properties, UUID}
import java.util.concurrent.TimeUnit
import javax.security.auth.login.Configuration
@@ -27,7 +27,6 @@ import javax.security.auth.login.Configuration
import scala.io.Source
import scala.jdk.CollectionConverters._
-import com.google.common.io.Files
import kafka.log.LogManager
import kafka.server.{HostedPartition, KafkaConfig, KafkaServer}
import kafka.server.checkpoints.OffsetCheckpointFile
@@ -53,7 +52,7 @@ import org.scalatest.concurrent.Eventually._
import org.scalatest.time.SpanSugar._
import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.kafka010.KafkaTokenUtil
import org.apache.spark.util.{SecurityUtils, ShutdownHookManager, Utils}
@@ -176,7 +175,7 @@ class KafkaTestUtils(
}
kdc.getKrb5conf.delete()
- Files.asCharSink(kdc.getKrb5conf, StandardCharsets.UTF_8).write(krb5confStr)
+ Files.writeString(kdc.getKrb5conf.toPath, krb5confStr)
logDebug(s"krb5.conf file content: $krb5confStr")
}
@@ -240,7 +239,7 @@ class KafkaTestUtils(
| principal="$kafkaServerUser@$realm";
|};
""".stripMargin.trim
- Files.asCharSink(file, StandardCharsets.UTF_8).write(content)
+ Files.writeString(file.toPath, content)
logDebug(s"Created JAAS file: ${file.getPath}")
logDebug(s"JAAS file content: $content")
file.getAbsolutePath()
@@ -455,7 +454,7 @@ class KafkaTestUtils(
}
private def getOffsets(topics: Set[String], offsetSpec: OffsetSpec): Map[TopicPartition, Long] = {
- val listOffsetsParams = adminClient.describeTopics(topics.asJava).all().get().asScala
+ val listOffsetsParams = adminClient.describeTopics(topics.asJava).allTopicNames().get().asScala
.flatMap { topicDescription =>
topicDescription._2.partitions().asScala.map { topicPartitionInfo =>
new TopicPartition(topicDescription._1, topicPartitionInfo.partition())
diff --git a/connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaDelegationTokenProvider.scala b/connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaDelegationTokenProvider.scala
index 8acdd01a15bab..bce318811dc27 100644
--- a/connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaDelegationTokenProvider.scala
+++ b/connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaDelegationTokenProvider.scala
@@ -24,7 +24,7 @@ import org.apache.hadoop.security.Credentials
import org.apache.kafka.common.security.auth.SecurityProtocol.{SASL_PLAINTEXT, SASL_SSL, SSL}
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLUSTER_ID, SERVICE_NAME}
import org.apache.spark.security.HadoopDelegationTokenProvider
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala
index 2320f1908da5a..24c3dd0cddf17 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import org.apache.kafka.clients.consumer._
import org.apache.kafka.common.TopicPartition
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.CONFIG
import org.apache.spark.kafka010.KafkaConfigUpdater
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala
index f7bea064d2d6c..beaac35720445 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala
@@ -27,7 +27,7 @@ import scala.jdk.CollectionConverters._
import org.apache.kafka.clients.consumer._
import org.apache.kafka.common.TopicPartition
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{OFFSET, TIME, TOPIC_PARTITION, TOPIC_PARTITION_OFFSET_RANGE}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{StreamingContext, Time}
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala
index 75b046430ef50..03701dbda0947 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala
@@ -26,7 +26,7 @@ import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord, KafkaC
import org.apache.kafka.common.{KafkaException, TopicPartition}
import org.apache.spark.TaskContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.kafka010.KafkaConfigUpdater
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala
index 2637034766574..af97d5464e004 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala
@@ -23,7 +23,7 @@ import org.apache.kafka.clients.consumer.{ ConsumerConfig, ConsumerRecord }
import org.apache.kafka.common.TopicPartition
import org.apache.spark.{Partition, SparkContext, TaskContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{FROM_OFFSET, PARTITION_ID, TOPIC, UNTIL_OFFSET}
import org.apache.spark.internal.config.Network._
import org.apache.spark.partial.{BoundedDouble, PartialResult}
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaUtils.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaUtils.scala
index d15e5e25f561d..6b0ec62947108 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaUtils.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaUtils.scala
@@ -24,7 +24,7 @@ import org.apache.kafka.common.TopicPartition
import org.apache.spark.SparkContext
import org.apache.spark.api.java.{ JavaRDD, JavaSparkContext }
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CONFIG, GROUP_ID}
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.StreamingContext
@@ -112,7 +112,7 @@ object KafkaUtils extends Logging {
locationStrategy: LocationStrategy,
consumerStrategy: ConsumerStrategy[K, V]
): InputDStream[ConsumerRecord[K, V]] = {
- val ppc = new DefaultPerPartitionConfig(ssc.sparkContext.getConf)
+ val ppc = new DefaultPerPartitionConfig(ssc.sparkContext.getReadOnlyConf)
createDirectStream[K, V](ssc, locationStrategy, consumerStrategy, ppc)
}
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/PerPartitionConfig.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/PerPartitionConfig.scala
index b261500a454a3..058ac00ae27dd 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/PerPartitionConfig.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/PerPartitionConfig.scala
@@ -19,7 +19,7 @@ package org.apache.spark.streaming.kafka010
import org.apache.kafka.common.TopicPartition
-import org.apache.spark.SparkConf
+import org.apache.spark.ReadOnlySparkConf
/**
* Interface for user-supplied configurations that can't otherwise be set via Spark properties,
@@ -37,7 +37,7 @@ abstract class PerPartitionConfig extends Serializable {
/**
* Default per-partition configuration
*/
-private class DefaultPerPartitionConfig(conf: SparkConf)
+private class DefaultPerPartitionConfig(conf: ReadOnlySparkConf)
extends PerPartitionConfig {
val maxRate = conf.get(MAX_RATE_PER_PARTITION)
val minRate = conf.get(MIN_RATE_PER_PARTITION)
diff --git a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
index 212693f6e02cc..a1264af021aa1 100644
--- a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
+++ b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
@@ -38,6 +38,7 @@ import org.scalatest.concurrent.Eventually.{eventually, interval, timeout}
import org.apache.spark._
import org.apache.spark.scheduler.ExecutorCacheTaskLocation
import org.apache.spark.streaming.kafka010.mocks.MockTime
+import org.apache.spark.util.Utils
class KafkaRDDSuite extends SparkFunSuite {
@@ -91,7 +92,7 @@ class KafkaRDDSuite extends SparkFunSuite {
val logs = new Pool[TopicPartition, UnifiedLog]()
val logDir = kafkaTestUtils.brokerLogDir
val dir = new File(logDir, topic + "-" + partition)
- dir.mkdirs()
+ Utils.createDirectory(dir)
val logProps = new ju.Properties()
logProps.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT)
logProps.put(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG, java.lang.Float.valueOf(0.1f))
diff --git a/connector/kinesis-asl/pom.xml b/connector/kinesis-asl/pom.xml
index 2d02b910bf43f..aa90e3c8a94f5 100644
--- a/connector/kinesis-asl/pom.xml
+++ b/connector/kinesis-asl/pom.xml
@@ -64,10 +64,16 @@
${aws.java.sdk.version}
- com.amazonaws
+ software.amazon.kinesis
amazon-kinesis-producer
${aws.kinesis.producer.version}
test
+
+
+ com.kjetland
+ mbknor-jackson-jsonschema_2.12
+
+
diff --git a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
index b391203b4b968..ac3622f93321a 100644
--- a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
+++ b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
@@ -29,7 +29,7 @@ import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord
import com.amazonaws.services.kinesis.model._
import org.apache.spark._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ERROR, NUM_RETRY}
import org.apache.spark.rdd.{BlockRDD, BlockRDDPartition}
import org.apache.spark.storage.BlockId
diff --git a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
index c52eeca1e48a1..b259a5337f37e 100644
--- a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
+++ b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
@@ -22,7 +22,7 @@ import scala.util.control.NonFatal
import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{SHARD_ID, WORKER_URL}
import org.apache.spark.streaming.Duration
import org.apache.spark.streaming.util.RecurringTimer
diff --git a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReadConfigurations.scala b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReadConfigurations.scala
index 871071e4677e3..c2d0a39d6ae99 100644
--- a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReadConfigurations.scala
+++ b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReadConfigurations.scala
@@ -44,9 +44,9 @@ private[kinesis] object KinesisReadConfigurations {
def apply(ssc: StreamingContext): KinesisReadConfigurations = {
KinesisReadConfigurations(
- maxRetries = ssc.sc.getConf.getInt(RETRY_MAX_ATTEMPTS_KEY, DEFAULT_MAX_RETRIES),
+ maxRetries = ssc.sc.getReadOnlyConf.getInt(RETRY_MAX_ATTEMPTS_KEY, DEFAULT_MAX_RETRIES),
retryWaitTimeMs = JavaUtils.timeStringAsMs(
- ssc.sc.getConf.get(RETRY_WAIT_TIME_KEY, DEFAULT_RETRY_WAIT_TIME)),
+ ssc.sc.getReadOnlyConf.get(RETRY_WAIT_TIME_KEY, DEFAULT_RETRY_WAIT_TIME)),
retryTimeoutMs = ssc.graph.batchDuration.milliseconds)
}
diff --git a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
index 953817e625e48..ab91431035fef 100644
--- a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
+++ b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
@@ -28,7 +28,7 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.{KinesisClientLib
import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel
import com.amazonaws.services.kinesis.model.Record
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.WORKER_URL
import org.apache.spark.storage.{StorageLevel, StreamBlockId}
import org.apache.spark.streaming.Duration
diff --git a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala
index cd740f971e484..8304ddda96dfa 100644
--- a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala
+++ b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala
@@ -26,7 +26,7 @@ import com.amazonaws.services.kinesis.clientlibrary.interfaces.{IRecordProcessor
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason
import com.amazonaws.services.kinesis.model.Record
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{KINESIS_REASON, RETRY_INTERVAL, SHARD_ID, WORKER_URL}
/**
diff --git a/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala b/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala
index 58b64ba11d35d..6b06f197a137d 100644
--- a/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala
+++ b/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala
@@ -18,15 +18,14 @@ package org.apache.spark.streaming.kinesis
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
+import java.util.concurrent.{Executors, TimeUnit}
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
-import com.amazonaws.services.kinesis.producer.{KinesisProducer => KPLProducer,
- KinesisProducerConfiguration, UserRecordResult}
import com.google.common.util.concurrent.{FutureCallback, Futures}
-
-import org.apache.spark.util.ThreadUtils
+import software.amazon.kinesis.producer.{KinesisProducer => KPLProducer,
+ KinesisProducerConfiguration, UserRecordResult}
private[kinesis] class KPLBasedKinesisTestUtils(streamShardCount: Int = 2)
extends KinesisTestUtils(streamShardCount) {
@@ -53,6 +52,7 @@ private[kinesis] class KPLDataGenerator(regionName: String) extends KinesisDataG
}
override def sendData(streamName: String, data: Seq[Int]): Map[String, Seq[(Int, String)]] = {
+ val executor = Executors.newSingleThreadExecutor()
val shardIdToSeqNumbers = new mutable.HashMap[String, ArrayBuffer[(Int, String)]]()
data.foreach { num =>
val str = num.toString
@@ -63,15 +63,17 @@ private[kinesis] class KPLDataGenerator(regionName: String) extends KinesisDataG
override def onSuccess(result: UserRecordResult): Unit = {
val shardId = result.getShardId
- val seqNumber = result.getSequenceNumber()
+ val seqNumber = result.getSequenceNumber
val sentSeqNumbers = shardIdToSeqNumbers.getOrElseUpdate(shardId,
new ArrayBuffer[(Int, String)]())
sentSeqNumbers += ((num, seqNumber))
}
}
- Futures.addCallback(future, kinesisCallBack, ThreadUtils.sameThreadExecutorService())
+ Futures.addCallback(future, kinesisCallBack, executor)
}
producer.flushSync()
- shardIdToSeqNumbers.toMap.transform((_, v) => v.toSeq)
+ executor.shutdown()
+ executor.awaitTermination(10, TimeUnit.SECONDS)
+ shardIdToSeqNumbers.toMap.transform((_, v) => v.toSeq.sortBy(_._2))
}
}
diff --git a/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala b/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
index 652822c5fdc97..7674cef105e71 100644
--- a/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
+++ b/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
@@ -32,8 +32,9 @@ import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient
import com.amazonaws.services.dynamodbv2.document.DynamoDB
import com.amazonaws.services.kinesis.{AmazonKinesis, AmazonKinesisClient}
import com.amazonaws.services.kinesis.model._
+import com.amazonaws.waiters.WaiterParameters
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{STREAM_NAME, TABLE_NAME}
/**
@@ -61,6 +62,8 @@ private[kinesis] class KinesisTestUtils(streamShardCount: Int = 2) extends Loggi
client
}
+ private lazy val streamExistsWaiter = kinesisClient.waiters().streamExists()
+
private lazy val dynamoDB = {
val dynamoDBClient = new AmazonDynamoDBClient(new DefaultAWSCredentialsProviderChain())
dynamoDBClient.setRegion(RegionUtils.getRegion(regionName))
@@ -184,18 +187,9 @@ private[kinesis] class KinesisTestUtils(streamShardCount: Int = 2) extends Loggi
}
private def waitForStreamToBeActive(streamNameToWaitFor: String): Unit = {
- val startTimeNs = System.nanoTime()
- while (System.nanoTime() - startTimeNs < TimeUnit.SECONDS.toNanos(createStreamTimeoutSeconds)) {
- Thread.sleep(TimeUnit.SECONDS.toMillis(describeStreamPollTimeSeconds))
- describeStream(streamNameToWaitFor).foreach { description =>
- val streamStatus = description.getStreamStatus()
- logDebug(s"\t- current state: $streamStatus\n")
- if ("ACTIVE".equals(streamStatus)) {
- return
- }
- }
- }
- require(false, s"Stream $streamName never became active")
+ val describeStreamRequest = new DescribeStreamRequest()
+ .withStreamName(streamNameToWaitFor)
+ streamExistsWaiter.run(new WaiterParameters(describeStreamRequest))
}
}
diff --git a/connector/profiler/README.md b/connector/profiler/README.md
index 9e0a58444c605..742b1a8bb5da3 100644
--- a/connector/profiler/README.md
+++ b/connector/profiler/README.md
@@ -16,7 +16,7 @@ or
## Executor Code Profiling
-The spark profiler module enables code profiling of executors in cluster mode based on the [async profiler](https://github.com/async-profiler/async-profiler/blob/v3.0/README.md), a low overhead sampling profiler. This allows a Spark application to capture CPU and memory profiles for application running on a cluster which can later be analyzed for performance issues. The profiler captures [Java Flight Recorder (jfr)](https://access.redhat.com/documentation/es-es/red_hat_build_of_openjdk/17/html/using_jdk_flight_recorder_with_red_hat_build_of_openjdk/openjdk-flight-recorded-overview) files for each executor; these can be read by many tools including Java Mission Control and Intellij.
+The spark profiler module enables code profiling of executors in cluster mode based on the [async profiler](https://github.com/async-profiler/async-profiler/blob/v4.0/README.md), a low overhead sampling profiler. This allows a Spark application to capture CPU and memory profiles for application running on a cluster which can later be analyzed for performance issues. The profiler captures [Java Flight Recorder (jfr)](https://access.redhat.com/documentation/es-es/red_hat_build_of_openjdk/17/html/using_jdk_flight_recorder_with_red_hat_build_of_openjdk/openjdk-flight-recorded-overview) files for each executor; these can be read by many tools including Java Mission Control and Intellij.
The profiler writes the jfr files to the executor's working directory in the executor's local file system and the files can grow to be large, so it is advisable that the executor machines have adequate storage. The profiler can be configured to copy the jfr files to a hdfs location before the executor shuts down.
@@ -95,7 +95,7 @@ Then enable the profiling in the configuration.
| event=wall,interval=10ms,alloc=2m,lock=10ms,chunktime=300s |
Arguments to pass to the Async Profiler. Detailed options are documented in the comments here:
- Profiler arguments.
+ Profiler options.
Note that the arguments to start, stop, specify output format, and output file do not have to be specified.
|
4.0.0 |
diff --git a/connector/profiler/src/main/scala/org/apache/spark/profiler/ProfilerPlugin.scala b/connector/profiler/src/main/scala/org/apache/spark/profiler/ProfilerPlugin.scala
index db3de90099f7c..cb341aef3a0d8 100644
--- a/connector/profiler/src/main/scala/org/apache/spark/profiler/ProfilerPlugin.scala
+++ b/connector/profiler/src/main/scala/org/apache/spark/profiler/ProfilerPlugin.scala
@@ -23,7 +23,7 @@ import scala.util.Random
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, PluginContext, SparkPlugin}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.EXECUTOR_ID
/**
diff --git a/connector/profiler/src/main/scala/org/apache/spark/profiler/SparkAsyncProfiler.scala b/connector/profiler/src/main/scala/org/apache/spark/profiler/SparkAsyncProfiler.scala
index 125ce63620b63..02d39a2c435b3 100644
--- a/connector/profiler/src/main/scala/org/apache/spark/profiler/SparkAsyncProfiler.scala
+++ b/connector/profiler/src/main/scala/org/apache/spark/profiler/SparkAsyncProfiler.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.fs.permission.FsPermission
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext.DRIVER_IDENTIFIER
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PATH
import org.apache.spark.util.{ThreadUtils, Utils}
@@ -36,7 +36,6 @@ import org.apache.spark.util.{ThreadUtils, Utils}
private[spark] class SparkAsyncProfiler(conf: SparkConf, executorId: String) extends Logging {
private var running = false
- private val enableProfiler = conf.get(PROFILER_EXECUTOR_ENABLED)
private val profilerOptions = conf.get(PROFILER_ASYNC_PROFILER_OPTIONS)
private val profilerDfsDirOpt = conf.get(PROFILER_DFS_DIR)
private val profilerLocalDir = conf.get(PROFILER_LOCAL_DIR)
@@ -66,9 +65,15 @@ private[spark] class SparkAsyncProfiler(conf: SparkConf, executorId: String) ext
private var threadpool: ScheduledExecutorService = _
@volatile private var writing: Boolean = false
+ lazy private val extractionDir =
+ Utils.createTempDir(Utils.getLocalDir(conf), "asyncProfiler").toPath
+
val profiler: Option[AsyncProfiler] = {
Option(
- if (enableProfiler && AsyncProfilerLoader.isSupported) AsyncProfilerLoader.load() else null
+ if (AsyncProfilerLoader.isSupported) {
+ AsyncProfilerLoader.setExtractionDirectory(extractionDir)
+ AsyncProfilerLoader.load()
+ } else null
)
}
diff --git a/connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java b/connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java
index 48c61e80d6655..3a38a52f7ab5d 100644
--- a/connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java
+++ b/connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java
@@ -295,7 +295,7 @@ private void reportTimer(String name, Timer timer) {
reportMetered(sanitizedName, timer, group, "calls");
} catch (GangliaException e) {
LOGGER.warn("Unable to report timer {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, sanitizedName));
+ MDC.of(LogKeys.METRIC_NAME, sanitizedName));
}
}
@@ -306,7 +306,7 @@ private void reportMeter(String name, Meter meter) {
reportMetered(sanitizedName, meter, group, "events");
} catch (GangliaException e) {
LOGGER.warn("Unable to report meter {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, name));
+ MDC.of(LogKeys.METRIC_NAME, name));
}
}
@@ -338,7 +338,7 @@ private void reportHistogram(String name, Histogram histogram) {
announceIfEnabled(P999, sanitizedName, group, snapshot.get999thPercentile(), "");
} catch (GangliaException e) {
LOGGER.warn("Unable to report histogram {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, sanitizedName));
+ MDC.of(LogKeys.METRIC_NAME, sanitizedName));
}
}
@@ -349,7 +349,7 @@ private void reportCounter(String name, Counter counter) {
announce(prefix(sanitizedName, COUNT.getCode()), group, Long.toString(counter.getCount()), GMetricType.DOUBLE, "");
} catch (GangliaException e) {
LOGGER.warn("Unable to report counter {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, name));
+ MDC.of(LogKeys.METRIC_NAME, name));
}
}
@@ -363,7 +363,7 @@ private void reportGauge(String name, Gauge gauge) {
announce(name(prefix, sanitizedName), group, value, type, "");
} catch (GangliaException e) {
LOGGER.warn("Unable to report gauge {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, name));
+ MDC.of(LogKeys.METRIC_NAME, name));
}
}
diff --git a/core/benchmarks/ChecksumBenchmark-jdk21-results.txt b/core/benchmarks/ChecksumBenchmark-jdk21-results.txt
index 0065bd684beba..983a194577dc8 100644
--- a/core/benchmarks/ChecksumBenchmark-jdk21-results.txt
+++ b/core/benchmarks/ChecksumBenchmark-jdk21-results.txt
@@ -2,12 +2,12 @@
Benchmark Checksum Algorithms
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Checksum Algorithms: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Adler32 11110 11117 9 0.0 10849608.9 1.0X
-CRC32 2763 2768 4 0.0 2698564.7 4.0X
-CRC32C 2249 2303 48 0.0 2195990.4 4.9X
+Adler32 11120 11122 2 0.0 10859406.5 1.0X
+CRC32 2776 2777 1 0.0 2711240.8 4.0X
+CRC32C 2117 2122 8 0.0 2067087.9 5.3X
diff --git a/core/benchmarks/ChecksumBenchmark-results.txt b/core/benchmarks/ChecksumBenchmark-results.txt
index e907066d649c3..fde01a888d64b 100644
--- a/core/benchmarks/ChecksumBenchmark-results.txt
+++ b/core/benchmarks/ChecksumBenchmark-results.txt
@@ -2,12 +2,12 @@
Benchmark Checksum Algorithms
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Checksum Algorithms: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Adler32 11096 11101 4 0.0 10835715.4 1.0X
-CRC32 2715 2716 1 0.0 2651776.1 4.1X
-CRC32C 1629 1647 15 0.0 1591225.5 6.8X
+Adler32 11207 11212 5 0.0 10943898.5 1.0X
+CRC32 2770 2775 5 0.0 2705151.3 4.0X
+CRC32C 2516 2532 23 0.0 2457304.0 4.5X
diff --git a/core/benchmarks/CoalescedRDDBenchmark-jdk21-results.txt b/core/benchmarks/CoalescedRDDBenchmark-jdk21-results.txt
index 88142e681790d..14ae5f884c612 100644
--- a/core/benchmarks/CoalescedRDDBenchmark-jdk21-results.txt
+++ b/core/benchmarks/CoalescedRDDBenchmark-jdk21-results.txt
@@ -2,39 +2,39 @@
Coalesced RDD , large scale
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Coalesced RDD: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------
-Coalesce Num Partitions: 100 Num Hosts: 1 255 263 12 0.4 2548.3 1.0X
-Coalesce Num Partitions: 100 Num Hosts: 5 133 135 2 0.8 1332.7 1.9X
-Coalesce Num Partitions: 100 Num Hosts: 10 99 103 4 1.0 994.6 2.6X
-Coalesce Num Partitions: 100 Num Hosts: 20 104 106 2 1.0 1035.4 2.5X
-Coalesce Num Partitions: 100 Num Hosts: 40 92 104 16 1.1 919.5 2.8X
-Coalesce Num Partitions: 100 Num Hosts: 80 99 99 1 1.0 986.7 2.6X
-Coalesce Num Partitions: 500 Num Hosts: 1 880 892 13 0.1 8796.0 0.3X
-Coalesce Num Partitions: 500 Num Hosts: 5 247 254 7 0.4 2474.6 1.0X
-Coalesce Num Partitions: 500 Num Hosts: 10 167 169 1 0.6 1672.5 1.5X
-Coalesce Num Partitions: 500 Num Hosts: 20 127 128 1 0.8 1269.1 2.0X
-Coalesce Num Partitions: 500 Num Hosts: 40 107 107 1 0.9 1065.5 2.4X
-Coalesce Num Partitions: 500 Num Hosts: 80 109 112 3 0.9 1090.6 2.3X
-Coalesce Num Partitions: 1000 Num Hosts: 1 1651 1677 23 0.1 16508.5 0.2X
-Coalesce Num Partitions: 1000 Num Hosts: 5 407 418 14 0.2 4073.4 0.6X
-Coalesce Num Partitions: 1000 Num Hosts: 10 249 250 2 0.4 2485.6 1.0X
-Coalesce Num Partitions: 1000 Num Hosts: 20 165 170 7 0.6 1654.8 1.5X
-Coalesce Num Partitions: 1000 Num Hosts: 40 129 130 3 0.8 1287.6 2.0X
-Coalesce Num Partitions: 1000 Num Hosts: 80 108 110 2 0.9 1082.9 2.4X
-Coalesce Num Partitions: 5000 Num Hosts: 1 7737 7757 19 0.0 77368.9 0.0X
-Coalesce Num Partitions: 5000 Num Hosts: 5 1784 1790 5 0.1 17837.1 0.1X
-Coalesce Num Partitions: 5000 Num Hosts: 10 923 938 20 0.1 9226.0 0.3X
-Coalesce Num Partitions: 5000 Num Hosts: 20 520 526 9 0.2 5196.1 0.5X
-Coalesce Num Partitions: 5000 Num Hosts: 40 314 315 1 0.3 3137.0 0.8X
-Coalesce Num Partitions: 5000 Num Hosts: 80 201 213 13 0.5 2012.4 1.3X
-Coalesce Num Partitions: 10000 Num Hosts: 1 14476 14509 28 0.0 144760.1 0.0X
-Coalesce Num Partitions: 10000 Num Hosts: 5 3401 3429 41 0.0 34012.2 0.1X
-Coalesce Num Partitions: 10000 Num Hosts: 10 1753 1759 9 0.1 17526.3 0.1X
-Coalesce Num Partitions: 10000 Num Hosts: 20 927 936 9 0.1 9272.2 0.3X
-Coalesce Num Partitions: 10000 Num Hosts: 40 510 518 12 0.2 5097.7 0.5X
-Coalesce Num Partitions: 10000 Num Hosts: 80 303 307 3 0.3 3034.1 0.8X
+Coalesce Num Partitions: 100 Num Hosts: 1 245 254 11 0.4 2451.6 1.0X
+Coalesce Num Partitions: 100 Num Hosts: 5 116 122 5 0.9 1163.8 2.1X
+Coalesce Num Partitions: 100 Num Hosts: 10 108 109 1 0.9 1076.0 2.3X
+Coalesce Num Partitions: 100 Num Hosts: 20 93 102 10 1.1 934.6 2.6X
+Coalesce Num Partitions: 100 Num Hosts: 40 100 100 1 1.0 997.2 2.5X
+Coalesce Num Partitions: 100 Num Hosts: 80 93 96 5 1.1 928.3 2.6X
+Coalesce Num Partitions: 500 Num Hosts: 1 848 858 11 0.1 8484.8 0.3X
+Coalesce Num Partitions: 500 Num Hosts: 5 244 246 2 0.4 2440.6 1.0X
+Coalesce Num Partitions: 500 Num Hosts: 10 167 169 2 0.6 1670.3 1.5X
+Coalesce Num Partitions: 500 Num Hosts: 20 128 130 3 0.8 1281.3 1.9X
+Coalesce Num Partitions: 500 Num Hosts: 40 105 106 1 1.0 1050.1 2.3X
+Coalesce Num Partitions: 500 Num Hosts: 80 96 96 1 1.0 955.8 2.6X
+Coalesce Num Partitions: 1000 Num Hosts: 1 1599 1605 5 0.1 15991.4 0.2X
+Coalesce Num Partitions: 1000 Num Hosts: 5 403 411 7 0.2 4026.5 0.6X
+Coalesce Num Partitions: 1000 Num Hosts: 10 251 258 7 0.4 2506.6 1.0X
+Coalesce Num Partitions: 1000 Num Hosts: 20 167 172 9 0.6 1667.8 1.5X
+Coalesce Num Partitions: 1000 Num Hosts: 40 125 126 1 0.8 1249.5 2.0X
+Coalesce Num Partitions: 1000 Num Hosts: 80 107 112 6 0.9 1069.6 2.3X
+Coalesce Num Partitions: 5000 Num Hosts: 1 7327 7352 22 0.0 73268.6 0.0X
+Coalesce Num Partitions: 5000 Num Hosts: 5 1826 1833 6 0.1 18259.3 0.1X
+Coalesce Num Partitions: 5000 Num Hosts: 10 953 958 5 0.1 9525.3 0.3X
+Coalesce Num Partitions: 5000 Num Hosts: 20 524 526 2 0.2 5239.3 0.5X
+Coalesce Num Partitions: 5000 Num Hosts: 40 317 339 22 0.3 3173.5 0.8X
+Coalesce Num Partitions: 5000 Num Hosts: 80 204 207 2 0.5 2043.3 1.2X
+Coalesce Num Partitions: 10000 Num Hosts: 1 13804 13819 14 0.0 138035.6 0.0X
+Coalesce Num Partitions: 10000 Num Hosts: 5 3691 3707 14 0.0 36911.4 0.1X
+Coalesce Num Partitions: 10000 Num Hosts: 10 1854 1860 5 0.1 18542.8 0.1X
+Coalesce Num Partitions: 10000 Num Hosts: 20 977 980 3 0.1 9767.3 0.3X
+Coalesce Num Partitions: 10000 Num Hosts: 40 538 542 4 0.2 5376.1 0.5X
+Coalesce Num Partitions: 10000 Num Hosts: 80 320 325 8 0.3 3204.4 0.8X
diff --git a/core/benchmarks/CoalescedRDDBenchmark-results.txt b/core/benchmarks/CoalescedRDDBenchmark-results.txt
index d3a2cc1a6e108..5d005c30d397b 100644
--- a/core/benchmarks/CoalescedRDDBenchmark-results.txt
+++ b/core/benchmarks/CoalescedRDDBenchmark-results.txt
@@ -2,39 +2,39 @@
Coalesced RDD , large scale
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Coalesced RDD: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------
-Coalesce Num Partitions: 100 Num Hosts: 1 150 156 7 0.7 1501.3 1.0X
-Coalesce Num Partitions: 100 Num Hosts: 5 109 113 4 0.9 1087.1 1.4X
-Coalesce Num Partitions: 100 Num Hosts: 10 94 106 11 1.1 943.2 1.6X
-Coalesce Num Partitions: 100 Num Hosts: 20 92 99 8 1.1 924.6 1.6X
-Coalesce Num Partitions: 100 Num Hosts: 40 91 97 6 1.1 906.9 1.7X
-Coalesce Num Partitions: 100 Num Hosts: 80 92 98 7 1.1 916.3 1.6X
-Coalesce Num Partitions: 500 Num Hosts: 1 317 323 7 0.3 3174.6 0.5X
-Coalesce Num Partitions: 500 Num Hosts: 5 139 140 2 0.7 1394.3 1.1X
-Coalesce Num Partitions: 500 Num Hosts: 10 114 129 19 0.9 1144.9 1.3X
-Coalesce Num Partitions: 500 Num Hosts: 20 107 116 15 0.9 1065.8 1.4X
-Coalesce Num Partitions: 500 Num Hosts: 40 99 104 5 1.0 995.0 1.5X
-Coalesce Num Partitions: 500 Num Hosts: 80 96 101 5 1.0 960.9 1.6X
-Coalesce Num Partitions: 1000 Num Hosts: 1 523 533 9 0.2 5231.6 0.3X
-Coalesce Num Partitions: 1000 Num Hosts: 5 188 188 0 0.5 1882.4 0.8X
-Coalesce Num Partitions: 1000 Num Hosts: 10 131 133 2 0.8 1314.5 1.1X
-Coalesce Num Partitions: 1000 Num Hosts: 20 112 114 3 0.9 1120.6 1.3X
-Coalesce Num Partitions: 1000 Num Hosts: 40 105 105 0 1.0 1047.7 1.4X
-Coalesce Num Partitions: 1000 Num Hosts: 80 98 105 5 1.0 984.4 1.5X
-Coalesce Num Partitions: 5000 Num Hosts: 1 2321 2342 28 0.0 23213.0 0.1X
-Coalesce Num Partitions: 5000 Num Hosts: 5 686 697 17 0.1 6858.9 0.2X
-Coalesce Num Partitions: 5000 Num Hosts: 10 386 391 4 0.3 3863.4 0.4X
-Coalesce Num Partitions: 5000 Num Hosts: 20 252 253 2 0.4 2518.2 0.6X
-Coalesce Num Partitions: 5000 Num Hosts: 40 168 171 4 0.6 1683.5 0.9X
-Coalesce Num Partitions: 5000 Num Hosts: 80 132 137 5 0.8 1322.0 1.1X
-Coalesce Num Partitions: 10000 Num Hosts: 1 4250 4268 16 0.0 42497.7 0.0X
-Coalesce Num Partitions: 10000 Num Hosts: 5 1401 1408 7 0.1 14008.0 0.1X
-Coalesce Num Partitions: 10000 Num Hosts: 10 724 725 2 0.1 7237.0 0.2X
-Coalesce Num Partitions: 10000 Num Hosts: 20 396 402 5 0.3 3959.1 0.4X
-Coalesce Num Partitions: 10000 Num Hosts: 40 255 257 2 0.4 2550.1 0.6X
-Coalesce Num Partitions: 10000 Num Hosts: 80 181 183 2 0.6 1811.0 0.8X
+Coalesce Num Partitions: 100 Num Hosts: 1 136 142 5 0.7 1359.0 1.0X
+Coalesce Num Partitions: 100 Num Hosts: 5 108 109 1 0.9 1081.7 1.3X
+Coalesce Num Partitions: 100 Num Hosts: 10 91 96 9 1.1 906.7 1.5X
+Coalesce Num Partitions: 100 Num Hosts: 20 89 96 10 1.1 892.2 1.5X
+Coalesce Num Partitions: 100 Num Hosts: 40 86 94 10 1.2 864.5 1.6X
+Coalesce Num Partitions: 100 Num Hosts: 80 93 94 1 1.1 927.9 1.5X
+Coalesce Num Partitions: 500 Num Hosts: 1 311 316 8 0.3 3107.4 0.4X
+Coalesce Num Partitions: 500 Num Hosts: 5 134 135 1 0.7 1337.6 1.0X
+Coalesce Num Partitions: 500 Num Hosts: 10 136 137 1 0.7 1357.9 1.0X
+Coalesce Num Partitions: 500 Num Hosts: 20 108 108 0 0.9 1080.2 1.3X
+Coalesce Num Partitions: 500 Num Hosts: 40 101 103 2 1.0 1005.5 1.4X
+Coalesce Num Partitions: 500 Num Hosts: 80 107 118 11 0.9 1075.0 1.3X
+Coalesce Num Partitions: 1000 Num Hosts: 1 529 537 7 0.2 5292.2 0.3X
+Coalesce Num Partitions: 1000 Num Hosts: 5 190 191 1 0.5 1901.0 0.7X
+Coalesce Num Partitions: 1000 Num Hosts: 10 141 142 1 0.7 1409.8 1.0X
+Coalesce Num Partitions: 1000 Num Hosts: 20 131 132 1 0.8 1311.2 1.0X
+Coalesce Num Partitions: 1000 Num Hosts: 40 107 116 8 0.9 1069.6 1.3X
+Coalesce Num Partitions: 1000 Num Hosts: 80 105 108 4 0.9 1054.8 1.3X
+Coalesce Num Partitions: 5000 Num Hosts: 1 2333 2344 14 0.0 23327.2 0.1X
+Coalesce Num Partitions: 5000 Num Hosts: 5 698 698 1 0.1 6975.9 0.2X
+Coalesce Num Partitions: 5000 Num Hosts: 10 400 405 4 0.2 4004.7 0.3X
+Coalesce Num Partitions: 5000 Num Hosts: 20 266 269 3 0.4 2661.9 0.5X
+Coalesce Num Partitions: 5000 Num Hosts: 40 175 182 6 0.6 1748.0 0.8X
+Coalesce Num Partitions: 5000 Num Hosts: 80 144 147 4 0.7 1439.3 0.9X
+Coalesce Num Partitions: 10000 Num Hosts: 1 4245 4252 6 0.0 42448.1 0.0X
+Coalesce Num Partitions: 10000 Num Hosts: 5 1416 1423 7 0.1 14155.3 0.1X
+Coalesce Num Partitions: 10000 Num Hosts: 10 738 744 6 0.1 7383.3 0.2X
+Coalesce Num Partitions: 10000 Num Hosts: 20 417 421 4 0.2 4166.4 0.3X
+Coalesce Num Partitions: 10000 Num Hosts: 40 273 280 8 0.4 2734.6 0.5X
+Coalesce Num Partitions: 10000 Num Hosts: 80 185 189 3 0.5 1851.3 0.7X
diff --git a/core/benchmarks/KryoBenchmark-jdk21-results.txt b/core/benchmarks/KryoBenchmark-jdk21-results.txt
index ca03441d01a87..718f0746b6e5b 100644
--- a/core/benchmarks/KryoBenchmark-jdk21-results.txt
+++ b/core/benchmarks/KryoBenchmark-jdk21-results.txt
@@ -2,27 +2,27 @@
Benchmark Kryo Unsafe vs safe Serialization
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark Kryo Unsafe vs safe Serialization: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-basicTypes: Int with unsafe:true 166 169 6 6.0 165.7 1.0X
-basicTypes: Long with unsafe:true 178 182 4 5.6 177.5 0.9X
-basicTypes: Float with unsafe:true 183 191 9 5.5 182.7 0.9X
-basicTypes: Double with unsafe:true 186 193 7 5.4 186.4 0.9X
-Array: Int with unsafe:true 1 1 0 745.8 1.3 123.5X
-Array: Long with unsafe:true 2 3 0 451.7 2.2 74.8X
-Array: Float with unsafe:true 1 1 0 743.0 1.3 123.1X
-Array: Double with unsafe:true 2 2 0 475.6 2.1 78.8X
-Map of string->Double with unsafe:true 27 28 1 37.0 27.0 6.1X
-basicTypes: Int with unsafe:false 198 199 1 5.1 197.7 0.8X
-basicTypes: Long with unsafe:false 220 221 1 4.5 219.8 0.8X
-basicTypes: Float with unsafe:false 206 208 1 4.8 206.3 0.8X
-basicTypes: Double with unsafe:false 222 225 2 4.5 221.9 0.7X
-Array: Int with unsafe:false 13 14 1 78.0 12.8 12.9X
-Array: Long with unsafe:false 21 21 1 48.2 20.8 8.0X
-Array: Float with unsafe:false 6 6 0 178.9 5.6 29.6X
-Array: Double with unsafe:false 15 16 0 65.3 15.3 10.8X
-Map of string->Double with unsafe:false 28 29 2 35.2 28.4 5.8X
+basicTypes: Int with unsafe:true 167 170 4 6.0 167.1 1.0X
+basicTypes: Long with unsafe:true 179 185 7 5.6 179.4 0.9X
+basicTypes: Float with unsafe:true 184 190 5 5.4 184.5 0.9X
+basicTypes: Double with unsafe:true 185 191 8 5.4 185.1 0.9X
+Array: Int with unsafe:true 1 1 0 766.6 1.3 128.1X
+Array: Long with unsafe:true 2 2 0 491.0 2.0 82.0X
+Array: Float with unsafe:true 1 1 0 764.6 1.3 127.7X
+Array: Double with unsafe:true 2 2 0 488.3 2.0 81.6X
+Map of string->Double with unsafe:true 27 28 1 36.7 27.2 6.1X
+basicTypes: Int with unsafe:false 201 202 1 5.0 201.3 0.8X
+basicTypes: Long with unsafe:false 218 219 1 4.6 218.2 0.8X
+basicTypes: Float with unsafe:false 200 201 1 5.0 199.9 0.8X
+basicTypes: Double with unsafe:false 209 210 1 4.8 208.8 0.8X
+Array: Int with unsafe:false 13 13 0 79.2 12.6 13.2X
+Array: Long with unsafe:false 22 22 0 45.8 21.8 7.7X
+Array: Float with unsafe:false 13 14 1 77.6 12.9 13.0X
+Array: Double with unsafe:false 16 16 0 64.1 15.6 10.7X
+Map of string->Double with unsafe:false 28 28 1 36.3 27.5 6.1X
diff --git a/core/benchmarks/KryoBenchmark-results.txt b/core/benchmarks/KryoBenchmark-results.txt
index bffc9eafb79e3..aec0b52aba16b 100644
--- a/core/benchmarks/KryoBenchmark-results.txt
+++ b/core/benchmarks/KryoBenchmark-results.txt
@@ -2,27 +2,27 @@
Benchmark Kryo Unsafe vs safe Serialization
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark Kryo Unsafe vs safe Serialization: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-basicTypes: Int with unsafe:true 172 174 1 5.8 171.9 1.0X
-basicTypes: Long with unsafe:true 196 197 0 5.1 196.2 0.9X
-basicTypes: Float with unsafe:true 193 195 2 5.2 192.7 0.9X
-basicTypes: Double with unsafe:true 193 194 1 5.2 193.2 0.9X
-Array: Int with unsafe:true 1 1 0 715.2 1.4 122.9X
-Array: Long with unsafe:true 2 2 0 474.2 2.1 81.5X
-Array: Float with unsafe:true 1 1 0 718.2 1.4 123.5X
-Array: Double with unsafe:true 2 2 0 475.8 2.1 81.8X
-Map of string->Double with unsafe:true 27 28 0 36.7 27.2 6.3X
-basicTypes: Int with unsafe:false 207 209 5 4.8 207.3 0.8X
-basicTypes: Long with unsafe:false 239 241 2 4.2 238.9 0.7X
-basicTypes: Float with unsafe:false 215 217 2 4.6 215.4 0.8X
-basicTypes: Double with unsafe:false 220 225 7 4.5 220.2 0.8X
-Array: Int with unsafe:false 16 20 7 63.4 15.8 10.9X
-Array: Long with unsafe:false 22 22 0 45.9 21.8 7.9X
-Array: Float with unsafe:false 6 6 1 170.0 5.9 29.2X
-Array: Double with unsafe:false 10 10 0 98.6 10.1 16.9X
-Map of string->Double with unsafe:false 28 29 1 35.9 27.9 6.2X
+basicTypes: Int with unsafe:true 173 174 1 5.8 173.3 1.0X
+basicTypes: Long with unsafe:true 198 199 1 5.1 198.0 0.9X
+basicTypes: Float with unsafe:true 201 202 1 5.0 200.6 0.9X
+basicTypes: Double with unsafe:true 202 203 2 5.0 201.9 0.9X
+Array: Int with unsafe:true 1 2 0 730.8 1.4 126.6X
+Array: Long with unsafe:true 2 3 0 445.9 2.2 77.3X
+Array: Float with unsafe:true 1 1 0 761.1 1.3 131.9X
+Array: Double with unsafe:true 2 3 0 435.2 2.3 75.4X
+Map of string->Double with unsafe:true 27 28 1 37.1 27.0 6.4X
+basicTypes: Int with unsafe:false 218 219 1 4.6 218.4 0.8X
+basicTypes: Long with unsafe:false 242 243 1 4.1 242.2 0.7X
+basicTypes: Float with unsafe:false 221 221 1 4.5 220.7 0.8X
+basicTypes: Double with unsafe:false 228 230 1 4.4 228.1 0.8X
+Array: Int with unsafe:false 15 15 0 68.2 14.7 11.8X
+Array: Long with unsafe:false 22 22 0 45.8 21.8 7.9X
+Array: Float with unsafe:false 6 6 0 175.3 5.7 30.4X
+Array: Double with unsafe:false 10 10 0 100.5 9.9 17.4X
+Map of string->Double with unsafe:false 31 32 0 32.4 30.9 5.6X
diff --git a/core/benchmarks/KryoIteratorBenchmark-jdk21-results.txt b/core/benchmarks/KryoIteratorBenchmark-jdk21-results.txt
index 835b16e24d95f..4c65c4c5916c4 100644
--- a/core/benchmarks/KryoIteratorBenchmark-jdk21-results.txt
+++ b/core/benchmarks/KryoIteratorBenchmark-jdk21-results.txt
@@ -2,27 +2,27 @@
Benchmark of kryo asIterator on deserialization stream
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark of kryo asIterator on deserialization stream: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------------------------
-Colletion of int with 1 elements, useIterator: true 6 7 0 1.5 645.9 1.0X
-Colletion of int with 10 elements, useIterator: true 13 14 0 0.8 1330.8 0.5X
-Colletion of int with 100 elements, useIterator: true 80 81 1 0.1 7987.1 0.1X
-Colletion of string with 1 elements, useIterator: true 8 8 0 1.3 787.4 0.8X
-Colletion of string with 10 elements, useIterator: true 21 21 0 0.5 2113.7 0.3X
-Colletion of string with 100 elements, useIterator: true 161 162 1 0.1 16108.9 0.0X
-Colletion of Array[int] with 1 elements, useIterator: true 7 8 0 1.3 747.8 0.9X
-Colletion of Array[int] with 10 elements, useIterator: true 19 19 0 0.5 1879.8 0.3X
-Colletion of Array[int] with 100 elements, useIterator: true 140 141 1 0.1 14008.3 0.0X
-Colletion of int with 1 elements, useIterator: false 6 7 0 1.6 642.6 1.0X
-Colletion of int with 10 elements, useIterator: false 14 15 1 0.7 1414.6 0.5X
-Colletion of int with 100 elements, useIterator: false 87 88 1 0.1 8699.0 0.1X
-Colletion of string with 1 elements, useIterator: false 7 8 0 1.3 746.5 0.9X
-Colletion of string with 10 elements, useIterator: false 22 22 0 0.5 2192.3 0.3X
-Colletion of string with 100 elements, useIterator: false 161 162 2 0.1 16091.2 0.0X
-Colletion of Array[int] with 1 elements, useIterator: false 7 8 0 1.4 719.6 0.9X
-Colletion of Array[int] with 10 elements, useIterator: false 19 19 0 0.5 1869.6 0.3X
-Colletion of Array[int] with 100 elements, useIterator: false 138 139 1 0.1 13766.1 0.0X
+Colletion of int with 1 elements, useIterator: true 6 7 0 1.6 617.6 1.0X
+Colletion of int with 10 elements, useIterator: true 15 15 0 0.7 1457.7 0.4X
+Colletion of int with 100 elements, useIterator: true 95 96 1 0.1 9534.4 0.1X
+Colletion of string with 1 elements, useIterator: true 8 8 0 1.3 778.5 0.8X
+Colletion of string with 10 elements, useIterator: true 22 23 0 0.5 2215.9 0.3X
+Colletion of string with 100 elements, useIterator: true 163 164 1 0.1 16325.6 0.0X
+Colletion of Array[int] with 1 elements, useIterator: true 7 8 0 1.4 737.7 0.8X
+Colletion of Array[int] with 10 elements, useIterator: true 20 20 0 0.5 1960.3 0.3X
+Colletion of Array[int] with 100 elements, useIterator: true 146 147 1 0.1 14615.3 0.0X
+Colletion of int with 1 elements, useIterator: false 6 7 0 1.6 613.8 1.0X
+Colletion of int with 10 elements, useIterator: false 14 14 0 0.7 1366.8 0.5X
+Colletion of int with 100 elements, useIterator: false 86 87 1 0.1 8554.3 0.1X
+Colletion of string with 1 elements, useIterator: false 7 8 0 1.3 748.0 0.8X
+Colletion of string with 10 elements, useIterator: false 21 22 0 0.5 2129.9 0.3X
+Colletion of string with 100 elements, useIterator: false 160 161 0 0.1 15969.2 0.0X
+Colletion of Array[int] with 1 elements, useIterator: false 7 8 0 1.4 695.4 0.9X
+Colletion of Array[int] with 10 elements, useIterator: false 19 19 0 0.5 1872.0 0.3X
+Colletion of Array[int] with 100 elements, useIterator: false 139 140 1 0.1 13860.5 0.0X
diff --git a/core/benchmarks/KryoIteratorBenchmark-results.txt b/core/benchmarks/KryoIteratorBenchmark-results.txt
index 6caa842d0e4d2..edee20ae4be6c 100644
--- a/core/benchmarks/KryoIteratorBenchmark-results.txt
+++ b/core/benchmarks/KryoIteratorBenchmark-results.txt
@@ -2,27 +2,27 @@
Benchmark of kryo asIterator on deserialization stream
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark of kryo asIterator on deserialization stream: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------------------------
-Colletion of int with 1 elements, useIterator: true 6 6 0 1.6 629.3 1.0X
-Colletion of int with 10 elements, useIterator: true 14 14 0 0.7 1350.0 0.5X
-Colletion of int with 100 elements, useIterator: true 83 83 1 0.1 8255.8 0.1X
-Colletion of string with 1 elements, useIterator: true 8 8 0 1.3 750.8 0.8X
-Colletion of string with 10 elements, useIterator: true 21 21 1 0.5 2116.4 0.3X
-Colletion of string with 100 elements, useIterator: true 162 163 1 0.1 16191.1 0.0X
-Colletion of Array[int] with 1 elements, useIterator: true 7 8 0 1.4 732.3 0.9X
-Colletion of Array[int] with 10 elements, useIterator: true 19 19 0 0.5 1906.4 0.3X
-Colletion of Array[int] with 100 elements, useIterator: true 142 143 0 0.1 14222.9 0.0X
-Colletion of int with 1 elements, useIterator: false 6 6 0 1.7 604.6 1.0X
-Colletion of int with 10 elements, useIterator: false 13 13 0 0.8 1325.8 0.5X
-Colletion of int with 100 elements, useIterator: false 83 83 0 0.1 8261.2 0.1X
-Colletion of string with 1 elements, useIterator: false 7 8 1 1.4 719.2 0.9X
-Colletion of string with 10 elements, useIterator: false 22 22 0 0.5 2203.0 0.3X
-Colletion of string with 100 elements, useIterator: false 163 163 1 0.1 16294.8 0.0X
-Colletion of Array[int] with 1 elements, useIterator: false 7 7 0 1.5 674.8 0.9X
-Colletion of Array[int] with 10 elements, useIterator: false 18 19 0 0.6 1808.7 0.3X
-Colletion of Array[int] with 100 elements, useIterator: false 135 135 0 0.1 13481.7 0.0X
+Colletion of int with 1 elements, useIterator: true 7 8 1 1.4 727.2 1.0X
+Colletion of int with 10 elements, useIterator: true 14 15 1 0.7 1404.9 0.5X
+Colletion of int with 100 elements, useIterator: true 84 86 2 0.1 8356.0 0.1X
+Colletion of string with 1 elements, useIterator: true 8 9 1 1.2 831.5 0.9X
+Colletion of string with 10 elements, useIterator: true 23 24 1 0.4 2269.6 0.3X
+Colletion of string with 100 elements, useIterator: true 165 169 2 0.1 16465.8 0.0X
+Colletion of Array[int] with 1 elements, useIterator: true 8 9 2 1.2 815.2 0.9X
+Colletion of Array[int] with 10 elements, useIterator: true 21 22 1 0.5 2094.8 0.3X
+Colletion of Array[int] with 100 elements, useIterator: true 163 167 3 0.1 16329.2 0.0X
+Colletion of int with 1 elements, useIterator: false 7 8 1 1.5 686.2 1.1X
+Colletion of int with 10 elements, useIterator: false 15 16 1 0.7 1453.1 0.5X
+Colletion of int with 100 elements, useIterator: false 86 87 1 0.1 8577.6 0.1X
+Colletion of string with 1 elements, useIterator: false 8 9 1 1.3 790.0 0.9X
+Colletion of string with 10 elements, useIterator: false 23 24 1 0.4 2265.0 0.3X
+Colletion of string with 100 elements, useIterator: false 165 171 3 0.1 16477.9 0.0X
+Colletion of Array[int] with 1 elements, useIterator: false 8 8 0 1.3 752.5 1.0X
+Colletion of Array[int] with 10 elements, useIterator: false 20 22 1 0.5 2040.3 0.4X
+Colletion of Array[int] with 100 elements, useIterator: false 151 154 1 0.1 15091.7 0.0X
diff --git a/core/benchmarks/KryoSerializerBenchmark-jdk21-results.txt b/core/benchmarks/KryoSerializerBenchmark-jdk21-results.txt
index e2c03b0acd23c..b5e0f5ad14a33 100644
--- a/core/benchmarks/KryoSerializerBenchmark-jdk21-results.txt
+++ b/core/benchmarks/KryoSerializerBenchmark-jdk21-results.txt
@@ -2,11 +2,11 @@
Benchmark KryoPool vs old"pool of 1" implementation
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark KryoPool vs old"pool of 1" implementation: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------
-KryoPool:true 3610 5082 1510 0.0 7219633.5 1.0X
-KryoPool:false 5886 7699 1454 0.0 11772046.4 0.6X
+KryoPool:true 3650 5490 1771 0.0 7300894.5 1.0X
+KryoPool:false 6117 8058 1390 0.0 12233636.3 0.6X
diff --git a/core/benchmarks/KryoSerializerBenchmark-results.txt b/core/benchmarks/KryoSerializerBenchmark-results.txt
index 54b21654ce054..a47d35ce596d0 100644
--- a/core/benchmarks/KryoSerializerBenchmark-results.txt
+++ b/core/benchmarks/KryoSerializerBenchmark-results.txt
@@ -2,11 +2,11 @@
Benchmark KryoPool vs old"pool of 1" implementation
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark KryoPool vs old"pool of 1" implementation: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------
-KryoPool:true 3682 5317 1787 0.0 7363182.2 1.0X
-KryoPool:false 5853 7922 1339 0.0 11705848.9 0.6X
+KryoPool:true 3268 5114 1862 0.0 6535165.4 1.0X
+KryoPool:false 5600 7584 1247 0.0 11200022.2 0.6X
diff --git a/core/benchmarks/LZFBenchmark-jdk21-results.txt b/core/benchmarks/LZFBenchmark-jdk21-results.txt
index b96edb0de4fa8..0420f4417b23c 100644
--- a/core/benchmarks/LZFBenchmark-jdk21-results.txt
+++ b/core/benchmarks/LZFBenchmark-jdk21-results.txt
@@ -2,18 +2,18 @@
Benchmark LZFCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Compress small objects: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------
-Compression 256000000 int values in parallel 590 602 11 433.7 2.3 1.0X
-Compression 256000000 int values single-threaded 557 558 1 460.0 2.2 1.1X
+Compression 256000000 int values in parallel 594 606 11 430.9 2.3 1.0X
+Compression 256000000 int values single-threaded 570 572 1 449.0 2.2 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Compress large objects: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Compression 1024 array values in 1 threads 42 47 3 0.0 41185.8 1.0X
-Compression 1024 array values single-threaded 31 31 0 0.0 30214.3 1.4X
+Compression 1024 array values in 2 threads 23 25 1 0.0 22108.7 1.0X
+Compression 1024 array values single-threaded 32 33 0 0.0 31296.0 0.7X
diff --git a/core/benchmarks/LZFBenchmark-results.txt b/core/benchmarks/LZFBenchmark-results.txt
index d999bb12f1fbc..31da8cffbe948 100644
--- a/core/benchmarks/LZFBenchmark-results.txt
+++ b/core/benchmarks/LZFBenchmark-results.txt
@@ -2,18 +2,18 @@
Benchmark LZFCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Compress small objects: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------
-Compression 256000000 int values in parallel 608 619 8 420.8 2.4 1.0X
-Compression 256000000 int values single-threaded 584 586 2 438.3 2.3 1.0X
+Compression 256000000 int values in parallel 593 602 8 431.4 2.3 1.0X
+Compression 256000000 int values single-threaded 619 625 4 413.4 2.4 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Compress large objects: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Compression 1024 array values in 1 threads 37 46 6 0.0 35682.0 1.0X
-Compression 1024 array values single-threaded 32 33 0 0.0 31517.3 1.1X
+Compression 1024 array values in 1 threads 41 49 6 0.0 40204.2 1.0X
+Compression 1024 array values single-threaded 33 33 1 0.0 32005.2 1.3X
diff --git a/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt b/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt
index 2639b7194286f..12b00b218ff91 100644
--- a/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt
+++ b/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt
@@ -2,12 +2,12 @@
MapStatuses Convert Benchmark
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
MapStatuses Convert: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Num Maps: 50000 Fetch partitions:500 699 706 6 0.0 698685201.0 1.0X
-Num Maps: 50000 Fetch partitions:1000 1587 1596 11 0.0 1586755889.0 0.4X
-Num Maps: 50000 Fetch partitions:1500 2442 2479 33 0.0 2442011440.0 0.3X
+Num Maps: 50000 Fetch partitions:500 665 669 5 0.0 664867774.0 1.0X
+Num Maps: 50000 Fetch partitions:1000 1709 1724 13 0.0 1709114289.0 0.4X
+Num Maps: 50000 Fetch partitions:1500 2626 2663 36 0.0 2626369193.0 0.3X
diff --git a/core/benchmarks/MapStatusesConvertBenchmark-results.txt b/core/benchmarks/MapStatusesConvertBenchmark-results.txt
index bd17fb8c53dee..a444f525a3f94 100644
--- a/core/benchmarks/MapStatusesConvertBenchmark-results.txt
+++ b/core/benchmarks/MapStatusesConvertBenchmark-results.txt
@@ -2,12 +2,12 @@
MapStatuses Convert Benchmark
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
MapStatuses Convert: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Num Maps: 50000 Fetch partitions:500 750 763 16 0.0 749869953.0 1.0X
-Num Maps: 50000 Fetch partitions:1000 1826 1851 24 0.0 1825734215.0 0.4X
-Num Maps: 50000 Fetch partitions:1500 2846 2863 17 0.0 2845610666.0 0.3X
+Num Maps: 50000 Fetch partitions:500 718 733 13 0.0 717768191.0 1.0X
+Num Maps: 50000 Fetch partitions:1000 1724 1748 24 0.0 1723517357.0 0.4X
+Num Maps: 50000 Fetch partitions:1500 2652 2676 39 0.0 2652037093.0 0.3X
diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk21-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk21-results.txt
index 936af3f1dd149..291e53118b446 100644
--- a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk21-results.txt
+++ b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk21-results.txt
@@ -1,66 +1,66 @@
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization 85 88 4 2.3 425.9 1.0X
-Deserialization 141 148 8 1.4 707.0 0.6X
+Serialization 92 95 4 2.2 461.2 1.0X
+Deserialization 143 149 9 1.4 717.5 0.6X
-Compressed Serialized MapStatus sizes: 426.0 B
-Compressed Serialized Broadcast MapStatus sizes: 2.5 MiB
+Compressed Serialized MapStatus sizes: 425.0 B
+Compressed Serialized Broadcast MapStatus sizes: 3.2 MiB
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------
-Serialization 75 76 1 2.7 376.2 1.0X
-Deserialization 141 145 9 1.4 706.7 0.5X
+Serialization 86 88 1 2.3 429.9 1.0X
+Deserialization 143 146 6 1.4 716.5 0.6X
-Compressed Serialized MapStatus sizes: 2.5 MiB
+Compressed Serialized MapStatus sizes: 3.2 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------
-Serialization 161 166 8 1.2 805.6 1.0X
-Deserialization 158 163 13 1.3 790.6 1.0X
+Serialization 169 172 9 1.2 844.1 1.0X
+Deserialization 159 162 6 1.3 794.5 1.1X
-Compressed Serialized MapStatus sizes: 442.0 B
-Compressed Serialized Broadcast MapStatus sizes: 13.6 MiB
+Compressed Serialized MapStatus sizes: 443.0 B
+Compressed Serialized Broadcast MapStatus sizes: 14.3 MiB
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Serialization 143 143 0 1.4 712.6 1.0X
-Deserialization 158 164 9 1.3 789.7 0.9X
+Serialization 158 160 1 1.3 791.7 1.0X
+Deserialization 159 162 4 1.3 794.4 1.0X
-Compressed Serialized MapStatus sizes: 13.6 MiB
+Compressed Serialized MapStatus sizes: 14.3 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Serialization 699 725 44 0.3 3494.9 1.0X
-Deserialization 327 344 11 0.6 1633.4 2.1X
+Serialization 692 728 34 0.3 3459.4 1.0X
+Deserialization 333 364 31 0.6 1664.4 2.1X
-Compressed Serialized MapStatus sizes: 570.0 B
-Compressed Serialized Broadcast MapStatus sizes: 122.3 MiB
+Compressed Serialized MapStatus sizes: 566.0 B
+Compressed Serialized Broadcast MapStatus sizes: 123.4 MiB
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------
-Serialization 562 567 5 0.4 2809.3 1.0X
-Deserialization 320 341 18 0.6 1598.2 1.8X
+Serialization 577 583 9 0.3 2886.7 1.0X
+Deserialization 331 358 29 0.6 1654.0 1.7X
-Compressed Serialized MapStatus sizes: 122.3 MiB
+Compressed Serialized MapStatus sizes: 123.4 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt
index 48fe51f7cf938..10bb6d7193b30 100644
--- a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt
+++ b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt
@@ -1,66 +1,66 @@
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization 90 93 3 2.2 449.1 1.0X
-Deserialization 137 144 6 1.5 685.7 0.7X
+Serialization 95 98 2 2.1 476.7 1.0X
+Deserialization 145 150 8 1.4 725.8 0.7X
-Compressed Serialized MapStatus sizes: 426.0 B
-Compressed Serialized Broadcast MapStatus sizes: 2.5 MiB
+Compressed Serialized MapStatus sizes: 425.0 B
+Compressed Serialized Broadcast MapStatus sizes: 3.2 MiB
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------
-Serialization 75 76 1 2.7 376.3 1.0X
-Deserialization 137 142 6 1.5 683.4 0.6X
+Serialization 84 88 6 2.4 420.6 1.0X
+Deserialization 143 147 7 1.4 714.9 0.6X
-Compressed Serialized MapStatus sizes: 2.5 MiB
+Compressed Serialized MapStatus sizes: 3.2 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------
-Serialization 153 158 5 1.3 762.6 1.0X
-Deserialization 153 161 11 1.3 765.5 1.0X
+Serialization 163 170 10 1.2 815.3 1.0X
+Deserialization 161 166 7 1.2 803.7 1.0X
-Compressed Serialized MapStatus sizes: 442.0 B
-Compressed Serialized Broadcast MapStatus sizes: 13.6 MiB
+Compressed Serialized MapStatus sizes: 444.0 B
+Compressed Serialized Broadcast MapStatus sizes: 14.3 MiB
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Serialization 156 160 3 1.3 779.5 1.0X
-Deserialization 153 160 11 1.3 762.6 1.0X
+Serialization 152 156 2 1.3 759.0 1.0X
+Deserialization 162 165 3 1.2 809.9 0.9X
-Compressed Serialized MapStatus sizes: 13.6 MiB
+Compressed Serialized MapStatus sizes: 14.3 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Serialization 691 716 30 0.3 3457.5 1.0X
-Deserialization 330 342 9 0.6 1652.0 2.1X
+Serialization 736 754 19 0.3 3681.9 1.0X
+Deserialization 327 343 12 0.6 1635.2 2.3X
-Compressed Serialized MapStatus sizes: 569.0 B
-Compressed Serialized Broadcast MapStatus sizes: 122.3 MiB
+Compressed Serialized MapStatus sizes: 566.0 B
+Compressed Serialized Broadcast MapStatus sizes: 123.4 MiB
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------
-Serialization 573 581 8 0.3 2866.3 1.0X
-Deserialization 323 336 12 0.6 1616.5 1.8X
+Serialization 577 591 14 0.3 2885.6 1.0X
+Deserialization 328 335 6 0.6 1638.3 1.8X
-Compressed Serialized MapStatus sizes: 122.3 MiB
+Compressed Serialized MapStatus sizes: 123.4 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
diff --git a/core/benchmarks/PercentileHeapBenchmark-jdk21-results.txt b/core/benchmarks/PercentileHeapBenchmark-jdk21-results.txt
index 5d08b56176fbc..6c0384f6e602a 100644
--- a/core/benchmarks/PercentileHeapBenchmark-jdk21-results.txt
+++ b/core/benchmarks/PercentileHeapBenchmark-jdk21-results.txt
@@ -2,40 +2,40 @@
PercentileHeap Operations
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 10000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 1 1 0 10.2 98.3 1.0X
-Percentile: 0.9 1 1 0 17.4 57.3 1.7X
-Percentile: 0.95 0 0 0 22.8 43.9 2.2X
-Percentile: 0.99 0 0 0 30.4 32.9 3.0X
+Percentile: 0.5 1 1 0 9.6 103.9 1.0X
+Percentile: 0.9 1 1 0 17.5 57.2 1.8X
+Percentile: 0.95 0 0 0 22.9 43.7 2.4X
+Percentile: 0.99 0 0 0 30.6 32.7 3.2X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 50000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 5 5 0 9.6 104.6 1.0X
-Percentile: 0.9 3 3 0 16.8 59.6 1.8X
-Percentile: 0.95 2 2 0 20.8 48.1 2.2X
-Percentile: 0.99 2 2 0 29.3 34.2 3.1X
+Percentile: 0.5 6 6 0 8.6 116.1 1.0X
+Percentile: 0.9 3 3 0 16.9 59.3 2.0X
+Percentile: 0.95 2 2 0 22.2 45.0 2.6X
+Percentile: 0.99 2 2 0 29.4 34.0 3.4X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 100000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 12 12 0 8.2 121.5 1.0X
-Percentile: 0.9 6 7 0 15.6 64.1 1.9X
-Percentile: 0.95 5 5 0 19.9 50.1 2.4X
-Percentile: 0.99 4 4 0 28.3 35.3 3.4X
+Percentile: 0.5 13 13 0 7.9 125.8 1.0X
+Percentile: 0.9 7 7 1 15.2 65.7 1.9X
+Percentile: 0.95 5 5 0 20.5 48.8 2.6X
+Percentile: 0.99 4 4 0 27.3 36.6 3.4X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 200000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 26 26 0 7.8 128.2 1.0X
-Percentile: 0.9 14 14 0 14.5 68.8 1.9X
-Percentile: 0.95 11 11 0 19.0 52.7 2.4X
-Percentile: 0.99 7 7 0 27.1 36.9 3.5X
+Percentile: 0.5 26 28 2 7.6 131.9 1.0X
+Percentile: 0.9 14 15 1 14.3 69.8 1.9X
+Percentile: 0.95 10 10 0 19.2 52.1 2.5X
+Percentile: 0.99 7 8 0 27.2 36.8 3.6X
diff --git a/core/benchmarks/PercentileHeapBenchmark-results.txt b/core/benchmarks/PercentileHeapBenchmark-results.txt
index 2967772a8c273..da413900bc673 100644
--- a/core/benchmarks/PercentileHeapBenchmark-results.txt
+++ b/core/benchmarks/PercentileHeapBenchmark-results.txt
@@ -2,40 +2,40 @@
PercentileHeap Operations
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 10000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 1 1 0 9.9 101.1 1.0X
-Percentile: 0.9 1 1 0 17.2 58.3 1.7X
-Percentile: 0.95 0 0 0 22.9 43.6 2.3X
-Percentile: 0.99 0 0 0 30.5 32.8 3.1X
+Percentile: 0.5 1 1 0 10.0 100.1 1.0X
+Percentile: 0.9 1 1 0 17.0 58.9 1.7X
+Percentile: 0.95 0 0 0 22.0 45.5 2.2X
+Percentile: 0.99 0 0 0 28.9 34.6 2.9X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 50000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 5 6 0 9.3 107.0 1.0X
-Percentile: 0.9 3 3 0 16.8 59.4 1.8X
-Percentile: 0.95 2 2 0 21.8 45.9 2.3X
-Percentile: 0.99 2 2 0 28.8 34.8 3.1X
+Percentile: 0.5 5 6 0 9.4 106.0 1.0X
+Percentile: 0.9 3 3 0 16.5 60.4 1.8X
+Percentile: 0.95 3 3 0 19.9 50.4 2.1X
+Percentile: 0.99 2 2 0 26.9 37.1 2.9X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 100000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 12 12 0 8.2 122.2 1.0X
-Percentile: 0.9 6 7 0 15.8 63.5 1.9X
-Percentile: 0.95 5 5 0 20.4 49.0 2.5X
-Percentile: 0.99 3 3 0 28.7 34.9 3.5X
+Percentile: 0.5 12 12 0 8.2 121.9 1.0X
+Percentile: 0.9 7 7 0 15.0 66.7 1.8X
+Percentile: 0.95 5 6 1 19.2 52.0 2.3X
+Percentile: 0.99 4 4 0 27.1 36.9 3.3X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 200000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 26 26 0 7.7 130.1 1.0X
-Percentile: 0.9 14 14 0 14.4 69.4 1.9X
-Percentile: 0.95 10 11 1 19.3 51.8 2.5X
-Percentile: 0.99 7 7 0 27.6 36.2 3.6X
+Percentile: 0.5 26 26 0 7.8 128.2 1.0X
+Percentile: 0.9 14 15 1 13.9 71.7 1.8X
+Percentile: 0.95 11 11 0 18.7 53.5 2.4X
+Percentile: 0.99 8 8 0 26.1 38.4 3.3X
diff --git a/core/benchmarks/PersistenceEngineBenchmark-jdk21-results.txt b/core/benchmarks/PersistenceEngineBenchmark-jdk21-results.txt
index 49e54abebfea6..5ddf5a5f2193d 100644
--- a/core/benchmarks/PersistenceEngineBenchmark-jdk21-results.txt
+++ b/core/benchmarks/PersistenceEngineBenchmark-jdk21-results.txt
@@ -2,17 +2,17 @@
PersistenceEngineBenchmark
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
1000 Workers: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------------------
-ZooKeeperPersistenceEngine with JavaSerializer 6237 6453 236 0.0 6236919.5 1.0X
-FileSystemPersistenceEngine with JavaSerializer 2481 2495 12 0.0 2480642.5 2.5X
-FileSystemPersistenceEngine with JavaSerializer (lz4) 776 805 29 0.0 776056.9 8.0X
-FileSystemPersistenceEngine with JavaSerializer (lzf) 728 733 4 0.0 727884.9 8.6X
-FileSystemPersistenceEngine with JavaSerializer (snappy) 739 753 13 0.0 738838.5 8.4X
-FileSystemPersistenceEngine with JavaSerializer (zstd) 976 989 22 0.0 976199.8 6.4X
-RocksDBPersistenceEngine with JavaSerializer 283 286 4 0.0 282774.2 22.1X
-BlackHolePersistenceEngine 0 0 0 6.0 167.9 37143.3X
+ZooKeeperPersistenceEngine with JavaSerializer 7258 7339 74 0.0 7257580.1 1.0X
+FileSystemPersistenceEngine with JavaSerializer 2398 2417 23 0.0 2398207.0 3.0X
+FileSystemPersistenceEngine with JavaSerializer (lz4) 800 830 32 0.0 800356.5 9.1X
+FileSystemPersistenceEngine with JavaSerializer (lzf) 737 748 10 0.0 736571.2 9.9X
+FileSystemPersistenceEngine with JavaSerializer (snappy) 737 780 42 0.0 737277.3 9.8X
+FileSystemPersistenceEngine with JavaSerializer (zstd) 969 994 25 0.0 968711.2 7.5X
+RocksDBPersistenceEngine with JavaSerializer 280 281 1 0.0 279561.6 26.0X
+BlackHolePersistenceEngine 0 0 0 5.8 172.2 42150.9X
diff --git a/core/benchmarks/PersistenceEngineBenchmark-results.txt b/core/benchmarks/PersistenceEngineBenchmark-results.txt
index 91390591e312b..cf939af91f51d 100644
--- a/core/benchmarks/PersistenceEngineBenchmark-results.txt
+++ b/core/benchmarks/PersistenceEngineBenchmark-results.txt
@@ -2,17 +2,17 @@
PersistenceEngineBenchmark
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
1000 Workers: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------------------
-ZooKeeperPersistenceEngine with JavaSerializer 5746 5947 252 0.0 5746489.4 1.0X
-FileSystemPersistenceEngine with JavaSerializer 2520 2545 23 0.0 2520044.8 2.3X
-FileSystemPersistenceEngine with JavaSerializer (lz4) 782 807 35 0.0 782074.1 7.3X
-FileSystemPersistenceEngine with JavaSerializer (lzf) 731 768 37 0.0 731043.3 7.9X
-FileSystemPersistenceEngine with JavaSerializer (snappy) 817 831 17 0.0 816894.1 7.0X
-FileSystemPersistenceEngine with JavaSerializer (zstd) 950 959 14 0.0 950456.4 6.0X
-RocksDBPersistenceEngine with JavaSerializer 285 287 3 0.0 285388.0 20.1X
-BlackHolePersistenceEngine 0 0 0 5.9 170.2 33755.4X
+ZooKeeperPersistenceEngine with JavaSerializer 7382 7798 477 0.0 7382012.2 1.0X
+FileSystemPersistenceEngine with JavaSerializer 2502 2522 18 0.0 2502330.2 3.0X
+FileSystemPersistenceEngine with JavaSerializer (lz4) 788 819 27 0.0 787852.0 9.4X
+FileSystemPersistenceEngine with JavaSerializer (lzf) 725 763 38 0.0 724713.5 10.2X
+FileSystemPersistenceEngine with JavaSerializer (snappy) 739 754 16 0.0 738849.4 10.0X
+FileSystemPersistenceEngine with JavaSerializer (zstd) 949 966 27 0.0 949243.9 7.8X
+RocksDBPersistenceEngine with JavaSerializer 294 296 2 0.0 293576.3 25.1X
+BlackHolePersistenceEngine 0 0 0 6.4 156.9 47039.0X
diff --git a/core/benchmarks/PropertiesCloneBenchmark-jdk21-results.txt b/core/benchmarks/PropertiesCloneBenchmark-jdk21-results.txt
index 0dda0a89b6119..23d6905999ad8 100644
--- a/core/benchmarks/PropertiesCloneBenchmark-jdk21-results.txt
+++ b/core/benchmarks/PropertiesCloneBenchmark-jdk21-results.txt
@@ -2,39 +2,39 @@
Properties Cloning
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Empty Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.3 3145.0 1.0X
-Utils.cloneProperties 0 0 0 11.2 89.0 35.3X
+SerializationUtils.clone 0 0 0 0.3 3166.0 1.0X
+Utils.cloneProperties 0 0 0 11.2 89.0 35.6X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
System Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.0 148318.0 1.0X
-Utils.cloneProperties 0 0 0 0.2 4708.0 31.5X
+SerializationUtils.clone 0 0 0 0.0 163897.0 1.0X
+Utils.cloneProperties 0 0 0 0.2 4909.0 33.4X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Small Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.0 241993.0 1.0X
-Utils.cloneProperties 0 0 0 0.1 6812.0 35.5X
+SerializationUtils.clone 0 0 0 0.0 239028.0 1.0X
+Utils.cloneProperties 0 0 0 0.2 6011.0 39.8X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Medium Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 1 1 0 0.0 858356.0 1.0X
-Utils.cloneProperties 0 0 0 0.0 29876.0 28.7X
+SerializationUtils.clone 1 1 0 0.0 840785.0 1.0X
+Utils.cloneProperties 0 0 0 0.0 30567.0 27.5X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Large Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 2 2 0 0.0 1612428.0 1.0X
-Utils.cloneProperties 0 0 0 0.0 59872.0 26.9X
+SerializationUtils.clone 2 2 0 0.0 1587893.0 1.0X
+Utils.cloneProperties 0 0 0 0.0 60934.0 26.1X
diff --git a/core/benchmarks/PropertiesCloneBenchmark-results.txt b/core/benchmarks/PropertiesCloneBenchmark-results.txt
index 2cf84524d1c39..dec4955b45151 100644
--- a/core/benchmarks/PropertiesCloneBenchmark-results.txt
+++ b/core/benchmarks/PropertiesCloneBenchmark-results.txt
@@ -2,39 +2,39 @@
Properties Cloning
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Empty Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.3 3146.0 1.0X
-Utils.cloneProperties 0 0 0 11.1 90.0 35.0X
+SerializationUtils.clone 0 0 0 0.3 3145.0 1.0X
+Utils.cloneProperties 0 0 0 11.2 89.0 35.3X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
System Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.0 150510.0 1.0X
-Utils.cloneProperties 0 0 0 0.2 4017.0 37.5X
+SerializationUtils.clone 0 0 0 0.0 173615.0 1.0X
+Utils.cloneProperties 0 0 0 0.2 4308.0 40.3X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Small Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.0 250416.0 1.0X
-Utils.cloneProperties 0 0 0 0.1 7914.0 31.6X
+SerializationUtils.clone 0 0 0 0.0 252443.0 1.0X
+Utils.cloneProperties 0 0 0 0.1 10870.0 23.2X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Medium Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 1 1 0 0.0 853339.0 1.0X
-Utils.cloneProperties 0 0 0 0.0 39814.0 21.4X
+SerializationUtils.clone 1 1 0 0.0 866693.0 1.0X
+Utils.cloneProperties 0 0 0 0.0 53982.0 16.1X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Large Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 2 2 0 0.0 1604705.0 1.0X
-Utils.cloneProperties 0 0 0 0.0 78226.0 20.5X
+SerializationUtils.clone 2 2 0 0.0 1628029.0 1.0X
+Utils.cloneProperties 0 0 0 0.0 108032.0 15.1X
diff --git a/core/benchmarks/RocksDBBenchmark-jdk21-results.txt b/core/benchmarks/RocksDBBenchmark-jdk21-results.txt
new file mode 100644
index 0000000000000..440ab1dbfa046
--- /dev/null
+++ b/core/benchmarks/RocksDBBenchmark-jdk21-results.txt
@@ -0,0 +1,102 @@
+================================================================================================
+RocksDB Lifecycle Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+RocksDB Lifecycle Operations: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+DB Creation 5 6 1 0.0 5297960.0 1.0X
+DB Close 0 0 0 0.0 152524.0 34.7X
+
+
+================================================================================================
+Sequential Operations Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Writes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 23 24 1 0.0 22472.9 1.0X
+No Index 17 18 0 0.1 16869.5 1.3X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Updates: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 22 23 1 0.0 21651.8 1.0X
+No Index 10 11 0 0.1 10027.6 2.2X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Deletes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 11 12 0 0.1 10698.4 1.0X
+No Index 8 8 0 0.1 7430.8 1.4X
+
+
+================================================================================================
+Random Operations Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Writes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 24 25 1 0.0 23615.9 1.0X
+No Index 19 19 0 0.1 18099.8 1.3X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Updates: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 22 23 0 0.0 21808.2 1.0X
+No Index 10 11 1 0.1 10175.5 2.1X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Deletes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 11 12 0 0.1 10694.7 1.0X
+No Index 8 8 0 0.1 7443.8 1.4X
+
+
+================================================================================================
+Natural Index Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Natural Index - Create Iterator: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 0 0 0 1011.9 1.0 1.0X
+Descending 0 0 0 974.3 1.0 1.0X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Natural Index - Iteration: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 1 1 0 0.8 1293.2 1.0X
+Descending 1 2 0 0.8 1294.8 1.0X
+
+
+================================================================================================
+Ref Index Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Ref Index - Create Iterator: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 0 0 0 780.5 1.3 1.0X
+Descending 0 0 0 757.4 1.3 1.0X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Ref Index - Iteration: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 3 3 0 0.4 2749.3 1.0X
+Descending 3 3 1 0.4 2761.9 1.0X
+
+
diff --git a/core/benchmarks/RocksDBBenchmark-results.txt b/core/benchmarks/RocksDBBenchmark-results.txt
new file mode 100644
index 0000000000000..8e7087c9743dd
--- /dev/null
+++ b/core/benchmarks/RocksDBBenchmark-results.txt
@@ -0,0 +1,102 @@
+================================================================================================
+RocksDB Lifecycle Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+RocksDB Lifecycle Operations: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+DB Creation 5 6 1 0.0 4935909.0 1.0X
+DB Close 0 0 0 0.0 150520.0 32.8X
+
+
+================================================================================================
+Sequential Operations Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Writes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 25 26 2 0.0 24236.9 1.0X
+No Index 19 19 0 0.1 18209.6 1.3X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Updates: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 24 24 0 0.0 22973.3 1.0X
+No Index 11 12 0 0.1 10978.6 2.1X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Deletes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 12 12 0 0.1 11322.0 1.0X
+No Index 8 9 1 0.1 8160.9 1.4X
+
+
+================================================================================================
+Random Operations Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Writes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 25 26 1 0.0 24511.0 1.0X
+No Index 19 20 1 0.1 18639.0 1.3X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Updates: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 23 23 0 0.0 22343.3 1.0X
+No Index 11 11 0 0.1 10685.8 2.1X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Deletes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 12 12 0 0.1 11481.2 1.0X
+No Index 8 9 0 0.1 8133.2 1.4X
+
+
+================================================================================================
+Natural Index Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Natural Index - Create Iterator: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 0 0 0 938.6 1.1 1.0X
+Descending 0 0 0 874.5 1.1 0.9X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Natural Index - Iteration: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 1 1 0 0.7 1409.5 1.0X
+Descending 1 2 0 0.7 1421.3 1.0X
+
+
+================================================================================================
+Ref Index Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Ref Index - Create Iterator: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 0 0 0 741.0 1.3 1.0X
+Descending 0 0 0 705.2 1.4 1.0X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Ref Index - Iteration: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 3 3 0 0.4 2744.0 1.0X
+Descending 3 4 1 0.4 2761.7 1.0X
+
+
diff --git a/core/benchmarks/SorterBenchmark-jdk21-results.txt b/core/benchmarks/SorterBenchmark-jdk21-results.txt
index 91b725a0b493b..894d03a6203e4 100644
--- a/core/benchmarks/SorterBenchmark-jdk21-results.txt
+++ b/core/benchmarks/SorterBenchmark-jdk21-results.txt
@@ -2,25 +2,28 @@
key-value pairs sort
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
key-value pairs sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Tuple-sort using Arrays.sort() 17292 17584 413 1.4 691.7 1.0X
-KV-sort using Sorter 21245 21340 134 1.2 849.8 0.8X
+Tuple-sort using Arrays.sort() 16520 16936 588 1.5 660.8 1.0X
+Tuple-sort using Arrays.parallelSort() 8092 8946 1207 3.1 323.7 2.0X
+KV-sort using Sorter 19518 19614 137 1.3 780.7 0.8X
================================================================================================
primitive int array sort
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
-primitive int array sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------
-Java Arrays.sort() on non-primitive int array 14231 14561 468 1.8 569.2 1.0X
-Java Arrays.sort() on primitive int array 2141 2145 6 11.7 85.6 6.6X
-Sorter without key reuse on primitive int array 8607 8612 7 2.9 344.3 1.7X
-Sorter with key reuse on primitive int array 10602 10621 27 2.4 424.1 1.3X
+primitive int array sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+-------------------------------------------------------------------------------------------------------------------------------------
+Java Arrays.sort() on non-primitive int array 14997 15038 57 1.7 599.9 1.0X
+Java Arrays.parallelSort() on non-primitive int array 6581 6588 11 3.8 263.2 2.3X
+Java Arrays.sort() on primitive int array 2139 2139 1 11.7 85.5 7.0X
+Java Arrays.parallelSort() on primitive int array 837 871 55 29.9 33.5 17.9X
+Sorter without key reuse on primitive int array 8723 8724 1 2.9 348.9 1.7X
+Sorter with key reuse on primitive int array 10601 10603 3 2.4 424.0 1.4X
diff --git a/core/benchmarks/SorterBenchmark-results.txt b/core/benchmarks/SorterBenchmark-results.txt
index 6369727a85ca6..90d55c7733650 100644
--- a/core/benchmarks/SorterBenchmark-results.txt
+++ b/core/benchmarks/SorterBenchmark-results.txt
@@ -2,25 +2,28 @@
key-value pairs sort
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
key-value pairs sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Tuple-sort using Arrays.sort() 20545 20560 21 1.2 821.8 1.0X
-KV-sort using Sorter 26696 26797 143 0.9 1067.8 0.8X
+Tuple-sort using Arrays.sort() 18160 18205 63 1.4 726.4 1.0X
+Tuple-sort using Arrays.parallelSort() 10481 10688 292 2.4 419.3 1.7X
+KV-sort using Sorter 22496 22732 335 1.1 899.8 0.8X
================================================================================================
primitive int array sort
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
-primitive int array sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------
-Java Arrays.sort() on non-primitive int array 15325 15510 263 1.6 613.0 1.0X
-Java Arrays.sort() on primitive int array 2088 2141 76 12.0 83.5 7.3X
-Sorter without key reuse on primitive int array 8254 8262 10 3.0 330.2 1.9X
-Sorter with key reuse on primitive int array 10184 10186 3 2.5 407.3 1.5X
+primitive int array sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+-------------------------------------------------------------------------------------------------------------------------------------
+Java Arrays.sort() on non-primitive int array 14691 14734 62 1.7 587.6 1.0X
+Java Arrays.parallelSort() on non-primitive int array 7027 7521 698 3.6 281.1 2.1X
+Java Arrays.sort() on primitive int array 2166 2167 1 11.5 86.7 6.8X
+Java Arrays.parallelSort() on primitive int array 897 899 3 27.9 35.9 16.4X
+Sorter without key reuse on primitive int array 8211 8234 31 3.0 328.5 1.8X
+Sorter with key reuse on primitive int array 10170 10174 6 2.5 406.8 1.4X
diff --git a/core/benchmarks/XORShiftRandomBenchmark-jdk21-results.txt b/core/benchmarks/XORShiftRandomBenchmark-jdk21-results.txt
index faeb60e44af10..af5b5416fef2f 100644
--- a/core/benchmarks/XORShiftRandomBenchmark-jdk21-results.txt
+++ b/core/benchmarks/XORShiftRandomBenchmark-jdk21-results.txt
@@ -2,43 +2,43 @@
Pseudo random
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextInt: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
java.util.Random 453 453 0 220.7 4.5 1.0X
XORShiftRandom 186 186 0 536.7 1.9 2.4X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextLong: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 901 901 0 111.0 9.0 1.0X
-XORShiftRandom 373 373 0 268.2 3.7 2.4X
+java.util.Random 901 902 1 111.0 9.0 1.0X
+XORShiftRandom 468 472 4 213.6 4.7 1.9X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextDouble: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
java.util.Random 905 905 0 110.5 9.0 1.0X
-XORShiftRandom 373 375 3 268.2 3.7 2.4X
+XORShiftRandom 486 488 2 205.6 4.9 1.9X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextGaussian: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 3425 3428 3 29.2 34.3 1.0X
-XORShiftRandom 2487 2495 9 40.2 24.9 1.4X
+java.util.Random 3445 3447 3 29.0 34.4 1.0X
+XORShiftRandom 2620 2621 1 38.2 26.2 1.3X
================================================================================================
hash seed
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Hash seed: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-XORShiftRandom.hashSeed 1 1 0 12973.9 0.1 1.0X
+XORShiftRandom.hashSeed 1 1 0 12856.9 0.1 1.0X
diff --git a/core/benchmarks/XORShiftRandomBenchmark-results.txt b/core/benchmarks/XORShiftRandomBenchmark-results.txt
index 93f9adebb1e44..12ef298b2a937 100644
--- a/core/benchmarks/XORShiftRandomBenchmark-results.txt
+++ b/core/benchmarks/XORShiftRandomBenchmark-results.txt
@@ -2,43 +2,43 @@
Pseudo random
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextInt: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
java.util.Random 441 441 0 226.6 4.4 1.0X
XORShiftRandom 186 186 0 536.7 1.9 2.4X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextLong: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 913 914 0 109.5 9.1 1.0X
-XORShiftRandom 373 373 0 268.4 3.7 2.5X
+java.util.Random 902 903 1 110.9 9.0 1.0X
+XORShiftRandom 373 373 1 268.1 3.7 2.4X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextDouble: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 909 910 1 110.0 9.1 1.0X
-XORShiftRandom 373 373 0 268.3 3.7 2.4X
+java.util.Random 922 923 1 108.4 9.2 1.0X
+XORShiftRandom 373 373 0 268.2 3.7 2.5X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextGaussian: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 4031 4035 3 24.8 40.3 1.0X
-XORShiftRandom 2952 2965 19 33.9 29.5 1.4X
+java.util.Random 4148 4148 0 24.1 41.5 1.0X
+XORShiftRandom 3013 3014 2 33.2 30.1 1.4X
================================================================================================
hash seed
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Hash seed: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-XORShiftRandom.hashSeed 2 2 0 6455.0 0.2 1.0X
+XORShiftRandom.hashSeed 2 2 0 6455.8 0.2 1.0X
diff --git a/core/benchmarks/ZStandardBenchmark-jdk21-results.txt b/core/benchmarks/ZStandardBenchmark-jdk21-results.txt
index 41993e9cfa479..c2cdfe1b62df3 100644
--- a/core/benchmarks/ZStandardBenchmark-jdk21-results.txt
+++ b/core/benchmarks/ZStandardBenchmark-jdk21-results.txt
@@ -2,48 +2,81 @@
Benchmark ZStandardCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------------
-Compression 10000 times at level 1 without buffer pool 657 673 15 0.0 65664.9 1.0X
-Compression 10000 times at level 2 without buffer pool 718 726 13 0.0 71830.9 0.9X
-Compression 10000 times at level 3 without buffer pool 815 819 7 0.0 81453.1 0.8X
-Compression 10000 times at level 1 with buffer pool 598 600 1 0.0 59809.3 1.1X
-Compression 10000 times at level 2 with buffer pool 637 639 2 0.0 63710.0 1.0X
-Compression 10000 times at level 3 with buffer pool 754 757 3 0.0 75403.8 0.9X
-
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+Compression 10000 times at level 1 without buffer pool 651 668 15 0.0 65142.0 1.0X
+Compression 10000 times at level 2 without buffer pool 713 714 1 0.0 71290.7 0.9X
+Compression 10000 times at level 3 without buffer pool 806 810 7 0.0 80567.3 0.8X
+Compression 10000 times at level 1 with buffer pool 606 606 0 0.0 60551.9 1.1X
+Compression 10000 times at level 2 with buffer pool 635 636 0 0.0 63549.8 1.0X
+Compression 10000 times at level 3 with buffer pool 751 758 7 0.0 75071.4 0.9X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------------------
-Decompression 10000 times from level 1 without buffer pool 832 835 2 0.0 83247.0 1.0X
-Decompression 10000 times from level 2 without buffer pool 833 840 8 0.0 83277.8 1.0X
-Decompression 10000 times from level 3 without buffer pool 833 833 0 0.0 83280.6 1.0X
-Decompression 10000 times from level 1 with buffer pool 753 755 2 0.0 75333.7 1.1X
-Decompression 10000 times from level 2 with buffer pool 751 752 1 0.0 75115.3 1.1X
-Decompression 10000 times from level 3 with buffer pool 753 754 2 0.0 75254.0 1.1X
-
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+Decompression 10000 times from level 1 without buffer pool 816 817 1 0.0 81628.9 1.0X
+Decompression 10000 times from level 2 without buffer pool 813 815 2 0.0 81287.7 1.0X
+Decompression 10000 times from level 3 without buffer pool 814 816 1 0.0 81442.8 1.0X
+Decompression 10000 times from level 1 with buffer pool 748 749 1 0.0 74798.1 1.1X
+Decompression 10000 times from level 2 with buffer pool 747 747 0 0.0 74672.1 1.1X
+Decompression 10000 times from level 3 with buffer pool 745 746 1 0.0 74489.0 1.1X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 77 78 1 0.0 601317.6 1.0X
-Parallel Compression with 1 workers 64 70 4 0.0 499456.2 1.2X
-Parallel Compression with 2 workers 53 56 3 0.0 410610.6 1.5X
-Parallel Compression with 4 workers 45 47 1 0.0 350847.8 1.7X
-Parallel Compression with 8 workers 47 49 1 0.0 370647.8 1.6X
-Parallel Compression with 16 workers 50 52 1 0.0 390524.8 1.5X
-
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+Parallel Compression with 0 workers 65 66 1 0.0 509138.5 1.0X
+Parallel Compression with 1 workers 52 57 3 0.0 409248.7 1.2X
+Parallel Compression with 2 workers 44 46 1 0.0 346879.2 1.5X
+Parallel Compression with 4 workers 41 42 1 0.0 321240.6 1.6X
+Parallel Compression with 8 workers 43 45 1 0.0 332129.6 1.5X
+Parallel Compression with 16 workers 45 48 1 0.0 351091.1 1.5X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 243 244 2 0.0 1894990.4 1.0X
-Parallel Compression with 1 workers 299 300 2 0.0 2335128.6 0.8X
-Parallel Compression with 2 workers 167 177 11 0.0 1308212.2 1.4X
-Parallel Compression with 4 workers 161 163 3 0.0 1254638.0 1.5X
-Parallel Compression with 8 workers 166 170 4 0.0 1299104.0 1.5X
-Parallel Compression with 16 workers 167 170 2 0.0 1301666.4 1.5X
+Parallel Compression with 0 workers 229 230 0 0.0 1791804.4 1.0X
+Parallel Compression with 1 workers 250 252 2 0.0 1955552.0 0.9X
+Parallel Compression with 2 workers 140 150 22 0.0 1092933.3 1.6X
+Parallel Compression with 4 workers 130 135 3 0.0 1018288.4 1.8X
+Parallel Compression with 8 workers 135 139 2 0.0 1055476.2 1.7X
+Parallel Compression with 16 workers 136 138 2 0.0 1065506.9 1.7X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Compression at level 1: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 53 54 1 0.0 413563.3 1.0X
+Compression by strategy 1 53 54 0 0.0 413024.8 1.0X
+Compression by strategy 3 88 88 0 0.0 687058.0 0.6X
+Compression by strategy 5 227 228 0 0.0 1773158.9 0.2X
+Compression by strategy 7 254 255 0 0.0 1983889.4 0.2X
+Compression by strategy 9 254 254 0 0.0 1980912.4 0.2X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 64 66 1 0.0 501557.7 1.0X
+Compression by strategy 1 49 51 1 0.0 384968.8 1.3X
+Compression by strategy 3 86 87 1 0.0 674617.3 0.7X
+Compression by strategy 5 228 229 1 0.0 1780155.1 0.3X
+Compression by strategy 7 273 274 0 0.0 2133499.5 0.2X
+Compression by strategy 9 273 273 1 0.0 2131740.8 0.2X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 229 230 0 0.0 1792203.1 1.0X
+Compression by strategy 1 49 50 1 0.0 385407.3 4.7X
+Compression by strategy 3 86 87 1 0.0 672585.7 2.7X
+Compression by strategy 5 230 231 1 0.0 1795155.1 1.0X
+Compression by strategy 7 274 274 0 0.0 2138474.2 0.8X
+Compression by strategy 9 274 275 0 0.0 2144319.6 0.8X
diff --git a/core/benchmarks/ZStandardBenchmark-results.txt b/core/benchmarks/ZStandardBenchmark-results.txt
index 66aa149515a2e..9051cea3a3e26 100644
--- a/core/benchmarks/ZStandardBenchmark-results.txt
+++ b/core/benchmarks/ZStandardBenchmark-results.txt
@@ -2,48 +2,81 @@
Benchmark ZStandardCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
-AMD EPYC 7763 64-Core Processor
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------------
-Compression 10000 times at level 1 without buffer pool 651 654 3 0.0 65120.6 1.0X
-Compression 10000 times at level 2 without buffer pool 701 702 1 0.0 70069.5 0.9X
-Compression 10000 times at level 3 without buffer pool 788 790 2 0.0 78845.5 0.8X
-Compression 10000 times at level 1 with buffer pool 577 578 2 0.0 57688.1 1.1X
-Compression 10000 times at level 2 with buffer pool 607 608 1 0.0 60653.3 1.1X
-Compression 10000 times at level 3 with buffer pool 716 717 0 0.0 71634.8 0.9X
-
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
-AMD EPYC 7763 64-Core Processor
+Compression 10000 times at level 1 without buffer pool 190 191 0 0.1 18996.4 1.0X
+Compression 10000 times at level 2 without buffer pool 214 215 0 0.0 21407.3 0.9X
+Compression 10000 times at level 3 without buffer pool 811 812 1 0.0 81094.7 0.2X
+Compression 10000 times at level 1 with buffer pool 640 640 0 0.0 63962.8 0.3X
+Compression 10000 times at level 2 with buffer pool 661 663 1 0.0 66123.6 0.3X
+Compression 10000 times at level 3 with buffer pool 758 758 0 0.0 75773.8 0.3X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------------------
-Decompression 10000 times from level 1 without buffer pool 621 621 0 0.0 62063.9 1.0X
-Decompression 10000 times from level 2 without buffer pool 620 621 1 0.0 62003.0 1.0X
-Decompression 10000 times from level 3 without buffer pool 620 622 1 0.0 62017.4 1.0X
-Decompression 10000 times from level 1 with buffer pool 544 544 0 0.0 54377.6 1.1X
-Decompression 10000 times from level 2 with buffer pool 544 544 1 0.0 54404.3 1.1X
-Decompression 10000 times from level 3 with buffer pool 543 544 1 0.0 54310.2 1.1X
-
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
-AMD EPYC 7763 64-Core Processor
+Decompression 10000 times from level 1 without buffer pool 166 167 0 0.1 16645.6 1.0X
+Decompression 10000 times from level 2 without buffer pool 166 166 0 0.1 16558.6 1.0X
+Decompression 10000 times from level 3 without buffer pool 166 167 0 0.1 16629.4 1.0X
+Decompression 10000 times from level 1 with buffer pool 596 597 1 0.0 59613.2 0.3X
+Decompression 10000 times from level 2 with buffer pool 596 596 0 0.0 59576.6 0.3X
+Decompression 10000 times from level 3 with buffer pool 596 596 0 0.0 59585.5 0.3X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 77 79 1 0.0 603481.7 1.0X
-Parallel Compression with 1 workers 65 68 3 0.0 504835.7 1.2X
-Parallel Compression with 2 workers 50 55 3 0.0 393366.7 1.5X
-Parallel Compression with 4 workers 45 48 2 0.0 352491.0 1.7X
-Parallel Compression with 8 workers 46 49 1 0.0 362934.5 1.7X
-Parallel Compression with 16 workers 50 53 1 0.0 393219.4 1.5X
-
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
-AMD EPYC 7763 64-Core Processor
+Parallel Compression with 0 workers 71 72 0 0.0 553736.9 1.0X
+Parallel Compression with 1 workers 54 57 3 0.0 418869.9 1.3X
+Parallel Compression with 2 workers 45 47 1 0.0 354347.0 1.6X
+Parallel Compression with 4 workers 44 45 1 0.0 341461.1 1.6X
+Parallel Compression with 8 workers 44 46 1 0.0 343621.1 1.6X
+Parallel Compression with 16 workers 46 49 1 0.0 361864.3 1.5X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 237 239 1 0.0 1852985.0 1.0X
-Parallel Compression with 1 workers 299 301 2 0.0 2339270.2 0.8X
-Parallel Compression with 2 workers 167 174 6 0.0 1303651.0 1.4X
-Parallel Compression with 4 workers 160 165 3 0.0 1248593.3 1.5X
-Parallel Compression with 8 workers 166 170 3 0.0 1295084.3 1.4X
-Parallel Compression with 16 workers 165 169 3 0.0 1288571.2 1.4X
+Parallel Compression with 0 workers 257 258 1 0.0 2005669.8 1.0X
+Parallel Compression with 1 workers 270 271 2 0.0 2109299.0 1.0X
+Parallel Compression with 2 workers 154 159 4 0.0 1203914.0 1.7X
+Parallel Compression with 4 workers 134 141 3 0.0 1050495.0 1.9X
+Parallel Compression with 8 workers 141 145 2 0.0 1100891.4 1.8X
+Parallel Compression with 16 workers 141 144 2 0.0 1099121.7 1.8X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Compression at level 1: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 58 60 1 0.0 456774.1 1.0X
+Compression by strategy 1 59 60 1 0.0 461118.8 1.0X
+Compression by strategy 3 97 98 1 0.0 756127.9 0.6X
+Compression by strategy 5 255 256 1 0.0 1990790.0 0.2X
+Compression by strategy 7 281 282 1 0.0 2191773.0 0.2X
+Compression by strategy 9 281 282 1 0.0 2191917.5 0.2X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 71 72 1 0.0 554718.3 1.0X
+Compression by strategy 1 54 55 1 0.0 425679.9 1.3X
+Compression by strategy 3 96 97 1 0.0 751904.8 0.7X
+Compression by strategy 5 256 257 1 0.0 1997970.2 0.3X
+Compression by strategy 7 307 308 1 0.0 2400837.0 0.2X
+Compression by strategy 9 307 308 1 0.0 2397268.7 0.2X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 257 258 1 0.0 2006904.2 1.0X
+Compression by strategy 1 55 56 1 0.0 427945.1 4.7X
+Compression by strategy 3 96 97 1 0.0 748419.3 2.7X
+Compression by strategy 5 257 258 1 0.0 2006747.8 1.0X
+Compression by strategy 7 306 307 1 0.0 2391141.9 0.8X
+Compression by strategy 9 306 307 0 0.0 2394100.3 0.8X
diff --git a/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt b/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt
index 58be348a69c9b..7630433a2f7a4 100644
--- a/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt
+++ b/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt
@@ -2,48 +2,48 @@
Benchmark ZStandardCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------------
-Compression 4 times at level 1 without buffer pool 2852 2858 10 0.0 712887837.3 1.0X
-Compression 4 times at level 2 without buffer pool 4417 4418 1 0.0 1104333096.8 0.6X
-Compression 4 times at level 3 without buffer pool 6594 6602 12 0.0 1648568689.0 0.4X
-Compression 4 times at level 1 with buffer pool 2875 2876 2 0.0 718686835.3 1.0X
-Compression 4 times at level 2 with buffer pool 4415 4422 10 0.0 1103705078.0 0.6X
-Compression 4 times at level 3 with buffer pool 6749 6774 35 0.0 1687199785.0 0.4X
+Compression 4 times at level 1 without buffer pool 2567 2573 9 0.0 641834272.8 1.0X
+Compression 4 times at level 2 without buffer pool 4171 4173 2 0.0 1042717610.7 0.6X
+Compression 4 times at level 3 without buffer pool 6202 6205 4 0.0 1550416592.3 0.4X
+Compression 4 times at level 1 with buffer pool 2562 2562 0 0.0 640541340.8 1.0X
+Compression 4 times at level 2 with buffer pool 4172 4173 1 0.0 1043037897.5 0.6X
+Compression 4 times at level 3 with buffer pool 6199 6203 4 0.0 1549874630.3 0.4X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------------
-Decompression 4 times from level 1 without buffer pool 1084 1140 79 0.0 271079599.8 1.0X
-Decompression 4 times from level 2 without buffer pool 1341 1363 31 0.0 335200382.3 0.8X
-Decompression 4 times from level 3 without buffer pool 1620 1632 18 0.0 404881883.0 0.7X
-Decompression 4 times from level 1 with buffer pool 1110 1144 48 0.0 277478874.5 1.0X
-Decompression 4 times from level 2 with buffer pool 1306 1363 80 0.0 326419359.8 0.8X
-Decompression 4 times from level 3 with buffer pool 1566 1585 27 0.0 391420983.5 0.7X
+Decompression 4 times from level 1 without buffer pool 856 875 17 0.0 214044635.0 1.0X
+Decompression 4 times from level 2 without buffer pool 1141 1147 10 0.0 285134371.0 0.8X
+Decompression 4 times from level 3 without buffer pool 1345 1345 0 0.0 336239892.8 0.6X
+Decompression 4 times from level 1 with buffer pool 844 851 12 0.0 210917530.8 1.0X
+Decompression 4 times from level 2 with buffer pool 1111 1126 22 0.0 277630951.0 0.8X
+Decompression 4 times from level 3 with buffer pool 1338 1348 13 0.0 334616619.0 0.6X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 2160 2166 9 0.0 539942027.3 1.0X
-Parallel Compression with 1 workers 2245 2248 4 0.0 561232751.0 1.0X
-Parallel Compression with 2 workers 1180 1183 4 0.0 295021164.8 1.8X
-Parallel Compression with 4 workers 832 840 12 0.0 207891005.5 2.6X
-Parallel Compression with 8 workers 971 1002 44 0.0 242832737.3 2.2X
-Parallel Compression with 16 workers 1329 1365 51 0.0 332270317.7 1.6X
+Parallel Compression with 0 workers 2099 2099 0 0.0 524768920.0 1.0X
+Parallel Compression with 1 workers 2119 2119 0 0.0 529758581.0 1.0X
+Parallel Compression with 2 workers 1087 1090 4 0.0 271687124.0 1.9X
+Parallel Compression with 4 workers 761 764 2 0.0 190289606.5 2.8X
+Parallel Compression with 8 workers 782 789 7 0.0 195588720.8 2.7X
+Parallel Compression with 16 workers 897 908 10 0.0 224272497.0 2.3X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 9168 9276 154 0.0 2291883601.5 1.0X
-Parallel Compression with 1 workers 8142 8244 145 0.0 2035422527.5 1.1X
-Parallel Compression with 2 workers 4703 4939 334 0.0 1175760328.8 1.9X
-Parallel Compression with 4 workers 4366 4426 84 0.0 1091576971.0 2.1X
-Parallel Compression with 8 workers 5395 5716 453 0.0 1348844072.5 1.7X
-Parallel Compression with 16 workers 6410 6573 230 0.0 1602434231.8 1.4X
+Parallel Compression with 0 workers 8240 8242 2 0.0 2060062709.3 1.0X
+Parallel Compression with 1 workers 6915 6924 13 0.0 1728757390.0 1.2X
+Parallel Compression with 2 workers 3663 3664 1 0.0 915832528.8 2.2X
+Parallel Compression with 4 workers 3239 3247 12 0.0 809829698.3 2.5X
+Parallel Compression with 8 workers 3594 3610 23 0.0 898542204.8 2.3X
+Parallel Compression with 16 workers 3888 3898 14 0.0 972017914.3 2.1X
diff --git a/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt b/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt
index cc439b099a6c5..01784a686f818 100644
--- a/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt
+++ b/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt
@@ -2,48 +2,48 @@
Benchmark ZStandardCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
-AMD EPYC 7763 64-Core Processor
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------------
-Compression 4 times at level 1 without buffer pool 2582 2583 1 0.0 645586401.5 1.0X
-Compression 4 times at level 2 without buffer pool 4128 4132 6 0.0 1032017030.8 0.6X
-Compression 4 times at level 3 without buffer pool 6222 6229 11 0.0 1555453651.0 0.4X
-Compression 4 times at level 1 with buffer pool 2581 2581 1 0.0 645260104.3 1.0X
-Compression 4 times at level 2 with buffer pool 4123 4129 9 0.0 1030789387.7 0.6X
-Compression 4 times at level 3 with buffer pool 6290 6299 13 0.0 1572478346.5 0.4X
+Compression 4 times at level 1 without buffer pool 2513 2519 10 0.0 628127696.5 1.0X
+Compression 4 times at level 2 without buffer pool 3914 3916 3 0.0 978555442.0 0.6X
+Compression 4 times at level 3 without buffer pool 5417 5424 9 0.0 1354342306.3 0.5X
+Compression 4 times at level 1 with buffer pool 2524 2526 4 0.0 630890054.5 1.0X
+Compression 4 times at level 2 with buffer pool 3911 3911 0 0.0 977705511.5 0.6X
+Compression 4 times at level 3 with buffer pool 5447 5451 5 0.0 1361830044.0 0.5X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
-AMD EPYC 7763 64-Core Processor
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------------
-Decompression 4 times from level 1 without buffer pool 898 902 4 0.0 224453665.0 1.0X
-Decompression 4 times from level 2 without buffer pool 1170 1174 5 0.0 292566849.8 0.8X
-Decompression 4 times from level 3 without buffer pool 1409 1411 2 0.0 352317771.3 0.6X
-Decompression 4 times from level 1 with buffer pool 900 901 2 0.0 224956421.3 1.0X
-Decompression 4 times from level 2 with buffer pool 1169 1174 7 0.0 292235703.0 0.8X
-Decompression 4 times from level 3 with buffer pool 1400 1403 3 0.0 350101054.0 0.6X
+Decompression 4 times from level 1 without buffer pool 1053 1056 4 0.0 263342668.0 1.0X
+Decompression 4 times from level 2 without buffer pool 1401 1403 3 0.0 350193821.5 0.8X
+Decompression 4 times from level 3 without buffer pool 1683 1683 1 0.0 420625735.0 0.6X
+Decompression 4 times from level 1 with buffer pool 1078 1079 2 0.0 269507650.5 1.0X
+Decompression 4 times from level 2 with buffer pool 1397 1403 8 0.0 349213546.5 0.8X
+Decompression 4 times from level 3 with buffer pool 1679 1680 2 0.0 419721856.3 0.6X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
-AMD EPYC 7763 64-Core Processor
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz
Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 2088 2098 15 0.0 521933319.5 1.0X
-Parallel Compression with 1 workers 2142 2145 4 0.0 535401971.0 1.0X
-Parallel Compression with 2 workers 1086 1090 5 0.0 271554405.7 1.9X
-Parallel Compression with 4 workers 789 799 10 0.0 197186788.0 2.6X
-Parallel Compression with 8 workers 930 951 18 0.0 232624325.3 2.2X
-Parallel Compression with 16 workers 1011 1014 5 0.0 252634970.3 2.1X
+Parallel Compression with 0 workers 1815 1825 14 0.0 453734902.0 1.0X
+Parallel Compression with 1 workers 1654 1663 12 0.0 413617141.5 1.1X
+Parallel Compression with 2 workers 857 859 2 0.0 214195314.3 2.1X
+Parallel Compression with 4 workers 777 781 3 0.0 194366718.5 2.3X
+Parallel Compression with 8 workers 784 786 3 0.0 195965343.5 2.3X
+Parallel Compression with 16 workers 836 858 19 0.0 208885356.8 2.2X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
-AMD EPYC 7763 64-Core Processor
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz
Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 8766 8812 64 0.0 2191521645.8 1.0X
-Parallel Compression with 1 workers 7808 7810 3 0.0 1951970435.0 1.1X
-Parallel Compression with 2 workers 4057 4096 56 0.0 1014125964.5 2.2X
-Parallel Compression with 4 workers 3508 3528 28 0.0 877027867.8 2.5X
-Parallel Compression with 8 workers 4026 4028 2 0.0 1006539516.7 2.2X
-Parallel Compression with 16 workers 4044 4052 11 0.0 1011070885.5 2.2X
+Parallel Compression with 0 workers 8302 8310 12 0.0 2075384302.0 1.0X
+Parallel Compression with 1 workers 6869 6871 3 0.0 1717196107.5 1.2X
+Parallel Compression with 2 workers 3570 3587 25 0.0 892437927.8 2.3X
+Parallel Compression with 4 workers 3152 3152 1 0.0 787940909.5 2.6X
+Parallel Compression with 8 workers 3463 3470 10 0.0 865739615.8 2.4X
+Parallel Compression with 16 workers 4061 4083 31 0.0 1015251097.3 2.0X
diff --git a/core/src/main/java/org/apache/spark/api/java/Optional.java b/core/src/main/java/org/apache/spark/api/java/Optional.java
index 362149c92145e..913ed966668d4 100644
--- a/core/src/main/java/org/apache/spark/api/java/Optional.java
+++ b/core/src/main/java/org/apache/spark/api/java/Optional.java
@@ -20,8 +20,6 @@
import java.io.Serializable;
import java.util.Objects;
-import com.google.common.base.Preconditions;
-
/**
* Like {@code java.util.Optional} in Java 8, {@code scala.Option} in Scala, and
* {@code com.google.common.base.Optional} in Google Guava, this class represents a
@@ -71,8 +69,7 @@ private Optional() {
}
private Optional(T value) {
- Preconditions.checkNotNull(value);
- this.value = value;
+ this.value = Objects.requireNonNull(value);
}
// java.util.Optional API (subset)
@@ -112,8 +109,7 @@ public static Optional ofNullable(T value) {
* @throws NullPointerException if this is empty (contains no value)
*/
public T get() {
- Preconditions.checkNotNull(value);
- return value;
+ return Objects.requireNonNull(value);
}
/**
diff --git a/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java b/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java
index 7dd87df713e6e..91ee2d8ccbe96 100644
--- a/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java
+++ b/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java
@@ -28,13 +28,11 @@
import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.concurrent.GuardedBy;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-
import org.apache.spark.internal.SparkLogger;
import org.apache.spark.internal.SparkLoggerFactory;
import org.apache.spark.internal.LogKeys;
import org.apache.spark.internal.MDC;
+import org.apache.spark.network.util.JavaUtils;
import org.apache.spark.util.ThreadUtils;
/**
@@ -105,7 +103,7 @@ public class ReadAheadInputStream extends InputStream {
*/
public ReadAheadInputStream(
InputStream inputStream, int bufferSizeInBytes) {
- Preconditions.checkArgument(bufferSizeInBytes > 0,
+ JavaUtils.checkArgument(bufferSizeInBytes > 0,
"bufferSizeInBytes should be greater than 0, but the value is " + bufferSizeInBytes);
activeBuffer = ByteBuffer.allocate(bufferSizeInBytes);
readAheadBuffer = ByteBuffer.allocate(bufferSizeInBytes);
@@ -120,8 +118,10 @@ private boolean isEndOfStream() {
private void checkReadException() throws IOException {
if (readAborted) {
- Throwables.throwIfInstanceOf(readException, IOException.class);
- Throwables.throwIfUnchecked(readException);
+ if (readException == null) throw new NullPointerException("readException is not captured.");
+ if (readException instanceof IOException ie) throw ie;
+ if (readException instanceof Error error) throw error;
+ if (readException instanceof RuntimeException re) throw re;
throw new IOException(readException);
}
}
@@ -213,7 +213,7 @@ private void closeUnderlyingInputStreamIfNecessary() {
try {
underlyingInputStream.close();
} catch (IOException e) {
- logger.warn("{}", e, MDC.of(LogKeys.ERROR$.MODULE$, e.getMessage()));
+ logger.warn("{}", e, MDC.of(LogKeys.ERROR, e.getMessage()));
}
}
}
diff --git a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
index e98554db22524..ab9e470e0c2c0 100644
--- a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
+++ b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
@@ -279,11 +279,11 @@ private long trySpillAndAcquire(
} catch (ClosedByInterruptException | InterruptedIOException e) {
// This called by user to kill a task (e.g: speculative task).
logger.error("Error while calling spill() on {}", e,
- MDC.of(LogKeys.MEMORY_CONSUMER$.MODULE$, consumerToSpill));
+ MDC.of(LogKeys.MEMORY_CONSUMER, consumerToSpill));
throw new RuntimeException(e.getMessage());
} catch (IOException e) {
logger.error("Error while calling spill() on {}", e,
- MDC.of(LogKeys.MEMORY_CONSUMER$.MODULE$, consumerToSpill));
+ MDC.of(LogKeys.MEMORY_CONSUMER, consumerToSpill));
// checkstyle.off: RegexpSinglelineJava
throw new SparkOutOfMemoryError(
"SPILL_OUT_OF_MEMORY",
@@ -320,7 +320,7 @@ public void releaseExecutionMemory(long size, MemoryConsumer consumer) {
*/
public void showMemoryUsage() {
logger.info("Memory used in task {}",
- MDC.of(LogKeys.TASK_ATTEMPT_ID$.MODULE$, taskAttemptId));
+ MDC.of(LogKeys.TASK_ATTEMPT_ID, taskAttemptId));
synchronized (this) {
long memoryAccountedForByConsumers = 0;
for (MemoryConsumer c: consumers) {
@@ -328,20 +328,20 @@ public void showMemoryUsage() {
memoryAccountedForByConsumers += totalMemUsage;
if (totalMemUsage > 0) {
logger.info("Acquired by {}: {}",
- MDC.of(LogKeys.MEMORY_CONSUMER$.MODULE$, c),
- MDC.of(LogKeys.MEMORY_SIZE$.MODULE$, Utils.bytesToString(totalMemUsage)));
+ MDC.of(LogKeys.MEMORY_CONSUMER, c),
+ MDC.of(LogKeys.MEMORY_SIZE, Utils.bytesToString(totalMemUsage)));
}
}
long memoryNotAccountedFor =
memoryManager.getExecutionMemoryUsageForTask(taskAttemptId) - memoryAccountedForByConsumers;
logger.info(
"{} bytes of memory were used by task {} but are not associated with specific consumers",
- MDC.of(LogKeys.MEMORY_SIZE$.MODULE$, memoryNotAccountedFor),
- MDC.of(LogKeys.TASK_ATTEMPT_ID$.MODULE$, taskAttemptId));
+ MDC.of(LogKeys.MEMORY_SIZE, memoryNotAccountedFor),
+ MDC.of(LogKeys.TASK_ATTEMPT_ID, taskAttemptId));
logger.info(
"{} bytes of memory are used for execution and {} bytes of memory are used for storage",
- MDC.of(LogKeys.EXECUTION_MEMORY_SIZE$.MODULE$, memoryManager.executionMemoryUsed()),
- MDC.of(LogKeys.STORAGE_MEMORY_SIZE$.MODULE$, memoryManager.storageMemoryUsed()));
+ MDC.of(LogKeys.EXECUTION_MEMORY_SIZE, memoryManager.executionMemoryUsed()),
+ MDC.of(LogKeys.STORAGE_MEMORY_SIZE, memoryManager.storageMemoryUsed()));
}
}
@@ -388,7 +388,7 @@ public MemoryBlock allocatePage(long size, MemoryConsumer consumer) {
page = memoryManager.tungstenMemoryAllocator().allocate(acquired);
} catch (OutOfMemoryError e) {
logger.warn("Failed to allocate a page ({} bytes), try again.",
- MDC.of(LogKeys.PAGE_SIZE$.MODULE$, acquired));
+ MDC.of(LogKeys.PAGE_SIZE, acquired));
// there is no enough memory actually, it means the actual free memory is smaller than
// MemoryManager thought, we should keep the acquired memory.
synchronized (this) {
diff --git a/core/src/main/java/org/apache/spark/shuffle/checksum/RowBasedChecksum.scala b/core/src/main/java/org/apache/spark/shuffle/checksum/RowBasedChecksum.scala
new file mode 100644
index 0000000000000..886296dc8a828
--- /dev/null
+++ b/core/src/main/java/org/apache/spark/shuffle/checksum/RowBasedChecksum.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.shuffle.checksum
+
+import scala.util.control.NonFatal
+
+import org.apache.spark.internal.Logging
+
+/**
+ * A class for computing checksum for input (key, value) pairs. The checksum is independent of
+ * the order of the input (key, value) pairs. It is done by computing a checksum for each row
+ * first, then computing the XOR and SUM for all the row checksums and mixing these two values
+ * as the final checksum.
+ */
+abstract class RowBasedChecksum() extends Serializable with Logging {
+ private val ROTATE_POSITIONS = 27
+ private var hasError: Boolean = false
+ private var checksumXor: Long = 0
+ private var checksumSum: Long = 0
+
+ /**
+ * Returns the checksum value. It returns the default checksum value (0) if there
+ * are any errors encountered during the checksum computation.
+ */
+ def getValue: Long = {
+ if (!hasError) {
+ // Here we rotate the `checksumSum` to transforms these two values into a single, strong
+ // composite checksum by ensuring their bit patterns are thoroughly mixed.
+ checksumXor ^ rotateLeft(checksumSum)
+ } else {
+ 0
+ }
+ }
+
+ /** Updates the row-based checksum with the given (key, value) pair. Not thread safe. */
+ def update(key: Any, value: Any): Unit = {
+ if (!hasError) {
+ try {
+ val rowChecksumValue = calculateRowChecksum(key, value)
+ checksumXor = checksumXor ^ rowChecksumValue
+ checksumSum += rowChecksumValue
+ } catch {
+ case NonFatal(e) =>
+ logError("Checksum computation encountered error: ", e)
+ hasError = true
+ }
+ }
+ }
+
+ /** Computes and returns the checksum value for the given (key, value) pair */
+ protected def calculateRowChecksum(key: Any, value: Any): Long
+
+ // Rotate the value by shifting the bits by `ROTATE_POSITIONS` positions to the left.
+ private def rotateLeft(value: Long): Long = {
+ (value << ROTATE_POSITIONS) | (value >>> (64 - ROTATE_POSITIONS))
+ }
+}
+
+object RowBasedChecksum {
+ def getAggregatedChecksumValue(rowBasedChecksums: Array[RowBasedChecksum]): Long = {
+ Option(rowBasedChecksums)
+ .map(_.foldLeft(0L)((acc, c) => acc * 31L + c.getValue))
+ .getOrElse(0L)
+ }
+}
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java
index 86f7d5143eff5..5acc66e120630 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java
@@ -32,6 +32,7 @@
import scala.Tuple2;
import scala.collection.Iterator;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.io.Closeables;
import org.apache.spark.internal.SparkLogger;
@@ -53,6 +54,7 @@
import org.apache.spark.scheduler.MapStatus$;
import org.apache.spark.serializer.Serializer;
import org.apache.spark.serializer.SerializerInstance;
+import org.apache.spark.shuffle.checksum.RowBasedChecksum;
import org.apache.spark.shuffle.ShuffleWriteMetricsReporter;
import org.apache.spark.shuffle.ShuffleWriter;
import org.apache.spark.storage.*;
@@ -104,6 +106,13 @@ final class BypassMergeSortShuffleWriter
private long[] partitionLengths;
/** Checksum calculator for each partition. Empty when shuffle checksum disabled. */
private final Checksum[] partitionChecksums;
+ /**
+ * Checksum calculator for each partition. Different from the above Checksum,
+ * RowBasedChecksum is independent of the input row order, which is used to
+ * detect whether different task attempts of the same partition produce different
+ * output data or not.
+ */
+ private final RowBasedChecksum[] rowBasedChecksums;
/**
* Are we in the process of stopping? Because map tasks can call stop() with success = true
@@ -132,6 +141,7 @@ final class BypassMergeSortShuffleWriter
this.serializer = dep.serializer();
this.shuffleExecutorComponents = shuffleExecutorComponents;
this.partitionChecksums = createPartitionChecksums(numPartitions, conf);
+ this.rowBasedChecksums = dep.rowBasedChecksums();
}
@Override
@@ -144,7 +154,7 @@ public void write(Iterator> records) throws IOException {
partitionLengths = mapOutputWriter.commitAllPartitions(
ShuffleChecksumHelper.EMPTY_CHECKSUM_VALUE).getPartitionLengths();
mapStatus = MapStatus$.MODULE$.apply(
- blockManager.shuffleServerId(), partitionLengths, mapId);
+ blockManager.shuffleServerId(), partitionLengths, mapId, getAggregatedChecksumValue());
return;
}
final SerializerInstance serInstance = serializer.newInstance();
@@ -171,7 +181,11 @@ public void write(Iterator> records) throws IOException {
while (records.hasNext()) {
final Product2 record = records.next();
final K key = record._1();
- partitionWriters[partitioner.getPartition(key)].write(key, record._2());
+ final int partitionId = partitioner.getPartition(key);
+ partitionWriters[partitionId].write(key, record._2());
+ if (rowBasedChecksums.length > 0) {
+ rowBasedChecksums[partitionId].update(key, record._2());
+ }
}
for (int i = 0; i < numPartitions; i++) {
@@ -182,7 +196,7 @@ public void write(Iterator> records) throws IOException {
partitionLengths = writePartitionedData(mapOutputWriter);
mapStatus = MapStatus$.MODULE$.apply(
- blockManager.shuffleServerId(), partitionLengths, mapId);
+ blockManager.shuffleServerId(), partitionLengths, mapId, getAggregatedChecksumValue());
} catch (Exception e) {
try {
mapOutputWriter.abort(e);
@@ -199,6 +213,17 @@ public long[] getPartitionLengths() {
return partitionLengths;
}
+ // For test only.
+ @VisibleForTesting
+ RowBasedChecksum[] getRowBasedChecksums() {
+ return rowBasedChecksums;
+ }
+
+ @VisibleForTesting
+ long getAggregatedChecksumValue() {
+ return RowBasedChecksum.getAggregatedChecksumValue(rowBasedChecksums);
+ }
+
/**
* Concatenate all of the per-partition files into a single combined file.
*
@@ -227,7 +252,7 @@ private long[] writePartitionedData(ShuffleMapOutputWriter mapOutputWriter) thro
}
if (!file.delete()) {
logger.error("Unable to delete file for partition {}",
- MDC.of(LogKeys.PARTITION_ID$.MODULE$, i));
+ MDC.of(LogKeys.PARTITION_ID, i));
}
}
}
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java
index de3c41a4b526b..f053135c4dbd7 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java
@@ -89,6 +89,11 @@ final class ShuffleExternalSorter extends MemoryConsumer implements ShuffleCheck
*/
private final int numElementsForSpillThreshold;
+ /**
+ * Force this sorter to spill when the in memory size in bytes is beyond this threshold.
+ */
+ private final long sizeInBytesForSpillThreshold;
+
/** The buffer size to use when writing spills using DiskBlockObjectWriter */
private final int fileBufferSizeBytes;
@@ -112,6 +117,7 @@ final class ShuffleExternalSorter extends MemoryConsumer implements ShuffleCheck
@Nullable private ShuffleInMemorySorter inMemSorter;
@Nullable private MemoryBlock currentPage = null;
private long pageCursor = -1;
+ private long totalPageMemoryUsageBytes = 0;
// Checksum calculator for each partition. Empty when shuffle checksum disabled.
private final Checksum[] partitionChecksums;
@@ -136,6 +142,8 @@ final class ShuffleExternalSorter extends MemoryConsumer implements ShuffleCheck
(int) (long) conf.get(package$.MODULE$.SHUFFLE_FILE_BUFFER_SIZE()) * 1024;
this.numElementsForSpillThreshold =
(int) conf.get(package$.MODULE$.SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD());
+ this.sizeInBytesForSpillThreshold =
+ (long) conf.get(package$.MODULE$.SHUFFLE_SPILL_MAX_SIZE_FORCE_SPILL_THRESHOLD());
this.writeMetrics = writeMetrics;
this.inMemSorter = new ShuffleInMemorySorter(
this, initialSize, (boolean) conf.get(package$.MODULE$.SHUFFLE_SORT_USE_RADIXSORT()));
@@ -162,11 +170,11 @@ private void writeSortedFile(boolean isFinalFile) {
if (!isFinalFile) {
logger.info(
"Task {} on Thread {} spilling sort data of {} to disk ({} {} so far)",
- MDC.of(LogKeys.TASK_ATTEMPT_ID$.MODULE$, taskContext.taskAttemptId()),
- MDC.of(LogKeys.THREAD_ID$.MODULE$, Thread.currentThread().getId()),
- MDC.of(LogKeys.MEMORY_SIZE$.MODULE$, Utils.bytesToString(getMemoryUsage())),
- MDC.of(LogKeys.NUM_SPILLS$.MODULE$, spills.size()),
- MDC.of(LogKeys.SPILL_TIMES$.MODULE$, spills.size() != 1 ? "times" : "time"));
+ MDC.of(LogKeys.TASK_ATTEMPT_ID, taskContext.taskAttemptId()),
+ MDC.of(LogKeys.THREAD_ID, Thread.currentThread().getId()),
+ MDC.of(LogKeys.MEMORY_SIZE, Utils.bytesToString(getMemoryUsage())),
+ MDC.of(LogKeys.NUM_SPILLS, spills.size()),
+ MDC.of(LogKeys.SPILL_TIMES, spills.size() != 1 ? "times" : "time"));
}
// This call performs the actual sort.
@@ -306,11 +314,7 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
}
private long getMemoryUsage() {
- long totalPageSize = 0;
- for (MemoryBlock page : allocatedPages) {
- totalPageSize += page.size();
- }
- return ((inMemSorter == null) ? 0 : inMemSorter.getMemoryUsage()) + totalPageSize;
+ return ((inMemSorter == null) ? 0 : inMemSorter.getMemoryUsage()) + totalPageMemoryUsageBytes;
}
private void updatePeakMemoryUsed() {
@@ -334,6 +338,7 @@ private long freeMemory() {
for (MemoryBlock block : allocatedPages) {
memoryFreed += block.size();
freePage(block);
+ totalPageMemoryUsageBytes -= block.size();
}
allocatedPages.clear();
currentPage = null;
@@ -353,7 +358,7 @@ public void cleanupResources() {
for (SpillInfo spill : spills) {
if (spill.file.exists() && !spill.file.delete()) {
logger.error("Unable to delete spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, spill.file.getPath()));
+ MDC.of(LogKeys.PATH, spill.file.getPath()));
}
}
}
@@ -408,6 +413,7 @@ private void acquireNewPageIfNecessary(int required) {
currentPage = allocatePage(required);
pageCursor = currentPage.getBaseOffset();
allocatedPages.add(currentPage);
+ totalPageMemoryUsageBytes += currentPage.size();
}
}
@@ -417,11 +423,23 @@ private void acquireNewPageIfNecessary(int required) {
public void insertRecord(Object recordBase, long recordOffset, int length, int partitionId)
throws IOException {
- // for tests
assert(inMemSorter != null);
if (inMemSorter.numRecords() >= numElementsForSpillThreshold) {
- logger.info("Spilling data because number of spilledRecords crossed the threshold {}" +
- MDC.of(LogKeys.NUM_ELEMENTS_SPILL_THRESHOLD$.MODULE$, numElementsForSpillThreshold));
+ logger.info("Spilling data because number of spilledRecords ({}) crossed the threshold {}",
+ MDC.of(LogKeys.NUM_ELEMENTS_SPILL_RECORDS, inMemSorter.numRecords()),
+ MDC.of(LogKeys.NUM_ELEMENTS_SPILL_THRESHOLD, numElementsForSpillThreshold));
+ spill();
+ }
+
+ // TODO: Ideally we only need to check the spill threshold when new memory needs to be
+ // allocated (both this sorter and the underlying ShuffleInMemorySorter may allocate
+ // new memory), but it's simpler to check the total memory usage of these two sorters
+ // before inserting each record.
+ final long usedMemory = getMemoryUsage();
+ if (usedMemory >= sizeInBytesForSpillThreshold) {
+ logger.info("Spilling data because memory usage ({}) crossed the threshold {}",
+ MDC.of(LogKeys.SPILL_RECORDS_SIZE, usedMemory),
+ MDC.of(LogKeys.SPILL_RECORDS_SIZE_THRESHOLD, sizeInBytesForSpillThreshold));
spill();
}
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
index ac9d335d63591..e3ecfed323481 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
@@ -33,7 +33,6 @@
import scala.reflect.ClassTag$;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.ByteStreams;
import com.google.common.io.Closeables;
import org.apache.spark.*;
@@ -60,9 +59,11 @@
import org.apache.spark.shuffle.api.ShufflePartitionWriter;
import org.apache.spark.shuffle.api.SingleSpillShuffleMapOutputWriter;
import org.apache.spark.shuffle.api.WritableByteChannelWrapper;
+import org.apache.spark.shuffle.checksum.RowBasedChecksum;
import org.apache.spark.storage.BlockManager;
import org.apache.spark.storage.TimeTrackingOutputStream;
import org.apache.spark.unsafe.Platform;
+import org.apache.spark.util.ExposedBufferByteArrayOutputStream;
import org.apache.spark.util.Utils;
@Private
@@ -94,15 +95,16 @@ public class UnsafeShuffleWriter extends ShuffleWriter {
@Nullable private long[] partitionLengths;
private long peakMemoryUsedBytes = 0;
- /** Subclass of ByteArrayOutputStream that exposes `buf` directly. */
- private static final class MyByteArrayOutputStream extends ByteArrayOutputStream {
- MyByteArrayOutputStream(int size) { super(size); }
- public byte[] getBuf() { return buf; }
- }
-
- private MyByteArrayOutputStream serBuffer;
+ private ExposedBufferByteArrayOutputStream serBuffer;
private SerializationStream serOutputStream;
+ /**
+ * RowBasedChecksum calculator for each partition. RowBasedChecksum is independent
+ * of the input row order, which is used to detect whether different task attempts
+ * of the same partition produce different output data or not.
+ */
+ private final RowBasedChecksum[] rowBasedChecksums;
+
/**
* Are we in the process of stopping? Because map tasks can call stop() with success = true
* and then call stop() with success = false if they get an exception, we want to make sure
@@ -142,6 +144,7 @@ public UnsafeShuffleWriter(
(int) (long) sparkConf.get(package$.MODULE$.SHUFFLE_SORT_INIT_BUFFER_SIZE());
this.mergeBufferSizeInBytes =
(int) (long) sparkConf.get(package$.MODULE$.SHUFFLE_FILE_MERGE_BUFFER_SIZE()) * 1024;
+ this.rowBasedChecksums = dep.rowBasedChecksums();
open();
}
@@ -163,6 +166,17 @@ public long getPeakMemoryUsedBytes() {
return peakMemoryUsedBytes;
}
+ // For test only.
+ @VisibleForTesting
+ RowBasedChecksum[] getRowBasedChecksums() {
+ return rowBasedChecksums;
+ }
+
+ @VisibleForTesting
+ long getAggregatedChecksumValue() {
+ return RowBasedChecksum.getAggregatedChecksumValue(rowBasedChecksums);
+ }
+
/**
* This convenience method should only be called in test code.
*/
@@ -211,7 +225,7 @@ private void open() throws SparkException {
partitioner.numPartitions(),
sparkConf,
writeMetrics);
- serBuffer = new MyByteArrayOutputStream(DEFAULT_INITIAL_SER_BUFFER_SIZE);
+ serBuffer = new ExposedBufferByteArrayOutputStream(DEFAULT_INITIAL_SER_BUFFER_SIZE);
serOutputStream = serializer.serializeStream(serBuffer);
}
@@ -229,12 +243,12 @@ void closeAndWriteOutput() throws IOException {
for (SpillInfo spill : spills) {
if (spill.file.exists() && !spill.file.delete()) {
logger.error("Error while deleting spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, spill.file.getPath()));
+ MDC.of(LogKeys.PATH, spill.file.getPath()));
}
}
}
mapStatus = MapStatus$.MODULE$.apply(
- blockManager.shuffleServerId(), partitionLengths, mapId);
+ blockManager.shuffleServerId(), partitionLengths, mapId, getAggregatedChecksumValue());
}
@VisibleForTesting
@@ -252,6 +266,9 @@ void insertRecordIntoSorter(Product2 record) throws IOException {
sorter.insertRecord(
serBuffer.getBuf(), Platform.BYTE_ARRAY_OFFSET, serializedRecordSize, partitionId);
+ if (rowBasedChecksums.length > 0) {
+ rowBasedChecksums[partitionId].update(key, record._2());
+ }
}
@VisibleForTesting
@@ -404,7 +421,7 @@ private void mergeSpillsWithFileStream(
partitionInputStream = compressionCodec.compressedInputStream(
partitionInputStream);
}
- ByteStreams.copy(partitionInputStream, partitionOutput);
+ partitionInputStream.transferTo(partitionOutput);
copySpillThrewException = false;
} finally {
Closeables.close(partitionInputStream, copySpillThrewException);
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleExecutorComponents.java b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleExecutorComponents.java
index 38f0a60f8b0dd..e9f20962b1a48 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleExecutorComponents.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleExecutorComponents.java
@@ -17,9 +17,9 @@
package org.apache.spark.shuffle.sort.io;
-import java.util.Collections;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
import com.google.common.annotations.VisibleForTesting;
@@ -59,7 +59,7 @@ public void initializeExecutor(String appId, String execId, Map
}
blockResolver =
new IndexShuffleBlockResolver(
- sparkConf, blockManager, Collections.emptyMap() /* Shouldn't be accessed */
+ sparkConf, blockManager, new ConcurrentHashMap<>() /* Shouldn't be accessed */
);
}
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
index c0b9018c770a0..e51a48b019854 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
@@ -125,7 +125,7 @@ public void abort(Throwable error) throws IOException {
cleanUp();
if (outputTempFile != null && outputTempFile.exists() && !outputTempFile.delete()) {
log.warn("Failed to delete temporary shuffle file at {}",
- MDC.of(LogKeys.PATH$.MODULE$, outputTempFile.getAbsolutePath()));
+ MDC.of(LogKeys.PATH, outputTempFile.getAbsolutePath()));
}
}
diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 2a8e15cd09ccf..486bfd62bc97a 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -395,7 +395,7 @@ private void handleFailedDelete() {
File file = spillWriters.removeFirst().getFile();
if (file != null && file.exists() && !file.delete()) {
logger.error("Was unable to delete spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, file.getAbsolutePath()));
+ MDC.of(LogKeys.PATH, file.getAbsolutePath()));
}
}
}
@@ -897,7 +897,7 @@ public void free() {
if (file != null && file.exists()) {
if (!file.delete()) {
logger.error("Was unable to delete spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, file.getAbsolutePath()));
+ MDC.of(LogKeys.PATH, file.getAbsolutePath()));
}
}
}
diff --git a/core/src/main/java/org/apache/spark/util/EnumUtil.java b/core/src/main/java/org/apache/spark/util/EnumUtil.java
index c40c7e727613c..6c397640590fe 100644
--- a/core/src/main/java/org/apache/spark/util/EnumUtil.java
+++ b/core/src/main/java/org/apache/spark/util/EnumUtil.java
@@ -16,7 +16,8 @@
*/
package org.apache.spark.util;
-import com.google.common.base.Joiner;
+import java.util.StringJoiner;
+
import org.apache.spark.annotation.Private;
@Private
@@ -33,6 +34,14 @@ public static > E parseIgnoreCase(Class clz, String str) {
}
throw new IllegalArgumentException(
String.format("Illegal type='%s'. Supported type values: %s",
- str, Joiner.on(", ").join(constants)));
+ str, joinToString(constants)));
+ }
+
+ private static > String joinToString(E[] enums) {
+ StringJoiner stringJoiner = new StringJoiner(", ");
+ for (E e : enums) {
+ stringJoiner.add(e.name());
+ }
+ return stringJoiner.toString();
}
}
diff --git a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomPluginBase.scala b/core/src/main/java/org/apache/spark/util/ExposedBufferByteArrayOutputStream.java
similarity index 70%
rename from connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomPluginBase.scala
rename to core/src/main/java/org/apache/spark/util/ExposedBufferByteArrayOutputStream.java
index df73e0d9a0fb8..bd59bd176fb93 100644
--- a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomPluginBase.scala
+++ b/core/src/main/java/org/apache/spark/util/ExposedBufferByteArrayOutputStream.java
@@ -15,13 +15,12 @@
* limitations under the License.
*/
-package org.apache.connect.examples.serverlibrary
+package org.apache.spark.util;
-import org.apache.connect.examples.serverlibrary.proto
-import org.apache.spark.sql.connect.planner.SparkConnectPlanner
+import java.io.ByteArrayOutputStream;
-trait CustomPluginBase {
- protected def getCustomTable(table: proto.CustomTable): CustomTable = {
- CustomTable.getTable(table.getName, table.getPath)
- }
+/** Subclass of ByteArrayOutputStream that exposes `buf` directly. */
+public final class ExposedBufferByteArrayOutputStream extends ByteArrayOutputStream {
+ public ExposedBufferByteArrayOutputStream(int size) { super(size); }
+ public byte[] getBuf() { return buf; }
}
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/RadixSort.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/RadixSort.java
index 3dd318471008b..c8f984e856cef 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/RadixSort.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/RadixSort.java
@@ -17,10 +17,9 @@
package org.apache.spark.util.collection.unsafe.sort;
-import com.google.common.primitives.Ints;
-
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.array.LongArray;
+import org.apache.spark.network.util.JavaUtils;
public class RadixSort {
@@ -63,7 +62,7 @@ public static int sort(
}
}
}
- return Ints.checkedCast(inIndex);
+ return JavaUtils.checkedCast(inIndex);
}
/**
@@ -204,7 +203,7 @@ public static int sortKeyPrefixArray(
}
}
}
- return Ints.checkedCast(inIndex);
+ return JavaUtils.checkedCast(inIndex);
}
/**
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
index af421e903ba3f..71a826642b1be 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
@@ -27,7 +27,6 @@
import java.util.function.Supplier;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.io.IOUtils;
import org.apache.spark.TaskContext;
import org.apache.spark.executor.ShuffleWriteMetrics;
@@ -80,6 +79,11 @@ public final class UnsafeExternalSorter extends MemoryConsumer {
*/
private final int numElementsForSpillThreshold;
+ /**
+ * Force this sorter to spill when the in memory size in bytes is beyond this threshold.
+ */
+ private final long sizeInBytesForSpillThreshold;
+
/**
* Memory pages that hold the records being sorted. The pages in this list are freed when
* spilling, although in principle we could recycle these pages across spills (on the other hand,
@@ -92,6 +96,7 @@ public final class UnsafeExternalSorter extends MemoryConsumer {
// These variables are reset after spilling:
@Nullable private volatile UnsafeInMemorySorter inMemSorter;
+ private long totalPageMemoryUsageBytes = 0;
private MemoryBlock currentPage = null;
private long pageCursor = -1;
@@ -110,11 +115,13 @@ public static UnsafeExternalSorter createWithExistingInMemorySorter(
int initialSize,
long pageSizeBytes,
int numElementsForSpillThreshold,
+ long sizeInBytesForSpillThreshold,
UnsafeInMemorySorter inMemorySorter,
long existingMemoryConsumption) throws IOException {
UnsafeExternalSorter sorter = new UnsafeExternalSorter(taskMemoryManager, blockManager,
serializerManager, taskContext, recordComparatorSupplier, prefixComparator, initialSize,
- pageSizeBytes, numElementsForSpillThreshold, inMemorySorter, false /* ignored */);
+ pageSizeBytes, numElementsForSpillThreshold, sizeInBytesForSpillThreshold,
+ inMemorySorter, false /* ignored */);
sorter.spill(Long.MAX_VALUE, sorter);
taskContext.taskMetrics().incMemoryBytesSpilled(existingMemoryConsumption);
sorter.totalSpillBytes += existingMemoryConsumption;
@@ -133,10 +140,11 @@ public static UnsafeExternalSorter create(
int initialSize,
long pageSizeBytes,
int numElementsForSpillThreshold,
+ long sizeInBytesForSpillThreshold,
boolean canUseRadixSort) {
return new UnsafeExternalSorter(taskMemoryManager, blockManager, serializerManager,
taskContext, recordComparatorSupplier, prefixComparator, initialSize, pageSizeBytes,
- numElementsForSpillThreshold, null, canUseRadixSort);
+ numElementsForSpillThreshold, sizeInBytesForSpillThreshold, null, canUseRadixSort);
}
private UnsafeExternalSorter(
@@ -149,6 +157,7 @@ private UnsafeExternalSorter(
int initialSize,
long pageSizeBytes,
int numElementsForSpillThreshold,
+ long sizeInBytesForSpillThreshold,
@Nullable UnsafeInMemorySorter existingInMemorySorter,
boolean canUseRadixSort) {
super(taskMemoryManager, pageSizeBytes, taskMemoryManager.getTungstenMemoryMode());
@@ -178,6 +187,7 @@ private UnsafeExternalSorter(
this.inMemSorter = existingInMemorySorter;
}
this.peakMemoryUsedBytes = getMemoryUsage();
+ this.sizeInBytesForSpillThreshold = sizeInBytesForSpillThreshold;
this.numElementsForSpillThreshold = numElementsForSpillThreshold;
// Register a cleanup task with TaskContext to ensure that memory is guaranteed to be freed at
@@ -220,10 +230,10 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
}
logger.info("Thread {} spilling sort data of {} to disk ({} {} so far)",
- MDC.of(LogKeys.THREAD_ID$.MODULE$, Thread.currentThread().getId()),
- MDC.of(LogKeys.MEMORY_SIZE$.MODULE$, Utils.bytesToString(getMemoryUsage())),
- MDC.of(LogKeys.NUM_SPILL_WRITERS$.MODULE$, spillWriters.size()),
- MDC.of(LogKeys.SPILL_TIMES$.MODULE$, spillWriters.size() > 1 ? "times" : "time"));
+ MDC.of(LogKeys.THREAD_ID, Thread.currentThread().getId()),
+ MDC.of(LogKeys.MEMORY_SIZE, Utils.bytesToString(getMemoryUsage())),
+ MDC.of(LogKeys.NUM_SPILL_WRITERS, spillWriters.size()),
+ MDC.of(LogKeys.SPILL_TIMES, spillWriters.size() > 1 ? "times" : "time"));
ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics();
@@ -253,11 +263,7 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
* array.
*/
private long getMemoryUsage() {
- long totalPageSize = 0;
- for (MemoryBlock page : allocatedPages) {
- totalPageSize += page.size();
- }
- return ((inMemSorter == null) ? 0 : inMemSorter.getMemoryUsage()) + totalPageSize;
+ return ((inMemSorter == null) ? 0 : inMemSorter.getMemoryUsage()) + totalPageMemoryUsageBytes;
}
private void updatePeakMemoryUsed() {
@@ -309,6 +315,7 @@ private long freeMemory() {
for (MemoryBlock block : pagesToFree) {
memoryFreed += block.size();
freePage(block);
+ totalPageMemoryUsageBytes -= block.size();
}
return memoryFreed;
}
@@ -339,7 +346,7 @@ private void deleteSpillFiles() {
if (file != null && file.exists()) {
if (!file.delete()) {
logger.error("Was unable to delete spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, file.getAbsolutePath()));
+ MDC.of(LogKeys.PATH, file.getAbsolutePath()));
}
}
}
@@ -367,6 +374,7 @@ public void cleanupResources() {
} finally {
for (MemoryBlock pageToFree : pagesToFree) {
freePage(pageToFree);
+ totalPageMemoryUsageBytes -= pageToFree.size();
}
if (inMemSorterToFree != null) {
inMemSorterToFree.freeMemory();
@@ -437,6 +445,7 @@ private void acquireNewPageIfNecessary(int required) {
currentPage = allocatePage(required);
pageCursor = currentPage.getBaseOffset();
allocatedPages.add(currentPage);
+ totalPageMemoryUsageBytes += currentPage.size();
}
}
@@ -480,8 +489,21 @@ public void insertRecord(
assert(inMemSorter != null);
if (inMemSorter.numRecords() >= numElementsForSpillThreshold) {
- logger.info("Spilling data because number of spilledRecords crossed the threshold {}",
- MDC.of(LogKeys.NUM_ELEMENTS_SPILL_THRESHOLD$.MODULE$, numElementsForSpillThreshold));
+ logger.info("Spilling data because number of spilledRecords ({}) crossed the threshold {}",
+ MDC.of(LogKeys.NUM_ELEMENTS_SPILL_RECORDS, inMemSorter.numRecords()),
+ MDC.of(LogKeys.NUM_ELEMENTS_SPILL_THRESHOLD, numElementsForSpillThreshold));
+ spill();
+ }
+
+ // TODO: Ideally we only need to check the spill threshold when new memory needs to be
+ // allocated (both this sorter and the underlying UnsafeInMemorySorter may allocate
+ // new memory), but it's simpler to check the total memory usage of these two sorters
+ // before inserting each record.
+ final long usedMemory = getMemoryUsage();
+ if (usedMemory >= sizeInBytesForSpillThreshold) {
+ logger.info("Spilling data because memory usage ({}) crossed the threshold {}",
+ MDC.of(LogKeys.SPILL_RECORDS_SIZE, usedMemory),
+ MDC.of(LogKeys.SPILL_RECORDS_SIZE_THRESHOLD, sizeInBytesForSpillThreshold));
spill();
}
@@ -867,7 +889,7 @@ public void close() throws IOException {
private void closeIfPossible(UnsafeSorterIterator iterator) {
if (iterator instanceof Closeable closeable) {
- IOUtils.closeQuietly((closeable));
+ Utils.closeQuietly((closeable));
}
}
}
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
index 0693f8cb1a808..6674fcd73b92b 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
@@ -17,7 +17,6 @@
package org.apache.spark.util.collection.unsafe.sort;
-import com.google.common.io.ByteStreams;
import com.google.common.io.Closeables;
import org.apache.spark.SparkEnv;
import org.apache.spark.TaskContext;
@@ -27,6 +26,7 @@
import org.apache.spark.internal.SparkLoggerFactory;
import org.apache.spark.io.NioBufferedFileInputStream;
import org.apache.spark.io.ReadAheadInputStream;
+import org.apache.spark.network.util.JavaUtils;
import org.apache.spark.serializer.SerializerManager;
import org.apache.spark.storage.BlockId;
import org.apache.spark.unsafe.Platform;
@@ -128,7 +128,7 @@ public void loadNext() throws IOException {
arr = new byte[recordLength];
baseObject = arr;
}
- ByteStreams.readFully(in, arr, 0, recordLength);
+ JavaUtils.readFully(in, arr, 0, recordLength);
numRecordsRemaining--;
if (numRecordsRemaining == 0) {
close();
diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index 3f95515c04d29..2fdb8debf1909 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -24,7 +24,7 @@ import java.util.function.Consumer
import scala.collection.mutable.{ArrayBuffer, HashSet}
import scala.jdk.CollectionConverters._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerStageCompleted}
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 47f287293974f..ee1e4dafcfc1a 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -27,7 +27,7 @@ import scala.util.{Failure, Success => ScalaSuccess, Try}
import org.apache.spark.annotation.{Experimental, Since}
import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.memory.TaskMemoryManager
import org.apache.spark.metrics.source.Source
diff --git a/core/src/main/scala/org/apache/spark/ContextCleaner.scala b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
index fb56389cde77e..54ea8c94daac1 100644
--- a/core/src/main/scala/org/apache/spark/ContextCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
@@ -24,7 +24,7 @@ import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue, Scheduled
import scala.jdk.CollectionConverters._
import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ACCUMULATOR_ID, BROADCAST_ID, LISTENER, RDD_ID, SHUFFLE_ID}
import org.apache.spark.internal.config._
import org.apache.spark.rdd.{RDD, ReliableRDDCheckpointData}
diff --git a/core/src/main/scala/org/apache/spark/Dependency.scala b/core/src/main/scala/org/apache/spark/Dependency.scala
index 573608c4327e0..c436025e06bb8 100644
--- a/core/src/main/scala/org/apache/spark/Dependency.scala
+++ b/core/src/main/scala/org/apache/spark/Dependency.scala
@@ -24,11 +24,12 @@ import scala.reflect.ClassTag
import org.roaringbitmap.RoaringBitmap
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rdd.RDD
import org.apache.spark.serializer.Serializer
import org.apache.spark.shuffle.{ShuffleHandle, ShuffleWriteProcessor}
+import org.apache.spark.shuffle.checksum.RowBasedChecksum
import org.apache.spark.storage.BlockManagerId
import org.apache.spark.util.Utils
@@ -59,6 +60,9 @@ abstract class NarrowDependency[T](_rdd: RDD[T]) extends Dependency[T] {
override def rdd: RDD[T] = _rdd
}
+object ShuffleDependency {
+ private[spark] val EMPTY_ROW_BASED_CHECKSUMS: Array[RowBasedChecksum] = Array.empty
+}
/**
* :: DeveloperApi ::
@@ -74,6 +78,7 @@ abstract class NarrowDependency[T](_rdd: RDD[T]) extends Dependency[T] {
* @param aggregator map/reduce-side aggregator for RDD's shuffle
* @param mapSideCombine whether to perform partial aggregation (also known as map-side combine)
* @param shuffleWriterProcessor the processor to control the write behavior in ShuffleMapTask
+ * @param rowBasedChecksums the row-based checksums for each shuffle partition
*/
@DeveloperApi
class ShuffleDependency[K: ClassTag, V: ClassTag, C: ClassTag](
@@ -83,9 +88,31 @@ class ShuffleDependency[K: ClassTag, V: ClassTag, C: ClassTag](
val keyOrdering: Option[Ordering[K]] = None,
val aggregator: Option[Aggregator[K, V, C]] = None,
val mapSideCombine: Boolean = false,
- val shuffleWriterProcessor: ShuffleWriteProcessor = new ShuffleWriteProcessor)
+ val shuffleWriterProcessor: ShuffleWriteProcessor = new ShuffleWriteProcessor,
+ val rowBasedChecksums: Array[RowBasedChecksum] = ShuffleDependency.EMPTY_ROW_BASED_CHECKSUMS,
+ val checksumMismatchFullRetryEnabled: Boolean = false)
extends Dependency[Product2[K, V]] with Logging {
+ def this(
+ rdd: RDD[_ <: Product2[K, V]],
+ partitioner: Partitioner,
+ serializer: Serializer,
+ keyOrdering: Option[Ordering[K]],
+ aggregator: Option[Aggregator[K, V, C]],
+ mapSideCombine: Boolean,
+ shuffleWriterProcessor: ShuffleWriteProcessor) = {
+ this(
+ rdd,
+ partitioner,
+ serializer,
+ keyOrdering,
+ aggregator,
+ mapSideCombine,
+ shuffleWriterProcessor,
+ ShuffleDependency.EMPTY_ROW_BASED_CHECKSUMS
+ )
+ }
+
if (mapSideCombine) {
require(aggregator.isDefined, "Map-side combine without Aggregator specified!")
}
diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index dd131e443135f..16ec7689d5261 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -26,7 +26,7 @@ import scala.util.control.NonFatal
import com.codahale.metrics.{Counter, Gauge, MetricRegistry}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.DECOMMISSION_ENABLED
@@ -614,7 +614,7 @@ private[spark] class ExecutorAllocationManager(
} else {
executorMonitor.executorsKilled(executorsRemoved.toSeq)
}
- logInfo(log"Executors ${MDC(EXECUTOR_IDS, executorsRemoved.mkString(","))}" +
+ logInfo(log"Executors ${MDC(EXECUTOR_IDS, executorsRemoved.mkString(","))} " +
log"removed due to idle timeout.")
executorsRemoved.toSeq
} else {
diff --git a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
index 92aea5959aab7..40ea5c59ec04e 100644
--- a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
+++ b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
@@ -23,7 +23,7 @@ import scala.collection.mutable.{HashMap, Map}
import scala.concurrent.Future
import org.apache.spark.executor.ExecutorMetrics
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.Network
import org.apache.spark.rpc.{IsolatedThreadSafeRpcEndpoint, RpcCallContext, RpcEnv}
diff --git a/core/src/main/scala/org/apache/spark/Heartbeater.scala b/core/src/main/scala/org/apache/spark/Heartbeater.scala
index 090458eecf182..8302aa17a9942 100644
--- a/core/src/main/scala/org/apache/spark/Heartbeater.scala
+++ b/core/src/main/scala/org/apache/spark/Heartbeater.scala
@@ -48,6 +48,13 @@ private[spark] class Heartbeater(
heartbeater.scheduleAtFixedRate(heartbeatTask, initialDelay, intervalMs, TimeUnit.MILLISECONDS)
}
+ /**
+ * Reports a heartbeat.
+ */
+ def doReportHeartbeat(): Unit = {
+ reportHeartbeat()
+ }
+
/** Stops the heartbeat thread. */
def stop(): Unit = {
heartbeater.shutdown()
diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
index a660bccd2e68f..334eb832c4c2b 100644
--- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
+++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue, ThreadPoolE
import java.util.concurrent.locks.ReentrantReadWriteLock
import scala.collection
-import scala.collection.mutable.{HashMap, ListBuffer, Map}
+import scala.collection.mutable.{HashMap, ListBuffer, Map, Set}
import scala.concurrent.{ExecutionContext, Future}
import scala.concurrent.duration.Duration
import scala.jdk.CollectionConverters._
@@ -34,7 +34,7 @@ import org.apache.commons.io.output.{ByteArrayOutputStream => ApacheByteArrayOut
import org.roaringbitmap.RoaringBitmap
import org.apache.spark.broadcast.{Broadcast, BroadcastManager}
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.io.CompressionCodec
@@ -99,6 +99,12 @@ private class ShuffleStatus(
*/
val mapStatusesDeleted = new Array[MapStatus](numPartitions)
+ /**
+ * Keep the indices of the Map tasks whose checksums are different across retries.
+ * Exposed for testing.
+ */
+ private[spark] val checksumMismatchIndices: Set[Int] = Set()
+
/**
* MergeStatus for each shuffle partition when push-based shuffle is enabled. The index of the
* array is the shuffle partition id (reduce id). Each value in the array is the MergeStatus for
@@ -159,9 +165,11 @@ private class ShuffleStatus(
/**
* Register a map output. If there is already a registered location for the map output then it
- * will be replaced by the new location.
+ * will be replaced by the new location. Returns true if the checksum in the new MapStatus is
+ * different from a previous registered MapStatus. Otherwise, returns false.
*/
- def addMapOutput(mapIndex: Int, status: MapStatus): Unit = withWriteLock {
+ def addMapOutput(mapIndex: Int, status: MapStatus): Boolean = withWriteLock {
+ var isChecksumMismatch: Boolean = false
val currentMapStatus = mapStatuses(mapIndex)
if (currentMapStatus == null) {
_numAvailableMapOutputs += 1
@@ -169,8 +177,19 @@ private class ShuffleStatus(
} else {
mapIdToMapIndex.remove(currentMapStatus.mapId)
}
+ logDebug(s"Checksum of map output for task ${status.mapId} is ${status.checksumValue}")
+
+ val preStatus =
+ if (mapStatuses(mapIndex) != null) mapStatuses(mapIndex) else mapStatusesDeleted(mapIndex)
+ if (preStatus != null && preStatus.checksumValue != status.checksumValue) {
+ logInfo(s"Checksum of map output changes from ${preStatus.checksumValue} to " +
+ s"${status.checksumValue} for task ${status.mapId}.")
+ checksumMismatchIndices.add(mapIndex)
+ isChecksumMismatch = true
+ }
mapStatuses(mapIndex) = status
mapIdToMapIndex(status.mapId) = mapIndex
+ isChecksumMismatch
}
/**
@@ -838,7 +857,7 @@ private[spark] class MapOutputTrackerMaster(
}
}
- def registerMapOutput(shuffleId: Int, mapIndex: Int, status: MapStatus): Unit = {
+ def registerMapOutput(shuffleId: Int, mapIndex: Int, status: MapStatus): Boolean = {
shuffleStatuses(shuffleId).addMapOutput(mapIndex, status)
}
diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala b/core/src/main/scala/org/apache/spark/SecurityManager.scala
index c951876e62034..1461677219bc1 100644
--- a/core/src/main/scala/org/apache/spark/SecurityManager.scala
+++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.io.Text
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.UI._
import org.apache.spark.launcher.SparkLauncher
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index bbf2809b7db23..a1353bf7cec06 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import org.apache.avro.{Schema, SchemaNormalization}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.History._
@@ -255,6 +255,12 @@ trait ReadOnlySparkConf {
throw new IllegalArgumentException(s"Illegal value for config key $key: ${e.getMessage}", e)
}
}
+
+ /**
+ * By using this instead of System.getenv(), environment variables can be mocked
+ * in unit tests.
+ */
+ private[spark] def getenv(name: String): String = System.getenv(name)
}
/**
@@ -528,12 +534,6 @@ class SparkConf(loadDefaults: Boolean)
cloned
}
- /**
- * By using this instead of System.getenv(), environment variables can be mocked
- * in unit tests.
- */
- private[spark] def getenv(name: String): String = System.getenv(name)
-
/**
* Checks for illegal or deprecated config settings. Throws an exception for the former. Not
* idempotent - may mutate this conf object to convert deprecated settings to supported ones.
@@ -667,7 +667,7 @@ private[spark] object SparkConf extends Logging {
DeprecatedConfig("spark.shuffle.spill", "1.6", "Not used anymore."),
DeprecatedConfig("spark.rpc", "2.0", "Not used anymore."),
DeprecatedConfig("spark.scheduler.executorTaskBlacklistTime", "2.1.0",
- "Please use the new excludedOnFailure options, spark.excludeOnFailure.*"),
+ "Not used anymore. Please use the new excludedOnFailure options, spark.excludeOnFailure.*"),
DeprecatedConfig("spark.yarn.am.port", "2.0.0", "Not used anymore"),
DeprecatedConfig("spark.executor.port", "2.0.0", "Not used anymore"),
DeprecatedConfig("spark.rpc.numRetries", "2.2.0", "Not used anymore"),
@@ -680,29 +680,32 @@ private[spark] object SparkConf extends Logging {
DeprecatedConfig("spark.executor.plugins", "3.0.0",
"Feature replaced with new plugin API. See Monitoring documentation."),
DeprecatedConfig("spark.blacklist.enabled", "3.1.0",
- "Please use spark.excludeOnFailure.enabled"),
+ "Not used anymore. Please use spark.excludeOnFailure.enabled"),
DeprecatedConfig("spark.blacklist.task.maxTaskAttemptsPerExecutor", "3.1.0",
- "Please use spark.excludeOnFailure.task.maxTaskAttemptsPerExecutor"),
+ "Not used anymore. Please use spark.excludeOnFailure.task.maxTaskAttemptsPerExecutor"),
DeprecatedConfig("spark.blacklist.task.maxTaskAttemptsPerNode", "3.1.0",
- "Please use spark.excludeOnFailure.task.maxTaskAttemptsPerNode"),
+ "Not used anymore. Please use spark.excludeOnFailure.task.maxTaskAttemptsPerNode"),
DeprecatedConfig("spark.blacklist.application.maxFailedTasksPerExecutor", "3.1.0",
- "Please use spark.excludeOnFailure.application.maxFailedTasksPerExecutor"),
+ "Not used anymore. Please use " +
+ "spark.excludeOnFailure.application.maxFailedTasksPerExecutor"),
DeprecatedConfig("spark.blacklist.stage.maxFailedTasksPerExecutor", "3.1.0",
- "Please use spark.excludeOnFailure.stage.maxFailedTasksPerExecutor"),
+ "Not used anymore. Please use spark.excludeOnFailure.stage.maxFailedTasksPerExecutor"),
DeprecatedConfig("spark.blacklist.application.maxFailedExecutorsPerNode", "3.1.0",
- "Please use spark.excludeOnFailure.application.maxFailedExecutorsPerNode"),
+ "Not used anymore. Please use " +
+ "spark.excludeOnFailure.application.maxFailedExecutorsPerNode"),
DeprecatedConfig("spark.blacklist.stage.maxFailedExecutorsPerNode", "3.1.0",
- "Please use spark.excludeOnFailure.stage.maxFailedExecutorsPerNode"),
+ "Not used anymore. Please use spark.excludeOnFailure.stage.maxFailedExecutorsPerNode"),
DeprecatedConfig("spark.blacklist.timeout", "3.1.0",
- "Please use spark.excludeOnFailure.timeout"),
+ "Not used anymore. Please use spark.excludeOnFailure.timeout"),
DeprecatedConfig("spark.blacklist.application.fetchFailure.enabled", "3.1.0",
- "Please use spark.excludeOnFailure.application.fetchFailure.enabled"),
+ "Not used anymore. Please use spark.excludeOnFailure.application.fetchFailure.enabled"),
DeprecatedConfig("spark.scheduler.blacklist.unschedulableTaskSetTimeout", "3.1.0",
- "Please use spark.scheduler.excludeOnFailure.unschedulableTaskSetTimeout"),
+ "Not used anymore. Please use " +
+ "spark.scheduler.excludeOnFailure.unschedulableTaskSetTimeout"),
DeprecatedConfig("spark.blacklist.killBlacklistedExecutors", "3.1.0",
- "Please use spark.excludeOnFailure.killExcludedExecutors"),
+ "Not used anymore. Please use spark.excludeOnFailure.killExcludedExecutors"),
DeprecatedConfig("spark.yarn.blacklist.executor.launch.blacklisting.enabled", "3.1.0",
- "Please use spark.yarn.executor.launch.excludeOnFailure.enabled"),
+ "Not used anymore. Please use spark.yarn.executor.launch.excludeOnFailure.enabled"),
DeprecatedConfig("spark.network.remoteReadNioBufferConversion", "3.5.2",
"Please open a JIRA ticket to report it if you need to use this configuration."),
DeprecatedConfig("spark.shuffle.unsafe.file.output.buffer", "4.0.0",
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 2e61773251b67..898bbad26b7e3 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -46,14 +46,14 @@ import org.apache.spark.deploy.{LocalSparkCluster, SparkHadoopUtil}
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.{Executor, ExecutorMetrics, ExecutorMetricsSource}
import org.apache.spark.input.{FixedLengthBinaryInputFormat, PortableDataStream, StreamInputFormat, WholeTextFileInputFormat}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.Tests._
import org.apache.spark.internal.config.UI._
import org.apache.spark.internal.plugin.PluginContainer
import org.apache.spark.io.CompressionCodec
-import org.apache.spark.launcher.JavaModuleOptions
+import org.apache.spark.launcher.{JavaModuleOptions, SparkLauncher}
import org.apache.spark.metrics.source.JVMCPUSource
import org.apache.spark.partial.{ApproximateEvaluator, PartialResult}
import org.apache.spark.rdd._
@@ -201,10 +201,10 @@ class SparkContext(config: SparkConf) extends Logging {
// log out Spark Version in Spark driver log
logInfo(log"Running Spark version ${MDC(LogKeys.SPARK_VERSION, SPARK_VERSION)}")
- logInfo(log"OS info ${MDC(LogKeys.OS_NAME, System.getProperty("os.name"))}," +
- log" ${MDC(LogKeys.OS_VERSION, System.getProperty("os.version"))}, " +
- log"${MDC(LogKeys.OS_ARCH, System.getProperty("os.arch"))}")
- logInfo(log"Java version ${MDC(LogKeys.JAVA_VERSION, System.getProperty("java.version"))}")
+ logInfo(log"OS info ${MDC(LogKeys.OS_NAME, Utils.osName)}," +
+ log" ${MDC(LogKeys.OS_VERSION, Utils.osVersion)}, " +
+ log"${MDC(LogKeys.OS_ARCH, Utils.osArch)}")
+ logInfo(log"Java version ${MDC(LogKeys.JAVA_VERSION, Utils.javaVersion)}")
/* ------------------------------------------------------------------------------------- *
| Private variables. These variables keep the internal state of the context, and are |
@@ -483,7 +483,6 @@ class SparkContext(config: SparkConf) extends Logging {
}
_listenerBus = new LiveListenerBus(_conf)
- _resourceProfileManager = new ResourceProfileManager(_conf, _listenerBus)
// Initialize the app status store and listener before SparkEnv is created so that it gets
// all events.
@@ -584,8 +583,9 @@ class SparkContext(config: SparkConf) extends Logging {
_heartbeatReceiver = env.rpcEnv.setupEndpoint(
HeartbeatReceiver.ENDPOINT_NAME, new HeartbeatReceiver(this))
- // Initialize any plugins before the task scheduler is initialized.
+ // Initialize any plugins before initializing the task scheduler and resource profile manager.
_plugins = PluginContainer(this, _resources.asJava)
+ _resourceProfileManager = new ResourceProfileManager(_conf, _listenerBus)
_env.initializeShuffleManager()
_env.initializeMemoryManager(SparkContext.numDriverCores(master, conf))
@@ -614,7 +614,7 @@ class SparkContext(config: SparkConf) extends Logging {
_heartbeater = new Heartbeater(
() => SparkContext.this.reportHeartBeat(_executorMetricsSource),
"driver-heartbeater",
- conf.get(EXECUTOR_HEARTBEAT_INTERVAL))
+ conf.get(DRIVER_METRICS_POLLING_INTERVAL))
_heartbeater.start()
// start TaskScheduler after taskScheduler sets DAGScheduler reference in DAGScheduler's
@@ -641,7 +641,8 @@ class SparkContext(config: SparkConf) extends Logging {
}
_ui.foreach(_.setAppId(_applicationId))
_env.blockManager.initialize(_applicationId)
- FallbackStorage.registerBlockManagerIfNeeded(_env.blockManager.master, _conf)
+ FallbackStorage.registerBlockManagerIfNeeded(
+ _env.blockManager.master, _conf, _hadoopConfiguration)
// The metrics system for Driver need to be set spark.app.id to app ID.
// So it should start after we get app ID from the task scheduler and set spark.app.id.
@@ -2934,8 +2935,14 @@ class SparkContext(config: SparkConf) extends Logging {
_driverLogger.foreach(_.startSync(_hadoopConfiguration))
}
- /** Post the application end event */
+ /** Post the application end event and report the final heartbeat */
private def postApplicationEnd(exitCode: Int): Unit = {
+ try {
+ _heartbeater.doReportHeartbeat()
+ } catch {
+ case t: Throwable =>
+ logInfo("Unable to report driver heartbeat metrics when stopping spark context", t);
+ }
listenerBus.post(SparkListenerApplicationEnd(System.currentTimeMillis, Some(exitCode)))
}
@@ -3413,27 +3420,21 @@ object SparkContext extends Logging {
* `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions`.
*/
private def supplementJavaModuleOptions(conf: SparkConf): Unit = {
- def supplement(key: OptionalConfigEntry[String]): Unit = {
- val v = conf.get(key) match {
- case Some(opts) => s"${JavaModuleOptions.defaultModuleOptions()} $opts"
- case None => JavaModuleOptions.defaultModuleOptions()
- }
- conf.set(key.key, v)
+ def supplement(key: String): Unit = {
+ val v = s"${JavaModuleOptions.defaultModuleOptions()} ${conf.get(key, "")}".trim()
+ conf.set(key, v)
}
- supplement(DRIVER_JAVA_OPTIONS)
- supplement(EXECUTOR_JAVA_OPTIONS)
+ supplement(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
+ supplement(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS)
}
private def supplementJavaIPv6Options(conf: SparkConf): Unit = {
- def supplement(key: OptionalConfigEntry[String]): Unit = {
- val v = conf.get(key) match {
- case Some(opts) => s"-Djava.net.preferIPv6Addresses=${Utils.preferIPv6} $opts"
- case None => s"-Djava.net.preferIPv6Addresses=${Utils.preferIPv6}"
- }
- conf.set(key.key, v)
+ def supplement(key: String): Unit = {
+ val v = s"-Djava.net.preferIPv6Addresses=${Utils.preferIPv6} ${conf.get(key, "")}".trim()
+ conf.set(key, v)
}
- supplement(DRIVER_JAVA_OPTIONS)
- supplement(EXECUTOR_JAVA_OPTIONS)
+ supplement(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
+ supplement(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS)
}
}
diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala
index bf6e30f5afade..796dbf4b6d5f8 100644
--- a/core/src/main/scala/org/apache/spark/SparkEnv.scala
+++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala
@@ -32,7 +32,7 @@ import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.api.python.{PythonWorker, PythonWorkerFactory}
import org.apache.spark.broadcast.BroadcastManager
import org.apache.spark.executor.ExecutorBackend
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.memory.{MemoryManager, UnifiedMemoryManager}
diff --git a/core/src/main/scala/org/apache/spark/TaskContext.scala b/core/src/main/scala/org/apache/spark/TaskContext.scala
index 5384fd86a8f19..e4cf5f4bf01a5 100644
--- a/core/src/main/scala/org/apache/spark/TaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/TaskContext.scala
@@ -50,6 +50,15 @@ object TaskContext {
}
}
+ def withTaskContext[T](context: TaskContext)(task: => T): T = {
+ try {
+ TaskContext.setTaskContext(context)
+ task
+ } finally {
+ TaskContext.unset()
+ }
+ }
+
private[this] val taskContext: ThreadLocal[TaskContext] = new ThreadLocal[TaskContext]
// Note: protected[spark] instead of private[spark] to prevent the following two from
diff --git a/core/src/main/scala/org/apache/spark/TaskContextImpl.scala b/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
index f0e844289b9db..a7f09028597af 100644
--- a/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
+++ b/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
@@ -25,7 +25,7 @@ import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.LISTENER
import org.apache.spark.memory.TaskMemoryManager
import org.apache.spark.metrics.MetricsSystem
diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala
index fed15a067c00f..aadfb2125cd69 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -20,7 +20,7 @@ package org.apache.spark
import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream}
import java.net.{HttpURLConnection, InetSocketAddress, URL}
import java.nio.charset.StandardCharsets
-import java.nio.file.{Files => JavaFiles, Paths}
+import java.nio.file.{Files, Paths}
import java.nio.file.attribute.PosixFilePermission.{OWNER_EXECUTE, OWNER_READ, OWNER_WRITE}
import java.security.SecureRandom
import java.security.cert.X509Certificate
@@ -37,8 +37,6 @@ import scala.reflect.{classTag, ClassTag}
import scala.sys.process.Process
import scala.util.Try
-import com.google.common.io.{ByteStreams, Files}
-import org.apache.commons.lang3.StringUtils
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.core.LoggerContext
import org.apache.logging.log4j.core.appender.ConsoleAppender
@@ -97,7 +95,7 @@ private[spark] object TestUtils extends SparkTestUtils {
files.foreach { case (k, v) =>
val entry = new JarEntry(k)
jarStream.putNextEntry(entry)
- ByteStreams.copy(new ByteArrayInputStream(v.getBytes(StandardCharsets.UTF_8)), jarStream)
+ new ByteArrayInputStream(v.getBytes(StandardCharsets.UTF_8)).transferTo(jarStream)
}
jarStream.close()
jarFile.toURI.toURL
@@ -133,7 +131,7 @@ private[spark] object TestUtils extends SparkTestUtils {
jarStream.putNextEntry(jarEntry)
val in = new FileInputStream(file)
- ByteStreams.copy(in, jarStream)
+ in.transferTo(jarStream)
in.close()
}
jarStream.close()
@@ -229,13 +227,13 @@ private[spark] object TestUtils extends SparkTestUtils {
def getAbsolutePathFromExecutable(executable: String): Option[String] = {
val command = if (Utils.isWindows) s"$executable.exe" else executable
if (command.split(File.separator, 2).length == 1 &&
- JavaFiles.isRegularFile(Paths.get(command)) &&
- JavaFiles.isExecutable(Paths.get(command))) {
+ Files.isRegularFile(Paths.get(command)) &&
+ Files.isExecutable(Paths.get(command))) {
Some(Paths.get(command).toAbsolutePath.toString)
} else {
sys.env("PATH").split(Pattern.quote(File.pathSeparator))
- .map(path => Paths.get(s"${StringUtils.strip(path, "\"")}${File.separator}$command"))
- .find(p => JavaFiles.isRegularFile(p) && JavaFiles.isExecutable(p))
+ .map(path => Paths.get(s"${Utils.strip(path, "\"")}${File.separator}$command"))
+ .find(p => Files.isRegularFile(p) && Files.isExecutable(p))
.map(_.toString)
}
}
@@ -413,7 +411,7 @@ private[spark] object TestUtils extends SparkTestUtils {
/** Creates a temp JSON file that contains the input JSON record. */
def createTempJsonFile(dir: File, prefix: String, jsonValue: JValue): String = {
val file = File.createTempFile(prefix, ".json", dir)
- JavaFiles.write(file.toPath, compact(render(jsonValue)).getBytes())
+ Files.write(file.toPath, compact(render(jsonValue)).getBytes())
file.getPath
}
@@ -421,8 +419,8 @@ private[spark] object TestUtils extends SparkTestUtils {
def createTempScriptWithExpectedOutput(dir: File, prefix: String, output: String): String = {
val file = File.createTempFile(prefix, ".sh", dir)
val script = s"cat < "NON_UDF"
@@ -91,6 +95,7 @@ private[spark] object PythonEvalType {
case SQL_COGROUPED_MAP_ARROW_UDF => "SQL_COGROUPED_MAP_ARROW_UDF"
case SQL_TABLE_UDF => "SQL_TABLE_UDF"
case SQL_ARROW_TABLE_UDF => "SQL_ARROW_TABLE_UDF"
+ case SQL_ARROW_UDTF => "SQL_ARROW_UDTF"
case SQL_TRANSFORM_WITH_STATE_PANDAS_UDF => "SQL_TRANSFORM_WITH_STATE_PANDAS_UDF"
case SQL_TRANSFORM_WITH_STATE_PANDAS_INIT_STATE_UDF =>
"SQL_TRANSFORM_WITH_STATE_PANDAS_INIT_STATE_UDF"
@@ -101,6 +106,8 @@ private[spark] object PythonEvalType {
// Arrow UDFs
case SQL_SCALAR_ARROW_UDF => "SQL_SCALAR_ARROW_UDF"
case SQL_SCALAR_ARROW_ITER_UDF => "SQL_SCALAR_ARROW_ITER_UDF"
+ case SQL_GROUPED_AGG_ARROW_UDF => "SQL_GROUPED_AGG_ARROW_UDF"
+ case SQL_WINDOW_AGG_ARROW_UDF => "SQL_WINDOW_AGG_ARROW_UDF"
}
}
@@ -170,6 +177,8 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
protected val faultHandlerEnabled: Boolean = conf.get(PYTHON_WORKER_FAULTHANLDER_ENABLED)
protected val idleTimeoutSeconds: Long = conf.get(PYTHON_WORKER_IDLE_TIMEOUT_SECONDS)
protected val killOnIdleTimeout: Boolean = conf.get(PYTHON_WORKER_KILL_ON_IDLE_TIMEOUT)
+ protected val tracebackDumpIntervalSeconds: Long =
+ conf.get(PYTHON_WORKER_TRACEBACK_DUMP_INTERVAL_SECONDS)
protected val hideTraceback: Boolean = false
protected val simplifiedTraceback: Boolean = false
@@ -267,6 +276,9 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
if (faultHandlerEnabled) {
envVars.put("PYTHON_FAULTHANDLER_DIR", faultHandlerLogDir.toString)
}
+ if (tracebackDumpIntervalSeconds > 0L) {
+ envVars.put("PYTHON_TRACEBACK_DUMP_INTERVAL_SECONDS", tracebackDumpIntervalSeconds.toString)
+ }
// allow the user to set the batch size for the BatchedSerializer on UDFs
envVars.put("PYTHON_UDF_BATCH_SIZE", batchSizeForPythonUDF.toString)
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
index 816ceea327aae..52ad6a93ce4c6 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
@@ -28,7 +28,7 @@ import scala.sys.process.Process
import org.apache.spark.{SparkContext, SparkEnv}
import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{PATH, PYTHON_PACKAGES, PYTHON_VERSION}
import org.apache.spark.util.ArrayImplicits.SparkArrayOps
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
index 64b29585a0d92..e02f10cc3fe69 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
@@ -32,9 +32,9 @@ import scala.jdk.OptionConverters._
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
-import org.apache.spark.internal.config.Python.{PYTHON_UNIX_DOMAIN_SOCKET_DIR, PYTHON_UNIX_DOMAIN_SOCKET_ENABLED}
+import org.apache.spark.internal.config.Python.PYTHON_FACTORY_IDLE_WORKER_MAX_POOL_SIZE
import org.apache.spark.security.SocketAuthHelper
import org.apache.spark.util.{RedirectThread, Utils}
@@ -95,11 +95,12 @@ private[spark] class PythonWorkerFactory(
// so we can also fall back to launching workers, pyspark/worker.py (by default) directly.
private val useDaemon = {
// This flag is ignored on Windows as it's unable to fork.
- !System.getProperty("os.name").startsWith("Windows") && useDaemonEnabled
+ !Utils.isWindows && useDaemonEnabled
}
- private val authHelper = new SocketAuthHelper(SparkEnv.get.conf)
- private val isUnixDomainSock = authHelper.conf.get(PYTHON_UNIX_DOMAIN_SOCKET_ENABLED)
+ private val conf = SparkEnv.get.conf
+ private val authHelper = new SocketAuthHelper(conf)
+ private val isUnixDomainSock = authHelper.isUnixDomainSock
@GuardedBy("self")
private var daemon: Process = null
@@ -111,7 +112,11 @@ private[spark] class PythonWorkerFactory(
@GuardedBy("self")
private var daemonSockPath: String = _
@GuardedBy("self")
- private val idleWorkers = new mutable.Queue[PythonWorker]()
+ // Visible for testing
+ private[spark] val idleWorkers = new mutable.Queue[PythonWorker]()
+ @GuardedBy("self")
+ private val maxIdleWorkerPoolSize =
+ conf.get(PYTHON_FACTORY_IDLE_WORKER_MAX_POOL_SIZE)
@GuardedBy("self")
private var lastActivityNs = 0L
new MonitorThread().start()
@@ -127,15 +132,16 @@ private[spark] class PythonWorkerFactory(
def create(): (PythonWorker, Option[ProcessHandle]) = {
if (useDaemon) {
self.synchronized {
- // Pull from idle workers until we one that is alive, otherwise create a new one.
+ // Pull from idle workers until we get one that is alive, otherwise create a new one.
while (idleWorkers.nonEmpty) {
val worker = idleWorkers.dequeue()
- val workerHandle = daemonWorkers(worker)
- if (workerHandle.isAlive()) {
- try {
- return (worker.refresh(), Some(workerHandle))
- } catch {
- case c: CancelledKeyException => /* pass */
+ daemonWorkers.get(worker).foreach { workerHandle =>
+ if (workerHandle.isAlive()) {
+ try {
+ return (worker.refresh(), Some(workerHandle))
+ } catch {
+ case _: CancelledKeyException => /* pass */
+ }
}
}
logWarning(log"Worker ${MDC(WORKER, worker)} " +
@@ -202,8 +208,7 @@ private[spark] class PythonWorkerFactory(
blockingMode: Boolean): (PythonWorker, Option[ProcessHandle]) = {
var serverSocketChannel: ServerSocketChannel = null
lazy val sockPath = new File(
- authHelper.conf.get(PYTHON_UNIX_DOMAIN_SOCKET_DIR)
- .getOrElse(System.getProperty("java.io.tmpdir")),
+ authHelper.sockDir,
s".${UUID.randomUUID()}.sock")
try {
if (isUnixDomainSock) {
@@ -306,8 +311,7 @@ private[spark] class PythonWorkerFactory(
if (isUnixDomainSock) {
workerEnv.put(
"PYTHON_WORKER_FACTORY_SOCK_DIR",
- authHelper.conf.get(PYTHON_UNIX_DOMAIN_SOCKET_DIR)
- .getOrElse(System.getProperty("java.io.tmpdir")))
+ authHelper.sockDir)
workerEnv.put("PYTHON_UNIX_DOMAIN_ENABLED", "True")
} else {
workerEnv.put("PYTHON_WORKER_FACTORY_SECRET", authHelper.secret)
@@ -482,6 +486,15 @@ private[spark] class PythonWorkerFactory(
if (useDaemon) {
self.synchronized {
lastActivityNs = System.nanoTime()
+ if (maxIdleWorkerPoolSize.exists(idleWorkers.size >= _)) {
+ val oldestWorker = idleWorkers.dequeue()
+ try {
+ stopWorker(oldestWorker)
+ } catch {
+ case e: Exception =>
+ logWarning("Failed to stop evicted worker", e)
+ }
+ }
idleWorkers.enqueue(worker)
}
} else {
diff --git a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
index 0fe57dd0bb0ae..e6f2a68567648 100644
--- a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
@@ -28,7 +28,7 @@ import net.razorvine.pickle.{Pickler, Unpickler}
import org.apache.spark.SparkException
import org.apache.spark.api.java.JavaRDD
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rdd.RDD
import org.apache.spark.util.ArrayImplicits._
diff --git a/core/src/main/scala/org/apache/spark/api/python/StreamingPythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/StreamingPythonRunner.scala
index 7eba574751b46..e61fa01db113b 100644
--- a/core/src/main/scala/org/apache/spark/api/python/StreamingPythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/StreamingPythonRunner.scala
@@ -23,7 +23,7 @@ import java.nio.channels.Channels
import scala.jdk.CollectionConverters._
import org.apache.spark.{SparkEnv, SparkPythonException}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{PYTHON_WORKER_MODULE, PYTHON_WORKER_RESPONSE, SESSION_ID}
import org.apache.spark.internal.config.BUFFER_SIZE
import org.apache.spark.internal.config.Python.{PYTHON_AUTH_SOCKET_TIMEOUT, PYTHON_UNIX_DOMAIN_SOCKET_ENABLED}
diff --git a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
index c3d01ec47458e..622833c9cd6af 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
@@ -26,7 +26,7 @@ import io.netty.handler.timeout.ReadTimeoutException
import org.apache.spark.{SparkConf, SparkEnv}
import org.apache.spark.api.r.SerDe._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.R._
import org.apache.spark.util.{ThreadUtils, Utils}
diff --git a/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala
index 3adb540a7ad18..ad1a274170a2a 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala
@@ -22,7 +22,7 @@ import java.io.Serializable
import scala.reflect.ClassTag
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.util.Utils
/**
diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
index 0c7ec5c1a98a7..0f93db9b96b58 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
@@ -27,7 +27,7 @@ import scala.reflect.ClassTag
import scala.util.Random
import org.apache.spark._
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.io.CompressionCodec
import org.apache.spark.serializer.Serializer
diff --git a/core/src/main/scala/org/apache/spark/deploy/Client.scala b/core/src/main/scala/org/apache/spark/deploy/Client.scala
index 226a6dcd36a16..2891247f20104 100644
--- a/core/src/main/scala/org/apache/spark/deploy/Client.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/Client.scala
@@ -31,7 +31,7 @@ import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.DeployMessages._
import org.apache.spark.deploy.master.{DriverState, Master}
import org.apache.spark.deploy.master.DriverState.DriverState
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.Network.RPC_ASK_TIMEOUT
import org.apache.spark.resource.ResourceUtils
diff --git a/core/src/main/scala/org/apache/spark/deploy/DriverTimeoutPlugin.scala b/core/src/main/scala/org/apache/spark/deploy/DriverTimeoutPlugin.scala
index 736c23556ec15..c1db3ddebc8ca 100644
--- a/core/src/main/scala/org/apache/spark/deploy/DriverTimeoutPlugin.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/DriverTimeoutPlugin.scala
@@ -23,7 +23,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.SparkContext
import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, PluginContext, SparkPlugin}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.DRIVER_TIMEOUT
import org.apache.spark.util.{SparkExitCode, ThreadUtils}
diff --git a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
index e21c772c00779..fec69aed48981 100644
--- a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.CountDownLatch
import scala.jdk.CollectionConverters._
import org.apache.spark.{SecurityManager, SparkConf}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.{AUTH_ENABLED, PORT, SHUFFLE_DB_BACKEND_KEY, SHUFFLE_DB_BACKEND_NAME}
import org.apache.spark.metrics.{MetricsSystem, MetricsSystemInstances}
import org.apache.spark.network.TransportContext
diff --git a/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala b/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
index 263b1a233b808..e03dc2ff00a08 100644
--- a/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
@@ -24,7 +24,7 @@ import scala.collection.mutable.ArrayBuffer
import org.apache.spark.SparkConf
import org.apache.spark.deploy.master.Master
import org.apache.spark.deploy.worker.Worker
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.rpc.RpcEnv
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala b/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala
index 5d996381a485e..f14c2dd693975 100644
--- a/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala
@@ -24,10 +24,8 @@ import java.util.zip.{ZipEntry, ZipOutputStream}
import scala.jdk.CollectionConverters._
-import com.google.common.io.{ByteStreams, Files}
-
import org.apache.spark.api.r.RUtils
-import org.apache.spark.internal.{LogEntry, Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{LogEntry, Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.{RedirectThread, Utils}
@@ -153,11 +151,11 @@ private[deploy] object RPackageUtils extends Logging {
if (verbose) {
print(log"Creating directory: ${MDC(PATH, dir)}", printStream)
}
- dir.mkdirs
+ Utils.createDirectory(dir)
} else {
val inStream = jar.getInputStream(entry)
val outPath = new File(tempDir, entryPath)
- Files.createParentDirs(outPath)
+ Utils.createParentDirs(outPath)
val outStream = new FileOutputStream(outPath)
if (verbose) {
print(log"Extracting ${MDC(JAR_ENTRY, entry)} to ${MDC(PATH, outPath)}", printStream)
@@ -251,7 +249,7 @@ private[deploy] object RPackageUtils extends Logging {
val fis = new FileInputStream(file)
val zipEntry = new ZipEntry(relPath)
zipOutputStream.putNextEntry(zipEntry)
- ByteStreams.copy(fis, zipOutputStream)
+ fis.transferTo(zipOutputStream)
zipOutputStream.closeEntry()
fis.close()
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/RedirectConsolePlugin.scala b/core/src/main/scala/org/apache/spark/deploy/RedirectConsolePlugin.scala
new file mode 100644
index 0000000000000..cc1995a264feb
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/deploy/RedirectConsolePlugin.scala
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy
+
+import java.io.{ByteArrayOutputStream, PrintStream}
+import java.util.{Collections, Map => JMap}
+
+import org.apache.spark.SparkContext
+import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, PluginContext, SparkPlugin}
+import org.apache.spark.internal.{Logging, SparkLoggerFactory}
+import org.apache.spark.internal.config._
+
+/**
+ * A built-in plugin to allow redirecting stdout/stderr to logging system (SLF4J).
+ */
+class RedirectConsolePlugin extends SparkPlugin {
+ override def driverPlugin(): DriverPlugin = new DriverRedirectConsolePlugin()
+
+ override def executorPlugin(): ExecutorPlugin = new ExecRedirectConsolePlugin()
+}
+
+object RedirectConsolePlugin {
+
+ def redirectStdoutToLog(): Unit = {
+ val stdoutLogger = SparkLoggerFactory.getLogger("stdout")
+ System.setOut(new LoggingPrintStream(stdoutLogger.info))
+ }
+
+ def redirectStderrToLog(): Unit = {
+ val stderrLogger = SparkLoggerFactory.getLogger("stderr")
+ System.setErr(new LoggingPrintStream(stderrLogger.error))
+ }
+}
+
+class DriverRedirectConsolePlugin extends DriverPlugin with Logging {
+
+ override def init(sc: SparkContext, ctx: PluginContext): JMap[String, String] = {
+ val outputs = sc.conf.get(DRIVER_REDIRECT_CONSOLE_OUTPUTS)
+ if (outputs.contains("stdout")) {
+ logInfo("Redirect driver's stdout to logging system.")
+ RedirectConsolePlugin.redirectStdoutToLog()
+ }
+ if (outputs.contains("stderr")) {
+ logInfo("Redirect driver's stderr to logging system.")
+ RedirectConsolePlugin.redirectStderrToLog()
+ }
+ Collections.emptyMap
+ }
+}
+
+class ExecRedirectConsolePlugin extends ExecutorPlugin with Logging {
+
+ override def init(ctx: PluginContext, extraConf: JMap[String, String]): Unit = {
+ val outputs = ctx.conf.get(EXEC_REDIRECT_CONSOLE_OUTPUTS)
+ if (outputs.contains("stdout")) {
+ logInfo("Redirect executor's stdout to logging system.")
+ RedirectConsolePlugin.redirectStdoutToLog()
+ }
+ if (outputs.contains("stderr")) {
+ logInfo("Redirect executor's stderr to logging system.")
+ RedirectConsolePlugin.redirectStderrToLog()
+ }
+ }
+}
+
+private[spark] class LoggingPrintStream(redirect: String => Unit)
+ extends PrintStream(new LineBuffer(4 * 1024 * 1024)) {
+
+ override def write(b: Int): Unit = {
+ super.write(b)
+ tryLogCurrentLine()
+ }
+
+ override def write(buf: Array[Byte], off: Int, len: Int): Unit = {
+ super.write(buf, off, len)
+ tryLogCurrentLine()
+ }
+
+ private def tryLogCurrentLine(): Unit = this.synchronized {
+ out.asInstanceOf[LineBuffer].tryGenerateContext.foreach { logContext =>
+ redirect(logContext)
+ }
+ }
+}
+
+/**
+ * Cache bytes before line ending. When current line is ended or the bytes size reaches the
+ * threshold, it can generate the line.
+ */
+private[spark] object LineBuffer {
+ private val LF_BYTES = System.lineSeparator.getBytes
+ private val LF_LENGTH = LF_BYTES.length
+}
+
+private[spark] class LineBuffer(lineMaxBytes: Long) extends ByteArrayOutputStream {
+
+ import LineBuffer._
+
+ def tryGenerateContext: Option[String] =
+ if (isLineEnded) {
+ try Some(new String(buf, 0, count - LF_LENGTH)) finally reset()
+ } else if (count >= lineMaxBytes) {
+ try Some(new String(buf, 0, count)) finally reset()
+ } else {
+ None
+ }
+
+ private def isLineEnded: Boolean = {
+ if (count < LF_LENGTH) return false
+ // fast return in UNIX-like OS when LF is single char '\n'
+ if (LF_LENGTH == 1) return LF_BYTES(0) == buf(count - 1)
+
+ var i = 0
+ do {
+ if (LF_BYTES(i) != buf(count - LF_LENGTH + i)) {
+ return false
+ }
+ i = i + 1
+ } while (i < LF_LENGTH)
+ true
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index ca932ef5dc05c..4af2aa6394b10 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -36,8 +36,8 @@ import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.hadoop.security.token.{Token, TokenIdentifier}
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier
-import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.{ReadOnlySparkConf, SparkConf, SparkException}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.BUFFER_SIZE
import org.apache.spark.util.ArrayImplicits._
import org.apache.spark.util.Utils
@@ -90,7 +90,7 @@ private[spark] class SparkHadoopUtil extends Logging {
* Appends spark.hadoop.* configurations from a [[SparkConf]] to a Hadoop
* configuration without the spark.hadoop. prefix.
*/
- def appendSparkHadoopConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = {
+ def appendSparkHadoopConfigs(conf: ReadOnlySparkConf, hadoopConf: Configuration): Unit = {
SparkHadoopUtil.appendSparkHadoopConfigs(conf, hadoopConf)
}
@@ -430,14 +430,14 @@ private[spark] object SparkHadoopUtil extends Logging {
* and if found on the classpath, those of core-site.xml.
* This is done before the spark overrides are applied.
*/
- private[spark] def newConfiguration(conf: SparkConf): Configuration = {
+ private[spark] def newConfiguration(conf: ReadOnlySparkConf): Configuration = {
val hadoopConf = new Configuration()
appendS3AndSparkHadoopHiveConfigurations(conf, hadoopConf)
hadoopConf
}
private def appendS3AndSparkHadoopHiveConfigurations(
- conf: SparkConf,
+ conf: ReadOnlySparkConf,
hadoopConf: Configuration): Unit = {
// Note: this null check is around more than just access to the "conf" object to maintain
// the behavior of the old implementation of this code, for backwards compatibility.
@@ -514,7 +514,7 @@ private[spark] object SparkHadoopUtil extends Logging {
}
}
- private def appendSparkHadoopConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = {
+ private def appendSparkHadoopConfigs(conf: ReadOnlySparkConf, hadoopConf: Configuration): Unit = {
// Copy any "spark.hadoop.foo=bar" spark properties into conf as "foo=bar"
for ((key, value) <- conf.getAll if key.startsWith("spark.hadoop.")) {
hadoopConf.set(key.substring("spark.hadoop.".length), value,
@@ -532,7 +532,7 @@ private[spark] object SparkHadoopUtil extends Logging {
}
}
- private def appendSparkHiveConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = {
+ private def appendSparkHiveConfigs(conf: ReadOnlySparkConf, hadoopConf: Configuration): Unit = {
// Copy any "spark.hive.foo=bar" spark properties into conf as "hive.foo=bar"
for ((key, value) <- conf.getAll if key.startsWith("spark.hive.")) {
hadoopConf.set(key.substring("spark.".length), value, SOURCE_SPARK_HIVE)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkPipelines.scala b/core/src/main/scala/org/apache/spark/deploy/SparkPipelines.scala
new file mode 100644
index 0000000000000..713937cadabfb
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkPipelines.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy
+
+import java.util
+import java.util.Locale
+
+import scala.collection.mutable.ArrayBuffer
+import scala.jdk.CollectionConverters._
+
+import org.apache.spark.SparkUserAppException
+import org.apache.spark.internal.Logging
+import org.apache.spark.launcher.SparkLauncher.SPARK_API_MODE
+import org.apache.spark.launcher.SparkSubmitArgumentsParser
+import org.apache.spark.util.SparkExitCode
+
+/**
+ * Outer implementation of the spark-pipelines command line interface. Responsible for routing
+ * spark-submit args to spark-submit, and pipeline-specific args to the inner Python CLI
+ * implementation that loads the user code and submits it to the backend.
+ */
+object SparkPipelines extends Logging {
+ def main(args: Array[String]): Unit = {
+ val pipelinesCliFile = args(0)
+ val sparkSubmitAndPipelinesArgs = args.slice(1, args.length)
+ SparkSubmit.main(
+ constructSparkSubmitArgs(pipelinesCliFile, sparkSubmitAndPipelinesArgs).toArray)
+ }
+
+ protected[deploy] def constructSparkSubmitArgs(
+ pipelinesCliFile: String,
+ args: Array[String]): Seq[String] = {
+ val (sparkSubmitArgs, pipelinesArgs) = splitArgs(args)
+ (sparkSubmitArgs ++ Seq(pipelinesCliFile) ++ pipelinesArgs)
+ }
+
+ /**
+ * Split the arguments into spark-submit args (--master, --remote, etc.) and pipeline args
+ * (run, --spec, etc.).
+ */
+ private def splitArgs(args: Array[String]): (Seq[String], Seq[String]) = {
+ val sparkSubmitArgs = new ArrayBuffer[String]()
+ val pipelinesArgs = new ArrayBuffer[String]()
+ var remote = "local"
+
+ new SparkSubmitArgumentsParser() {
+ parse(util.Arrays.asList(args: _*))
+
+ override protected def handle(opt: String, value: String): Boolean = {
+ if (opt == "--remote") {
+ remote = value
+ } else if (opt == "--class") {
+ logError("--class argument not supported.")
+ throw SparkUserAppException(SparkExitCode.EXIT_FAILURE)
+ } else if ((opt == "--conf" || opt == "-c") && value.startsWith(s"$SPARK_API_MODE=")) {
+ val apiMode = value.stripPrefix(s"$SPARK_API_MODE=").trim
+ if (apiMode.toLowerCase(Locale.ROOT) != "connect") {
+ logError(
+ s"$SPARK_API_MODE must be 'connect' (was '$apiMode'). " +
+ "Declarative Pipelines currently only supports Spark Connect."
+ )
+ throw SparkUserAppException(SparkExitCode.EXIT_FAILURE)
+ }
+ } else if (Seq("--name", "-h", "--help").contains(opt)) {
+ pipelinesArgs += opt
+ if (value != null && value.nonEmpty) {
+ pipelinesArgs += value
+ }
+ } else {
+ sparkSubmitArgs += opt
+ if (value != null) {
+ sparkSubmitArgs += value
+ }
+ }
+
+ true
+ }
+
+ override protected def handleExtraArgs(extra: util.List[String]): Unit = {
+ pipelinesArgs.appendAll(extra.asScala)
+ }
+
+ override protected def handleUnknown(opt: String): Boolean = {
+ pipelinesArgs += opt
+ true
+ }
+ }
+
+ sparkSubmitArgs += "--conf"
+ sparkSubmitArgs += s"$SPARK_API_MODE=connect"
+ sparkSubmitArgs += "--remote"
+ sparkSubmitArgs += remote
+ (sparkSubmitArgs.toSeq, pipelinesArgs.toSeq)
+ }
+
+}
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index c529e37e7e1bf..4fcc3af1d5e2d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -30,7 +30,6 @@ import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
import scala.util.{Properties, Try}
-import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.conf.{Configuration => HadoopConfiguration}
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.security.UserGroupInformation
@@ -39,7 +38,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.spark._
import org.apache.spark.api.r.RUtils
import org.apache.spark.deploy.rest._
-import org.apache.spark.internal.{LogEntry, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{LogEntry, Logging, LogKeys}
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.UI._
import org.apache.spark.launcher.SparkLauncher
@@ -368,7 +367,7 @@ private[spark] class SparkSubmit extends Logging {
// install any R packages that may have been passed through --jars or --packages.
// Spark Packages may contain R source code inside the jar.
- if (args.isR && !StringUtils.isBlank(args.jars)) {
+ if (args.isR && !SparkStringUtils.isBlank(args.jars)) {
RPackageUtils.checkAndBuildRPackage(args.jars, printStream, args.verbose)
}
}
@@ -997,14 +996,14 @@ private[spark] class SparkSubmit extends Logging {
// TODO(SPARK-42375): Should point out the user-facing page here instead.
logInfo("You need to specify Spark Connect jars with --jars or --packages.")
}
- throw new SparkUserAppException(CLASS_NOT_FOUND_EXIT_STATUS)
+ throw new SparkUserAppException(SparkExitCode.CLASS_NOT_FOUND)
case e: NoClassDefFoundError =>
logError(log"Failed to load ${MDC(LogKeys.CLASS_NAME, childMainClass)}", e)
if (e.getMessage.contains("org/apache/hadoop/hive")) {
logInfo("Failed to load hive class.")
logInfo("You need to build Spark with -Phive and -Phive-thriftserver.")
}
- throw new SparkUserAppException(CLASS_NOT_FOUND_EXIT_STATUS)
+ throw new SparkUserAppException(SparkExitCode.CLASS_NOT_FOUND)
}
val app: SparkApplication = if (classOf[SparkApplication].isAssignableFrom(mainClass)) {
@@ -1023,11 +1022,20 @@ private[spark] class SparkSubmit extends Logging {
e
}
+ var exitCode: Int = 1
+ var cause: Throwable = null
try {
app.start(childArgs.toArray, sparkConf)
+ exitCode = 0
} catch {
case t: Throwable =>
- throw findCause(t)
+ cause = findCause(t)
+ cause match {
+ case e: SparkUserAppException =>
+ exitCode = e.exitCode
+ case _ =>
+ }
+ throw cause
} finally {
if (args.master.startsWith("k8s") && !isShell(args.primaryResource) &&
!isSqlShell(args.mainClass) && !isThriftServer(args.mainClass) &&
@@ -1038,6 +1046,12 @@ private[spark] class SparkSubmit extends Logging {
case e: Throwable => logError("Failed to close SparkContext", e)
}
}
+ if (sparkConf.get(SUBMIT_CALL_SYSTEM_EXIT_ON_MAIN_EXIT)) {
+ logInfo(
+ log"Calling System.exit() with exit code ${MDC(LogKeys.EXIT_CODE, exitCode)} " +
+ log"because ${MDC(LogKeys.CONFIG, SUBMIT_CALL_SYSTEM_EXIT_ON_MAIN_EXIT.key)}=true")
+ exitFn(exitCode, Option(cause))
+ }
}
}
@@ -1081,8 +1095,6 @@ object SparkSubmit extends CommandLineUtils with Logging {
private val SPARKR_PACKAGE_ARCHIVE = "sparkr.zip"
private val R_PACKAGE_ARCHIVE = "rpkg.zip"
- private val CLASS_NOT_FOUND_EXIT_STATUS = 101
-
// Following constants are visible for testing.
private[deploy] val YARN_CLUSTER_SUBMIT_CLASS =
"org.apache.spark.deploy.yarn.YarnClusterApplication"
@@ -1132,7 +1144,7 @@ object SparkSubmit extends CommandLineUtils with Logging {
super.doSubmit(args)
} catch {
case e: SparkUserAppException =>
- exitFn(e.exitCode)
+ exitFn(e.exitCode, Some(e))
}
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index f4884385555f6..3eb2fd2a0e4d2 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -28,7 +28,7 @@ import scala.util.Try
import org.apache.spark.{SparkConf, SparkException, SparkUserAppException}
import org.apache.spark.deploy.SparkSubmitAction._
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.DYN_ALLOCATION_ENABLED
import org.apache.spark.launcher.SparkSubmitArgumentsParser
diff --git a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
index 7b98461b01acf..5a1d58897703d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
@@ -27,7 +27,7 @@ import org.json4s.{DefaultFormats, Extraction, Formats}
import org.json4s.jackson.JsonMethods.{compact, render}
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.COMPONENT
import org.apache.spark.resource.{ResourceAllocation, ResourceID, ResourceInformation, ResourceRequirement}
import org.apache.spark.util.ArrayImplicits._
diff --git a/core/src/main/scala/org/apache/spark/deploy/Utils.scala b/core/src/main/scala/org/apache/spark/deploy/Utils.scala
index b3d871d75e6c7..9ff1333b03e62 100644
--- a/core/src/main/scala/org/apache/spark/deploy/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/Utils.scala
@@ -22,7 +22,7 @@ import java.io.File
import jakarta.servlet.http.HttpServletRequest
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{LOG_TYPE, PATH}
import org.apache.spark.ui.JettyUtils.createServletHandler
import org.apache.spark.ui.WebUI
diff --git a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
index b34e5c408c3be..2d742b31f99c2 100644
--- a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
@@ -29,7 +29,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.deploy.{ApplicationDescription, ExecutorState}
import org.apache.spark.deploy.DeployMessages._
import org.apache.spark.deploy.master.Master
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.resource.ResourceProfile
import org.apache.spark.rpc._
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala
index 8caf67ff4680b..e979671890eac 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala
@@ -28,7 +28,7 @@ import jakarta.servlet.{DispatcherType, Filter, FilterChain, ServletException, S
import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse}
import org.eclipse.jetty.servlet.FilterHolder
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.metrics.source.Source
import org.apache.spark.ui.SparkUI
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
index f3f7db6bb0aba..89f0d12935ce1 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
@@ -99,6 +99,15 @@ private[history] abstract class ApplicationHistoryProvider {
*/
def getListing(): Iterator[ApplicationInfo]
+ /**
+ * Returns a list of applications available for the history server to show.
+ *
+ * @param max The maximum number of applications to return
+ * @param predicate A function that filters the applications to be returned
+ * @return An iterator of matching applications up to the specified maximum
+ */
+ def getListing(max: Int)(predicate: ApplicationInfo => Boolean): Iterator[ApplicationInfo]
+
/**
* Returns the Spark UI for a specific application.
*
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala
index 20e60c679b06a..efa5b10c93f15 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala
@@ -23,7 +23,7 @@ import scala.util.control.NonFatal
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.deploy.history.EventFilter.FilterStatistics
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{LINE, LINE_NUM, PATH}
import org.apache.spark.scheduler._
import org.apache.spark.util.{JsonProtocol, Utils}
@@ -72,7 +72,6 @@ private[spark] object EventFilter extends Logging {
fs: FileSystem,
filters: Seq[EventFilter],
path: Path,
- jsonProtocol: JsonProtocol,
onAccepted: (String, SparkListenerEvent) => Unit,
onRejected: (String, SparkListenerEvent) => Unit,
onUnidentified: String => Unit): Unit = {
@@ -82,7 +81,7 @@ private[spark] object EventFilter extends Logging {
lines.zipWithIndex.foreach { case (line, lineNum) =>
try {
val event = try {
- Some(jsonProtocol.sparkEventFromJson(line))
+ Some(JsonProtocol.sparkEventFromJson(line))
} catch {
// ignore any exception occurred from unidentified json
case NonFatal(_) =>
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala
index 5cec3d2a9d28f..e0227641fc0eb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala
@@ -28,10 +28,10 @@ import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
import org.apache.spark.SparkConf
import org.apache.spark.deploy.history.EventFilter.FilterStatistics
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.scheduler.ReplayListenerBus
-import org.apache.spark.util.{JsonProtocol, Utils}
+import org.apache.spark.util.Utils
/**
* This class compacts the old event log files into one compact file, via two phases reading:
@@ -55,8 +55,6 @@ class EventLogFileCompactor(
require(maxFilesToRetain > 0, "Max event log files to retain should be higher than 0.")
- private val jsonProtocol = new JsonProtocol(sparkConf)
-
/**
* Compacts the old event log files into one compact file, and clean old event log files being
* compacted away.
@@ -115,7 +113,7 @@ class EventLogFileCompactor(
* them via replaying events in given files.
*/
private def initializeBuilders(fs: FileSystem, files: Seq[Path]): Seq[EventFilterBuilder] = {
- val bus = new ReplayListenerBus(jsonProtocol)
+ val bus = new ReplayListenerBus()
val builders = ServiceLoader.load(classOf[EventFilterBuilder],
Utils.getContextOrSparkClassLoader).asScala.toSeq
@@ -155,7 +153,7 @@ class EventLogFileCompactor(
val startTime = System.currentTimeMillis()
logWriter.start()
eventLogFiles.foreach { file =>
- EventFilter.applyFilterToFile(fs, filters, file.getPath, jsonProtocol,
+ EventFilter.applyFilterToFile(fs, filters, file.getPath,
onAccepted = (line, _) => logWriter.writeEvent(line, flushLogger = true),
onRejected = (_, _) => {},
onUnidentified = line => logWriter.writeEvent(line, flushLogger = true)
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
index 8827fcde7b733..1721ef51807a5 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
@@ -21,7 +21,6 @@ import java.io.{BufferedInputStream, InputStream}
import java.util.concurrent.ConcurrentHashMap
import java.util.zip.{ZipEntry, ZipOutputStream}
-import com.google.common.io.ByteStreams
import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
import org.apache.hadoop.hdfs.DFSInputStream
@@ -52,7 +51,7 @@ abstract class EventLogFileReader(
entryName: String): Unit = {
Utils.tryWithResource(fileSystem.open(path, 1 * 1024 * 1024)) { inputStream =>
zipStream.putNextEntry(new ZipEntry(entryName))
- ByteStreams.copy(inputStream, zipStream)
+ inputStream.transferTo(zipStream)
zipStream.closeEntry()
}
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
index 990ab680f3aaf..4e3bee1015ff3 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.permission.FsPermission
import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
@@ -43,6 +43,7 @@ import org.apache.spark.util.Utils
* spark.eventLog.compression.codec - The codec to compress logged events
* spark.eventLog.overwrite - Whether to overwrite any existing files
* spark.eventLog.buffer.kb - Buffer size to use when writing to output streams
+ * spark.eventLog.excludedPatterns - Specifes a comma-separated event names to be excluded
*
* Note that descendant classes can maintain its own parameters: refer the javadoc of each class
* for more details.
@@ -58,6 +59,8 @@ abstract class EventLogFileWriter(
protected val shouldCompress = sparkConf.get(EVENT_LOG_COMPRESS) &&
!sparkConf.get(EVENT_LOG_COMPRESSION_CODEC).equalsIgnoreCase("none")
+ protected val excludedPatterns = sparkConf.get(EVENT_LOG_EXCLUDED_PATTERNS)
+ .map(name => s"""{"Event":"$name"""")
protected val shouldOverwrite = sparkConf.get(EVENT_LOG_OVERWRITE)
protected val outputBufferSize = sparkConf.get(EVENT_LOG_OUTPUT_BUFFER_SIZE).toInt * 1024
protected val fileSystem = Utils.getHadoopFileSystem(logBaseDir, hadoopConf)
@@ -117,6 +120,7 @@ abstract class EventLogFileWriter(
}
protected def writeLine(line: String, flushLogger: Boolean = false): Unit = {
+ if (excludedPatterns.exists(line.startsWith(_))) return
// scalastyle:off println
writer.foreach(_.println(line))
// scalastyle:on println
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index ad372a414c007..4863291b529bb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -37,7 +37,7 @@ import org.apache.hadoop.security.AccessControlException
import org.apache.spark.{SecurityManager, SparkConf, SparkException}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
@@ -52,8 +52,9 @@ import org.apache.spark.status._
import org.apache.spark.status.KVUtils._
import org.apache.spark.status.api.v1.{ApplicationAttemptInfo, ApplicationInfo}
import org.apache.spark.ui.SparkUI
-import org.apache.spark.util.{CallerContext, Clock, JsonProtocol, SystemClock, ThreadUtils, Utils}
+import org.apache.spark.util.{CallerContext, Clock, SystemClock, ThreadUtils, Utils}
import org.apache.spark.util.ArrayImplicits._
+import org.apache.spark.util.SparkStringUtils.stringToSeq
import org.apache.spark.util.kvstore._
/**
@@ -101,7 +102,9 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
private val CLEAN_INTERVAL_S = conf.get(History.CLEANER_INTERVAL_S)
// Number of threads used to replay event logs.
- private val NUM_PROCESSING_THREADS = conf.get(History.NUM_REPLAY_THREADS)
+ private val numReplayThreads = conf.get(History.NUM_REPLAY_THREADS)
+ // Number of threads used to compact rolling event logs.
+ private val numCompactThreads = conf.get(History.NUM_COMPACT_THREADS)
private val logDir = conf.get(History.HISTORY_LOG_DIR)
@@ -208,7 +211,20 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
*/
private val replayExecutor: ExecutorService = {
if (!Utils.isTesting) {
- ThreadUtils.newDaemonFixedThreadPool(NUM_PROCESSING_THREADS, "log-replay-executor")
+ ThreadUtils.newDaemonBlockingThreadPoolExecutorService(
+ numReplayThreads, 1024, "log-replay-executor")
+ } else {
+ ThreadUtils.sameThreadExecutorService()
+ }
+ }
+
+ /**
+ * Fixed size thread pool to compact log files.
+ */
+ private val compactExecutor: ExecutorService = {
+ if (!Utils.isTesting) {
+ ThreadUtils.newDaemonBlockingThreadPoolExecutorService(
+ numCompactThreads, 1024, "log-compact-executor")
} else {
ThreadUtils.sameThreadExecutorService()
}
@@ -306,6 +322,14 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
.index("endTime").reverse())(_.toApplicationInfo()).iterator
}
+ override def getListing(max: Int)(
+ predicate: ApplicationInfo => Boolean): Iterator[ApplicationInfo] = {
+ // Return the filtered listing in end time descending order.
+ KVUtils.mapToSeqWithFilter(
+ listing.view(classOf[ApplicationInfoWrapper]).index("endTime").reverse(),
+ max)(_.toApplicationInfo())(predicate).iterator
+ }
+
override def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
try {
Some(load(appId).toApplicationInfo())
@@ -319,17 +343,14 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
override def getLastUpdatedTime(): Long = lastScanTime.get()
- /**
- * Split a comma separated String, filter out any empty items, and return a Sequence of strings
- */
- private def stringToSeq(list: String): Seq[String] = {
- list.split(',').map(_.trim).filter(_.nonEmpty).toImmutableArraySeq
- }
-
override def getAppUI(appId: String, attemptId: Option[String]): Option[LoadedAppUI] = {
+ val logPath = RollingEventLogFilesWriter.EVENT_LOG_DIR_NAME_PREFIX +
+ EventLogFileWriter.nameForAppAndAttempt(appId, attemptId)
val app = try {
load(appId)
} catch {
+ case _: NoSuchElementException if this.conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED) =>
+ loadFromFallbackLocation(appId, attemptId, logPath)
case _: NoSuchElementException =>
return None
}
@@ -351,6 +372,13 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
createInMemoryStore(attempt)
}
} catch {
+ case _: FileNotFoundException if this.conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED) =>
+ if (app.attempts.head.info.appSparkVersion == "unknown") {
+ listing.synchronized {
+ listing.delete(classOf[ApplicationInfoWrapper], appId)
+ }
+ }
+ return None
case _: FileNotFoundException =>
return None
}
@@ -370,6 +398,18 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
Some(loadedUI)
}
+ private def loadFromFallbackLocation(appId: String, attemptId: Option[String], logPath: String)
+ : ApplicationInfoWrapper = {
+ val date = new Date(0)
+ val lastUpdate = new Date()
+ val info = ApplicationAttemptInfo(
+ attemptId, date, date, lastUpdate, 0, "spark", false, "unknown")
+ addListing(new ApplicationInfoWrapper(
+ ApplicationInfo(appId, appId, None, None, None, None, List.empty),
+ List(new AttemptInfoWrapper(info, logPath, 0, Some(1), None, None, None, None))))
+ load(appId)
+ }
+
override def getEmptyListingHtml(): Seq[Node] = {
Did you specify the correct logging directory? Please verify your setting of
@@ -406,7 +446,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
initThread.interrupt()
initThread.join()
}
- Seq(pool, replayExecutor).foreach { executor =>
+ Seq(pool, replayExecutor, compactExecutor).foreach { executor =>
executor.shutdown()
if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
executor.shutdownNow()
@@ -462,7 +502,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
var count: Int = 0
try {
val newLastScanTime = clock.getTimeMillis()
- logDebug(s"Scanning $logDir with lastScanTime==$lastScanTime")
+ logInfo(log"Scanning ${MDC(HISTORY_DIR, logDir)} with " +
+ log"lastScanTime=${MDC(LAST_SCAN_TIME, lastScanTime)}")
// Mark entries that are processing as not stale. Such entries do not have a chance to be
// updated with the new 'lastProcessed' time and thus any entity that completes processing
@@ -470,7 +511,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
// and will be deleted from the UI until the next 'checkForLogs' run.
val notStale = mutable.HashSet[String]()
val updated = Option(fs.listStatus(new Path(logDir)))
- .map(_.toImmutableArraySeq).getOrElse(Nil)
+ .map(_.toImmutableArraySeq).getOrElse(Seq.empty)
.filter { entry => isAccessible(entry.getPath) }
.filter { entry =>
if (isProcessing(entry.getPath)) {
@@ -587,11 +628,11 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
}
if (updated.nonEmpty) {
- logDebug(s"New/updated attempts found: ${updated.size} ${updated.map(_.rootPath)}")
+ logInfo(log"New/updated attempts found: ${MDC(NUM_ATTEMPT, updated.size)}")
}
updated.foreach { entry =>
- submitLogProcessTask(entry.rootPath) { () =>
+ submitLogProcessTask(entry.rootPath, replayExecutor) { () =>
mergeApplicationListing(entry, newLastScanTime, true)
}
}
@@ -763,7 +804,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
// triggering another task for compaction task only if it succeeds
if (succeeded) {
- submitLogProcessTask(rootPath) { () => compact(reader) }
+ submitLogProcessTask(rootPath, compactExecutor) { () => compact(reader) }
}
}
}
@@ -802,7 +843,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
val shouldHalt = enableOptimizations &&
((!appCompleted && fastInProgressParsing) || reparseChunkSize > 0)
- val bus = new ReplayListenerBus(new JsonProtocol(conf))
+ val bus = new ReplayListenerBus()
val listener = new AppListingListener(reader, clock, shouldHalt)
bus.addListener(listener)
@@ -1119,7 +1160,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
// to parse the event logs in the SHS.
val replayConf = conf.clone().set(ASYNC_TRACKING_ENABLED, false)
val trackingStore = new ElementTrackingStore(store, replayConf)
- val replayBus = new ReplayListenerBus(new JsonProtocol(conf))
+ val replayBus = new ReplayListenerBus()
val listener = new AppStatusListener(trackingStore, replayConf, false,
lastUpdateTime = Some(lastUpdated))
replayBus.addListener(listener)
@@ -1431,13 +1472,14 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
}
/** NOTE: 'task' should ensure it executes 'endProcessing' at the end */
- private def submitLogProcessTask(rootPath: Path)(task: Runnable): Unit = {
+ private def submitLogProcessTask(
+ rootPath: Path, pool: ExecutorService)(task: Runnable): Unit = {
try {
processing(rootPath)
- replayExecutor.submit(task)
+ pool.submit(task)
} catch {
// let the iteration over the updated entries break, since an exception on
- // replayExecutor.submit (..) indicates the ExecutorService is unable
+ // pool.submit (..) indicates the ExecutorService is unable
// to take any more submissions at this time
case e: Exception =>
logError(s"Exception while submitting task", e)
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
index ff1629b698096..4eeddd7cc7098 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
@@ -109,7 +109,7 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
}
def shouldDisplayApplications(requestedIncomplete: Boolean): Boolean = {
- parent.getApplicationList().exists(isApplicationCompleted(_) != requestedIncomplete)
+ parent.getApplicationInfoList(1)(isApplicationCompleted(_) != requestedIncomplete).nonEmpty
}
private def makePageLink(request: HttpServletRequest, showIncomplete: Boolean): String = {
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index ce9f70c9a83ed..14daa5d88b1c4 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -28,7 +28,7 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.Utils.addRenderLogHandler
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.History
import org.apache.spark.internal.config.UI._
@@ -224,6 +224,11 @@ class HistoryServer(
getApplicationList()
}
+ override def getApplicationInfoList(max: Int)(
+ filter: ApplicationInfo => Boolean): Iterator[ApplicationInfo] = {
+ provider.getListing(max)(filter)
+ }
+
def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
provider.getApplicationInfo(appId)
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
index f1343a0551384..14d6e4a0381ae 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
@@ -83,7 +83,7 @@ private[history] class HistoryServerArguments(conf: SparkConf, args: Array[Strin
configs.sortBy(_.key).foreach { conf =>
sb.append(" ").append(conf.key.padTo(maxConfigLength, ' '))
var currentDocLen = 0
- val intention = "\n" + " " * (maxConfigLength + 2)
+ val intention = "\n" + " ".repeat(maxConfigLength + 2)
conf.doc.split("\\s+").foreach { word =>
if (currentDocLen + word.length > 60) {
sb.append(intention).append(" ").append(word)
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
index 122ed299242f5..753fe59dc0894 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
@@ -22,10 +22,8 @@ import java.util.concurrent.atomic.AtomicLong
import scala.collection.mutable.{HashMap, ListBuffer}
-import org.apache.commons.io.FileUtils
-
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.History
import org.apache.spark.internal.config.History._
@@ -76,7 +74,7 @@ private class HistoryServerDiskManager(
// Clean up any temporary stores during start up. This assumes that they're leftover from other
// instances and are not useful.
- tmpStoreDir.listFiles().foreach(FileUtils.deleteQuietly)
+ tmpStoreDir.listFiles().foreach(Utils.deleteQuietly)
// Go through the recorded store directories and remove any that may have been removed by
// external code.
@@ -214,7 +212,7 @@ private class HistoryServerDiskManager(
def committed(): Long = committedUsage.get()
private def deleteStore(path: File): Unit = {
- FileUtils.deleteDirectory(path)
+ Utils.deleteRecursively(path)
listing.delete(classOf[ApplicationStoreInfo], path.getAbsolutePath())
}
@@ -286,7 +284,7 @@ private class HistoryServerDiskManager(
}
/** Visible for testing. Return the size of a directory. */
- private[history] def sizeOf(path: File): Long = FileUtils.sizeOf(path)
+ private[history] def sizeOf(path: File): Long = Utils.sizeOf(path)
private[history] class Lease(val tmpPath: File, private val leased: Long) {
@@ -334,7 +332,7 @@ private class HistoryServerDiskManager(
/** Deletes the temporary directory created for the lease. */
def rollback(): Unit = {
updateUsage(-leased)
- FileUtils.deleteDirectory(tmpPath)
+ Utils.deleteRecursively(tmpPath)
}
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala
index 6e3dbb1170998..f9a6c7f64896c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicLong
import scala.collection.mutable.HashMap
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.History._
import org.apache.spark.io.CompressionCodec
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
index 4332544e4491c..0c9baf9e6f70c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
@@ -22,7 +22,7 @@ import java.nio.file.{FileAlreadyExistsException, Files, Paths}
import scala.reflect.ClassTag
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.io.CompressionCodec
import org.apache.spark.serializer.{DeserializationStream, SerializationStream, Serializer}
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 7d15744de6b45..95778771d4239 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -32,7 +32,7 @@ import org.apache.spark.deploy.master.DriverState.DriverState
import org.apache.spark.deploy.master.MasterMessages._
import org.apache.spark.deploy.master.ui.MasterWebUI
import org.apache.spark.deploy.rest.StandaloneRestServer
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.Deploy._
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala b/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala
index 964b115865aef..a751c7bfc67c4 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.master
import org.apache.spark.SparkConf
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.Deploy.{RECOVERY_COMPRESSION_CODEC, RECOVERY_DIRECTORY}
import org.apache.spark.io.CompressionCodec
import org.apache.spark.serializer.Serializer
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
index 6c7a8f582d915..66036e7a5e5ce 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
@@ -24,7 +24,7 @@ import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletRespons
import org.apache.spark.deploy.DeployMessages.{DecommissionWorkersOnHosts, MasterStateResponse, RequestMasterState}
import org.apache.spark.deploy.Utils.addRenderLogHandler
import org.apache.spark.deploy.master.Master
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{HOSTS, NUM_REMOVED_WORKERS}
import org.apache.spark.internal.config.DECOMMISSION_ENABLED
import org.apache.spark.internal.config.UI.MASTER_UI_DECOMMISSION_ALLOW_MODE
diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
index 3c0baacbf10be..6dc422515dbdb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
@@ -33,7 +33,7 @@ import jakarta.servlet.http.HttpServletResponse
import org.apache.spark.{SPARK_VERSION => sparkVersion, SparkConf, SparkException}
import org.apache.spark.deploy.SparkApplication
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
index f8afa86fd36fc..e172a06f0a32e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
@@ -32,7 +32,7 @@ import org.json4s._
import org.json4s.jackson.JsonMethods._
import org.apache.spark.{SPARK_VERSION => sparkVersion, SparkConf}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{MASTER_REST_SERVER_FILTERS, MASTER_REST_SERVER_MAX_THREADS, MASTER_REST_SERVER_VIRTUAL_THREADS}
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
index 3a262a0d19fb5..83f36e1245844 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.security.Credentials
import org.apache.hadoop.security.token.{Token, TokenIdentifier}
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.security.HadoopDelegationTokenProvider
import org.apache.spark.util.Utils
@@ -54,8 +54,8 @@ private[security] class HBaseDelegationTokenProvider
creds.addToken(token.getService, token)
} catch {
case NonFatal(e) =>
- logWarning(Utils.createFailedToGetTokenMessage(serviceName, e) + log" Retrying to fetch " +
- log"HBase security token with ${MDC(SERVICE_NAME, serviceName)} connection parameter.")
+ logWarning(Utils.createFailedToGetTokenMessage(serviceName) + log" Retrying to fetch " +
+ log"HBase security token with ${MDC(SERVICE_NAME, serviceName)} connection parameter.", e)
// Seems to be spark is trying to get the token from HBase 2.x.x version or above where the
// obtainToken(Configuration conf) API has been removed. Lets try obtaining the token from
// another compatible API of HBase service.
@@ -98,7 +98,7 @@ private[security] class HBaseDelegationTokenProvider
creds.addToken(token.getService, token)
} catch {
case NonFatal(e) =>
- logWarning(Utils.createFailedToGetTokenMessage(serviceName, e))
+ logWarning(Utils.createFailedToGetTokenMessage(serviceName), e)
} finally {
if (null != hbaseConnection) {
hbaseConnection.close()
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
index b9d88266ed538..bc77159415473 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
@@ -31,7 +31,7 @@ import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.rpc.RpcEndpointRef
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
index fc750b54d0b8e..3a38eb4a08d7d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier
import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.security.HadoopDelegationTokenProvider
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
index a3e7276fc83e1..5bc17e2164738 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
@@ -24,7 +24,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.{SecurityManager, SSLOptions}
import org.apache.spark.deploy.Command
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.launcher.WorkerCommandBuilder
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
index ca0e024ad1aed..28de0f45eb27f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
@@ -19,19 +19,18 @@ package org.apache.spark.deploy.worker
import java.io._
import java.net.URI
-import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.StandardOpenOption
import scala.jdk.CollectionConverters._
-import com.google.common.io.{Files, FileWriteMode}
-
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.{DriverDescription, SparkHadoopUtil}
import org.apache.spark.deploy.DeployMessages.DriverStateChanged
import org.apache.spark.deploy.StandaloneResourceUtils.prepareResourcesFile
import org.apache.spark.deploy.master.DriverState
import org.apache.spark.deploy.master.DriverState.DriverState
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{DRIVER_RESOURCES_FILE, SPARK_DRIVER_PREFIX}
import org.apache.spark.internal.config.UI.UI_REVERSE_PROXY
@@ -145,7 +144,7 @@ private[deploy] class DriverRunner(
*/
private def createWorkingDirectory(): File = {
val driverDir = new File(workDir, driverId)
- if (!driverDir.exists() && !driverDir.mkdirs()) {
+ if (!driverDir.exists() && !Utils.createDirectory(driverDir)) {
throw new IOException("Failed to create directory " + driverDir)
}
driverDir
@@ -215,8 +214,8 @@ private[deploy] class DriverRunner(
val stderr = new File(baseDir, "stderr")
val redactedCommand = Utils.redactCommandLineArgs(conf, builder.command.asScala.toSeq)
.mkString("\"", "\" \"", "\"")
- val header = "Launch Command: %s\n%s\n\n".format(redactedCommand, "=" * 40)
- Files.asCharSink(stderr, StandardCharsets.UTF_8, FileWriteMode.APPEND).write(header)
+ val header = "Launch Command: %s\n%s\n\n".format(redactedCommand, "=".repeat(40))
+ Files.writeString(stderr.toPath, header, StandardOpenOption.APPEND)
CommandUtils.redirectStream(process.getErrorStream, stderr)
}
runCommandWithRetry(ProcessBuilderLike(builder), initialize, supervise)
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala
index 4f42088903464..1d64dbc224c49 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala
@@ -21,7 +21,7 @@ import java.io.File
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.RPC_ADDRESS
import org.apache.spark.rpc.RpcEnv
import org.apache.spark.util._
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
index d21904dd16ea7..8f0b684a93e81 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
@@ -18,17 +18,15 @@
package org.apache.spark.deploy.worker
import java.io._
-import java.nio.charset.StandardCharsets
+import java.nio.file.Files
import scala.jdk.CollectionConverters._
-import com.google.common.io.Files
-
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.{ApplicationDescription, ExecutorState}
import org.apache.spark.deploy.DeployMessages.ExecutorStateChanged
import org.apache.spark.deploy.StandaloneResourceUtils.prepareResourcesFile
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.SPARK_EXECUTOR_PREFIX
import org.apache.spark.internal.config.UI._
@@ -184,14 +182,14 @@ private[deploy] class ExecutorRunner(
process = builder.start()
val header = "Spark Executor Command: %s\n%s\n\n".format(
- redactedCommand, "=" * 40)
+ redactedCommand, "=".repeat(40))
// Redirect its stdout and stderr to files
val stdout = new File(executorDir, "stdout")
stdoutAppender = FileAppender(process.getInputStream, stdout, conf, true)
val stderr = new File(executorDir, "stderr")
- Files.asCharSink(stderr, StandardCharsets.UTF_8).write(header)
+ Files.writeString(stderr.toPath, header)
stderrAppender = FileAppender(process.getErrorStream, stderr, conf, true)
state = ExecutorState.RUNNING
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index b2ec23887a400..98da33a429eca 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -37,7 +37,7 @@ import org.apache.spark.deploy.ExternalShuffleService
import org.apache.spark.deploy.StandaloneResourceUtils._
import org.apache.spark.deploy.master.{DriverState, Master}
import org.apache.spark.deploy.worker.ui.WorkerWebUI
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.Tests.IS_TESTING
import org.apache.spark.internal.config.UI._
@@ -600,7 +600,7 @@ private[deploy] class Worker(
// Create the executor's working directory
val executorDir = new File(workDir, appId + "/" + execId)
- if (!executorDir.mkdirs()) {
+ if (!Utils.createDirectory(executorDir)) {
throw new IOException("Failed to create directory " + executorDir)
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
index bd07a0ade523d..ef6d84631c83c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.worker
import java.util.concurrent.atomic.AtomicBoolean
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.LogKeys.WORKER_URL
import org.apache.spark.rpc._
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
index defce5acc6168..bdc143be91b94 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
@@ -23,7 +23,7 @@ import scala.xml.{Node, Unparsed}
import jakarta.servlet.http.HttpServletRequest
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{LOG_TYPE, PATH}
import org.apache.spark.ui.{UIUtils, WebUIPage}
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
index a30759e5d794e..206a6a0fe385c 100644
--- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
+++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@ -31,7 +31,7 @@ import org.apache.spark._
import org.apache.spark.TaskState.TaskState
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.worker.WorkerWatcher
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.network.netty.SparkTransportConf
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 3b066e15386b8..a14ba21a0c186 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -36,11 +36,11 @@ import scala.util.control.NonFatal
import com.google.common.cache.{Cache, CacheBuilder, RemovalListener, RemovalNotification}
import com.google.common.util.concurrent.ThreadFactoryBuilder
-import org.slf4j.MDC
+import org.slf4j.{MDC => SLF4JMDC}
import org.apache.spark._
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, LogKeys, MDC => LogMDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.{EXECUTOR_USER_CLASS_PATH_FIRST => EXECUTOR_USER_CLASS_PATH_FIRST_CONFIG}
@@ -82,12 +82,12 @@ private[spark] class Executor(
resources: immutable.Map[String, ResourceInformation])
extends Logging {
- logInfo(log"Starting executor ID ${LogMDC(LogKeys.EXECUTOR_ID, executorId)}" +
- log" on host ${LogMDC(HOST, executorHostname)}")
- logInfo(log"OS info ${LogMDC(OS_NAME, System.getProperty("os.name"))}," +
- log" ${LogMDC(OS_VERSION, System.getProperty("os.version"))}, " +
- log"${LogMDC(OS_ARCH, System.getProperty("os.arch"))}")
- logInfo(log"Java version ${LogMDC(JAVA_VERSION, System.getProperty("java.version"))}")
+ logInfo(log"Starting executor ID ${MDC(LogKeys.EXECUTOR_ID, executorId)}" +
+ log" on host ${MDC(HOST, executorHostname)}")
+ logInfo(log"OS info ${MDC(OS_NAME, Utils.osName)}," +
+ log" ${MDC(OS_VERSION, Utils.osVersion)}, " +
+ log"${MDC(OS_ARCH, Utils.osArch)}")
+ logInfo(log"Java version ${MDC(JAVA_VERSION, Utils.javaVersion)}")
private val executorShutdown = new AtomicBoolean(false)
val stopHookReference = ShutdownHookManager.addShutdownHook(
@@ -102,7 +102,7 @@ private[spark] class Executor(
private[executor] val taskNameMDCKey = if (conf.get(LEGACY_TASK_NAME_MDC_ENABLED)) {
"mdc.taskName"
} else {
- LogKeys.TASK_NAME.name
+ TASK_NAME.name.toLowerCase(Locale.ROOT)
}
// SPARK-40235: updateDependencies() uses a ReentrantLock instead of the `synchronized` keyword
@@ -223,7 +223,7 @@ private[spark] class Executor(
if (sessionBasedRoot.isDirectory && sessionBasedRoot.exists()) {
Utils.deleteRecursively(sessionBasedRoot)
}
- logInfo(log"Session evicted: ${LogMDC(SESSION_ID, state.sessionUUID)}")
+ logInfo(log"Session evicted: ${MDC(SESSION_ID, state.sessionUUID)}")
}
})
.build[String, IsolatedSessionState]
@@ -505,9 +505,9 @@ private[spark] class Executor(
@volatile var task: Task[Any] = _
def kill(interruptThread: Boolean, reason: String): Unit = {
- logInfo(log"Executor is trying to kill ${LogMDC(TASK_NAME, taskName)}, " +
- log"interruptThread: ${LogMDC(INTERRUPT_THREAD, interruptThread)}, " +
- log"reason: ${LogMDC(REASON, reason)}")
+ logInfo(log"Executor is trying to kill ${MDC(TASK_NAME, taskName)}, " +
+ log"interruptThread: ${MDC(INTERRUPT_THREAD, interruptThread)}, " +
+ log"reason: ${MDC(REASON, reason)}")
reasonIfKilled = Some(reason)
if (task != null) {
synchronized {
@@ -579,7 +579,7 @@ private[spark] class Executor(
} else 0L
Thread.currentThread.setContextClassLoader(isolatedSession.replClassLoader)
val ser = env.closureSerializer.newInstance()
- logInfo(log"Running ${LogMDC(TASK_NAME, taskName)}")
+ logInfo(log"Running ${MDC(TASK_NAME, taskName)}")
execBackend.statusUpdate(taskId, TaskState.RUNNING, EMPTY_BYTE_BUFFER)
var taskStartTimeNs: Long = 0
var taskStartCpu: Long = 0
@@ -653,7 +653,7 @@ private[spark] class Executor(
if (freedMemory > 0 && !threwException) {
val errMsg = log"Managed memory leak detected; size = " +
- log"${LogMDC(NUM_BYTES, freedMemory)} bytes, ${LogMDC(TASK_NAME, taskName)}"
+ log"${MDC(NUM_BYTES, freedMemory)} bytes, ${MDC(TASK_NAME, taskName)}"
if (conf.get(UNSAFE_EXCEPTION_ON_MEMORY_LEAK)) {
throw SparkException.internalError(errMsg.message, category = "EXECUTOR")
} else {
@@ -663,9 +663,9 @@ private[spark] class Executor(
if (releasedLocks.nonEmpty && !threwException) {
val errMsg =
- log"${LogMDC(NUM_RELEASED_LOCKS, releasedLocks.size)} block locks" +
- log" were not released by ${LogMDC(TASK_NAME, taskName)}\n" +
- log" ${LogMDC(RELEASED_LOCKS, releasedLocks.mkString("[", ", ", "]"))})"
+ log"${MDC(NUM_RELEASED_LOCKS, releasedLocks.size)} block locks" +
+ log" were not released by ${MDC(TASK_NAME, taskName)}\n" +
+ log" ${MDC(RELEASED_LOCKS, releasedLocks.mkString("[", ", ", "]"))})"
if (conf.get(STORAGE_EXCEPTION_PIN_LEAK)) {
throw SparkException.internalError(errMsg.message, category = "EXECUTOR")
} else {
@@ -677,10 +677,10 @@ private[spark] class Executor(
// uh-oh. it appears the user code has caught the fetch-failure without throwing any
// other exceptions. Its *possible* this is what the user meant to do (though highly
// unlikely). So we will log an error and keep going.
- logError(log"${LogMDC(TASK_NAME, taskName)} completed successfully though internally " +
+ logError(log"${MDC(TASK_NAME, taskName)} completed successfully though internally " +
log"it encountered unrecoverable fetch failures! Most likely this means user code " +
log"is incorrectly swallowing Spark's internal " +
- log"${LogMDC(CLASS_NAME, classOf[FetchFailedException])}", fetchFailure)
+ log"${MDC(CLASS_NAME, classOf[FetchFailedException])}", fetchFailure)
}
val taskFinishNs = System.nanoTime()
val taskFinishCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) {
@@ -746,10 +746,10 @@ private[spark] class Executor(
// directSend = sending directly back to the driver
val serializedResult: ByteBuffer = {
if (maxResultSize > 0 && resultSize > maxResultSize) {
- logWarning(log"Finished ${LogMDC(TASK_NAME, taskName)}. " +
+ logWarning(log"Finished ${MDC(TASK_NAME, taskName)}. " +
log"Result is larger than maxResultSize " +
- log"(${LogMDC(RESULT_SIZE_BYTES, Utils.bytesToString(resultSize))} > " +
- log"${LogMDC(RESULT_SIZE_BYTES_MAX, Utils.bytesToString(maxResultSize))}), " +
+ log"(${MDC(RESULT_SIZE_BYTES, Utils.bytesToString(resultSize))} > " +
+ log"${MDC(RESULT_SIZE_BYTES_MAX, Utils.bytesToString(maxResultSize))}), " +
log"dropping it.")
ser.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize))
} else if (resultSize > maxDirectResultSize) {
@@ -758,12 +758,12 @@ private[spark] class Executor(
blockId,
serializedDirectResult,
StorageLevel.MEMORY_AND_DISK_SER)
- logInfo(log"Finished ${LogMDC(TASK_NAME, taskName)}." +
- log" ${LogMDC(NUM_BYTES, resultSize)} bytes result sent via BlockManager)")
+ logInfo(log"Finished ${MDC(TASK_NAME, taskName)}." +
+ log" ${MDC(NUM_BYTES, resultSize)} bytes result sent via BlockManager)")
ser.serialize(new IndirectTaskResult[Any](blockId, resultSize))
} else {
- logInfo(log"Finished ${LogMDC(TASK_NAME, taskName)}." +
- log" ${LogMDC(NUM_BYTES, resultSize)} bytes result sent to driver")
+ logInfo(log"Finished ${MDC(TASK_NAME, taskName)}." +
+ log" ${MDC(NUM_BYTES, resultSize)} bytes result sent to driver")
// toByteBuffer is safe here, guarded by maxDirectResultSize
serializedDirectResult.toByteBuffer
}
@@ -775,8 +775,8 @@ private[spark] class Executor(
execBackend.statusUpdate(taskId, TaskState.FINISHED, serializedResult)
} catch {
case t: TaskKilledException =>
- logInfo(log"Executor killed ${LogMDC(TASK_NAME, taskName)}," +
- log" reason: ${LogMDC(REASON, t.reason)}")
+ logInfo(log"Executor killed ${MDC(TASK_NAME, taskName)}," +
+ log" reason: ${MDC(REASON, t.reason)}")
val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs)
// Here and below, put task metric peaks in an immutable.ArraySeq to expose them as an
@@ -789,8 +789,8 @@ private[spark] class Executor(
case _: InterruptedException | NonFatal(_) if
task != null && task.reasonIfKilled.isDefined =>
val killReason = task.reasonIfKilled.getOrElse("unknown reason")
- logInfo(log"Executor interrupted and killed ${LogMDC(TASK_NAME, taskName)}," +
- log" reason: ${LogMDC(REASON, killReason)}")
+ logInfo(log"Executor interrupted and killed ${MDC(TASK_NAME, taskName)}," +
+ log" reason: ${MDC(REASON, killReason)}")
val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs)
val metricPeaks = metricsPoller.getTaskMetricPeaks(taskId).toImmutableArraySeq
@@ -803,12 +803,12 @@ private[spark] class Executor(
if (!t.isInstanceOf[FetchFailedException]) {
// there was a fetch failure in the task, but some user code wrapped that exception
// and threw something else. Regardless, we treat it as a fetch failure.
- logWarning(log"${LogMDC(TASK_NAME, taskName)} encountered a " +
- log"${LogMDC(CLASS_NAME, classOf[FetchFailedException].getName)} " +
+ logWarning(log"${MDC(TASK_NAME, taskName)} encountered a " +
+ log"${MDC(CLASS_NAME, classOf[FetchFailedException].getName)} " +
log"and failed, but the " +
- log"${LogMDC(CLASS_NAME, classOf[FetchFailedException].getName)} " +
+ log"${MDC(CLASS_NAME, classOf[FetchFailedException].getName)} " +
log"was hidden by another exception. Spark is handling this like a fetch failure " +
- log"and ignoring the other exception: ${LogMDC(ERROR, t)}")
+ log"and ignoring the other exception: ${MDC(ERROR, t)}")
}
setTaskFinishedAndClearInterruptStatus()
plugins.foreach(_.onTaskFailed(reason))
@@ -823,13 +823,13 @@ private[spark] class Executor(
case t: Throwable if env.isStopped =>
// Log the expected exception after executor.stop without stack traces
// see: SPARK-19147
- logError(log"Exception in ${LogMDC(TASK_NAME, taskName)}: ${LogMDC(ERROR, t.getMessage)}")
+ logError(log"Exception in ${MDC(TASK_NAME, taskName)}: ${MDC(ERROR, t.getMessage)}")
case t: Throwable =>
// Attempt to exit cleanly by informing the driver of our failure.
// If anything goes wrong (or this was a fatal exception), we will delegate to
// the default uncaught exception handler, which will terminate the Executor.
- logError(log"Exception in ${LogMDC(TASK_NAME, taskName)}", t)
+ logError(log"Exception in ${MDC(TASK_NAME, taskName)}", t)
// SPARK-20904: Do not report failure to driver if if happened during shut down. Because
// libraries may set up shutdown hooks that race with running tasks during shutdown,
@@ -934,16 +934,16 @@ private[spark] class Executor(
private def setMDCForTask(taskName: String, mdc: Seq[(String, String)]): Unit = {
if (Executor.mdcIsSupported) {
- mdc.foreach { case (key, value) => MDC.put(key, value) }
+ mdc.foreach { case (key, value) => SLF4JMDC.put(key, value) }
// avoid overriding the takName by the user
- MDC.put(taskNameMDCKey, taskName)
+ SLF4JMDC.put(taskNameMDCKey, taskName)
}
}
private def cleanMDCForTask(taskName: String, mdc: Seq[(String, String)]): Unit = {
if (Executor.mdcIsSupported) {
- mdc.foreach { case (key, _) => MDC.remove(key) }
- MDC.remove(taskNameMDCKey)
+ mdc.foreach { case (key, _) => SLF4JMDC.remove(key) }
+ SLF4JMDC.remove(taskNameMDCKey)
}
}
@@ -1016,14 +1016,14 @@ private[spark] class Executor(
finished = true
} else {
val elapsedTimeMs = TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs)
- logWarning(log"Killed task ${LogMDC(TASK_ID, taskId)} " +
- log"is still running after ${LogMDC(TIME_UNITS, elapsedTimeMs)} ms")
+ logWarning(log"Killed task ${MDC(TASK_ID, taskId)} " +
+ log"is still running after ${MDC(TIME_UNITS, elapsedTimeMs)} ms")
if (takeThreadDump) {
try {
taskRunner.theadDump().foreach { thread =>
if (thread.threadName == taskRunner.threadName) {
- logWarning(log"Thread dump from task ${LogMDC(TASK_ID, taskId)}:\n" +
- log"${LogMDC(THREAD, thread.toString)}")
+ logWarning(log"Thread dump from task ${MDC(TASK_ID, taskId)}:\n" +
+ log"${MDC(THREAD, thread.toString)}")
}
}
} catch {
@@ -1037,8 +1037,8 @@ private[spark] class Executor(
if (!taskRunner.isFinished && timeoutExceeded()) {
val killTimeoutMs = TimeUnit.NANOSECONDS.toMillis(killTimeoutNs)
if (isLocal) {
- logError(log"Killed task ${LogMDC(TASK_ID, taskId)} could not be stopped within " +
- log"${LogMDC(TIMEOUT, killTimeoutMs)} ms; " +
+ logError(log"Killed task ${MDC(TASK_ID, taskId)} could not be stopped within " +
+ log"${MDC(TIMEOUT, killTimeoutMs)} ms; " +
log"not killing JVM because we are running in local mode.")
} else {
// In non-local-mode, the exception thrown here will bubble up to the uncaught exception
@@ -1093,8 +1093,8 @@ private[spark] class Executor(
logInfo(
log"Starting executor with user classpath" +
log" (userClassPathFirst =" +
- log" ${LogMDC(LogKeys.EXECUTOR_USER_CLASS_PATH_FIRST, userClassPathFirst)}): " +
- log"${LogMDC(URLS, urls.mkString("'", ",", "'"))}"
+ log" ${MDC(LogKeys.EXECUTOR_USER_CLASS_PATH_FIRST, userClassPathFirst)}): " +
+ log"${MDC(URLS, urls.mkString("'", ",", "'"))}"
)
if (useStub) {
@@ -1150,13 +1150,13 @@ private[spark] class Executor(
sessionUUID: String): ClassLoader = {
val classUri = sessionClassUri.getOrElse(conf.get("spark.repl.class.uri", null))
val classLoader = if (classUri != null) {
- logInfo(log"Using REPL class URI: ${LogMDC(LogKeys.URI, classUri)}")
+ logInfo(log"Using REPL class URI: ${MDC(LogKeys.URI, classUri)}")
new ExecutorClassLoader(conf, env, classUri, parent, userClassPathFirst)
} else {
parent
}
- logInfo(log"Created or updated repl class loader ${LogMDC(CLASS_LOADER, classLoader)}" +
- log" for ${LogMDC(SESSION_ID, sessionUUID)}.")
+ logInfo(log"Created or updated repl class loader ${MDC(CLASS_LOADER, classLoader)}" +
+ log" for ${MDC(SESSION_ID, sessionUUID)}.")
classLoader
}
@@ -1191,16 +1191,16 @@ private[spark] class Executor(
// Fetch missing dependencies
for ((name, timestamp) <- newFiles if state.currentFiles.getOrElse(name, -1L) < timestamp) {
- logInfo(log"Fetching ${LogMDC(FILE_NAME, name)} with" +
- log" timestamp ${LogMDC(TIMESTAMP, timestamp)}")
+ logInfo(log"Fetching ${MDC(FILE_NAME, name)} with" +
+ log" timestamp ${MDC(TIMESTAMP, timestamp)}")
// Fetch file with useCache mode, close cache for local mode.
Utils.fetchFile(name, root, conf, hadoopConf, timestamp, useCache = !isLocal)
state.currentFiles(name) = timestamp
}
for ((name, timestamp) <- newArchives if
state.currentArchives.getOrElse(name, -1L) < timestamp) {
- logInfo(log"Fetching ${LogMDC(ARCHIVE_NAME, name)} with" +
- log" timestamp ${LogMDC(TIMESTAMP, timestamp)}")
+ logInfo(log"Fetching ${MDC(ARCHIVE_NAME, name)} with" +
+ log" timestamp ${MDC(TIMESTAMP, timestamp)}")
val sourceURI = new URI(name)
val uriToDownload = Utils.getUriBuilder(sourceURI).fragment(null).build()
val source = Utils.fetchFile(uriToDownload.toString, Utils.createTempDir(), conf,
@@ -1209,10 +1209,10 @@ private[spark] class Executor(
root,
if (sourceURI.getFragment != null) sourceURI.getFragment else source.getName)
logInfo(
- log"Unpacking an archive ${LogMDC(ARCHIVE_NAME, name)}" +
- log" (${LogMDC(BYTE_SIZE, source.length)} bytes)" +
- log" from ${LogMDC(SOURCE_PATH, source.getAbsolutePath)}" +
- log" to ${LogMDC(DESTINATION_PATH, dest.getAbsolutePath)}")
+ log"Unpacking an archive ${MDC(ARCHIVE_NAME, name)}" +
+ log" (${MDC(BYTE_SIZE, source.length)} bytes)" +
+ log" from ${MDC(SOURCE_PATH, source.getAbsolutePath)}" +
+ log" to ${MDC(DESTINATION_PATH, dest.getAbsolutePath)}")
Utils.deleteRecursively(dest)
Utils.unpack(source, dest)
state.currentArchives(name) = timestamp
@@ -1223,8 +1223,8 @@ private[spark] class Executor(
.orElse(state.currentJars.get(localName))
.getOrElse(-1L)
if (currentTimeStamp < timestamp) {
- logInfo(log"Fetching ${LogMDC(JAR_URL, name)} with" +
- log" timestamp ${LogMDC(TIMESTAMP, timestamp)}")
+ logInfo(log"Fetching ${MDC(JAR_URL, name)} with" +
+ log" timestamp ${MDC(TIMESTAMP, timestamp)}")
// Fetch file with useCache mode, close cache for local mode.
Utils.fetchFile(name, root, conf,
hadoopConf, timestamp, useCache = !isLocal)
@@ -1232,8 +1232,8 @@ private[spark] class Executor(
// Add it to our class loader
val url = new File(root, localName).toURI.toURL
if (!state.urlClassLoader.getURLs().contains(url)) {
- logInfo(log"Adding ${LogMDC(LogKeys.URL, url)} to" +
- log" class loader ${LogMDC(UUID, state.sessionUUID)}")
+ logInfo(log"Adding ${MDC(LogKeys.URL, url)} to" +
+ log" class loader ${MDC(UUID, state.sessionUUID)}")
state.urlClassLoader.addURL(url)
if (isStubbingEnabledForState(state.sessionUUID)) {
renewClassLoader = true
@@ -1298,7 +1298,7 @@ private[spark] class Executor(
heartbeatFailures += 1
if (heartbeatFailures >= HEARTBEAT_MAX_FAILURES) {
logError(log"Exit as unable to send heartbeats to driver " +
- log"more than ${LogMDC(MAX_ATTEMPTS, HEARTBEAT_MAX_FAILURES)} times")
+ log"more than ${MDC(MAX_ATTEMPTS, HEARTBEAT_MAX_FAILURES)} times")
System.exit(ExecutorExitCode.HEARTBEAT_FAILURE)
}
}
@@ -1309,7 +1309,7 @@ private[spark] class Executor(
if (runner != null) {
runner.theadDump()
} else {
- logWarning(log"Failed to dump thread for task ${LogMDC(TASK_ID, taskId)}")
+ logWarning(log"Failed to dump thread for task ${MDC(TASK_ID, taskId)}")
None
}
}
@@ -1328,8 +1328,8 @@ private[spark] object Executor extends Logging {
try {
// This tests if any class initialization error is thrown
val testKey = System.nanoTime().toString
- MDC.put(testKey, "testValue")
- MDC.remove(testKey)
+ SLF4JMDC.put(testKey, "testValue")
+ SLF4JMDC.remove(testKey)
true
} catch {
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorClassLoader.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorClassLoader.scala
index 6f8138da6f4fb..faea70933071a 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorClassLoader.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorClassLoader.scala
@@ -30,7 +30,7 @@ import org.apache.xbean.asm9.Opcodes._
import org.apache.spark.{SparkConf, SparkEnv}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.util.ParentClassLoader
/**
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorLogUrlHandler.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorLogUrlHandler.scala
index 2202489509fc4..9807cb2c81e45 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorLogUrlHandler.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorLogUrlHandler.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicBoolean
import scala.util.matching.Regex
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
private[spark] class ExecutorLogUrlHandler(logUrlPattern: Option[String]) extends Logging {
diff --git a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
index b9a462d62e413..00334e9cc55f1 100644
--- a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
@@ -96,7 +96,14 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String = "/proc/") extends L
}
Utils.tryWithResource(openReader()) { in =>
val procInfo = in.readLine
- val procInfoSplit = procInfo.split(" ")
+ // The comm field, which is inside parentheses, could contain spaces. We should not split
+ // by those spaces as doing so could cause the numbers after it to be shifted.
+ val commStartIndex = procInfo.indexOf('(')
+ val commEndIndex = procInfo.lastIndexOf(')') + 1
+ val pidArray = Array(procInfo.substring(0, commStartIndex).trim)
+ val commArray = Array(procInfo.substring(commStartIndex, commEndIndex))
+ val splitAfterComm = procInfo.substring(commEndIndex).trim.split(" ")
+ val procInfoSplit = pidArray ++ commArray ++ splitAfterComm
val vmem = procInfoSplit(22).toLong
val rssMem = procInfoSplit(23).toLong * pageSize
if (procInfoSplit(1).toLowerCase(Locale.US).contains("java")) {
diff --git a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
index 3c3017a9a64c1..9211bfec5cbd5 100644
--- a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
+++ b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
@@ -21,7 +21,7 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, Da
import scala.jdk.CollectionConverters._
-import com.google.common.io.{ByteStreams, Closeables}
+import com.google.common.io.Closeables
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapreduce.{InputSplit, JobContext, RecordReader, TaskAttemptContext}
@@ -202,7 +202,7 @@ class PortableDataStream(
def toArray(): Array[Byte] = {
val stream = open()
try {
- ByteStreams.toByteArray(stream)
+ stream.readAllBytes()
} finally {
Closeables.close(stream, true)
}
diff --git a/core/src/main/scala/org/apache/spark/input/WholeTextFileRecordReader.scala b/core/src/main/scala/org/apache/spark/input/WholeTextFileRecordReader.scala
index 0bd2d551cc912..ba975237cb934 100644
--- a/core/src/main/scala/org/apache/spark/input/WholeTextFileRecordReader.scala
+++ b/core/src/main/scala/org/apache/spark/input/WholeTextFileRecordReader.scala
@@ -17,15 +17,16 @@
package org.apache.spark.input
-import com.google.common.io.{ByteStreams, Closeables}
+import com.google.common.io.Closeables
import org.apache.hadoop.conf.{Configurable => HConfigurable, Configuration}
import org.apache.hadoop.io.Text
-import org.apache.hadoop.io.compress.CompressionCodecFactory
import org.apache.hadoop.mapreduce.InputSplit
import org.apache.hadoop.mapreduce.RecordReader
import org.apache.hadoop.mapreduce.TaskAttemptContext
import org.apache.hadoop.mapreduce.lib.input.{CombineFileRecordReader, CombineFileSplit}
+import org.apache.spark.io.HadoopCodecStreams
+
/**
* A trait to implement [[org.apache.hadoop.conf.Configurable Configurable]] interface.
*/
@@ -69,15 +70,8 @@ private[spark] class WholeTextFileRecordReader(
override def nextKeyValue(): Boolean = {
if (!processed) {
- val conf = getConf
- val factory = new CompressionCodecFactory(conf)
- val codec = factory.getCodec(path) // infers from file ext.
- val fileIn = fs.open(path)
- val innerBuffer = if (codec != null) {
- ByteStreams.toByteArray(codec.createInputStream(fileIn))
- } else {
- ByteStreams.toByteArray(fileIn)
- }
+ val fileIn = HadoopCodecStreams.createInputStream(getConf, path)
+ val innerBuffer = fileIn.readAllBytes()
value = new Text(innerBuffer)
Closeables.close(fileIn, false)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index bbd4afcaebab4..d44d3c32dfef3 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -159,6 +159,13 @@ private[spark] object History {
.doubleConf
.createWithDefault(0.7d)
+ val EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED =
+ ConfigBuilder("spark.history.fs.eventLog.rolling.onDemandLoadEnabled")
+ .doc("Whether to look up rolling event log locations on demand manner before listing files.")
+ .version("4.1.0")
+ .booleanConf
+ .createWithDefault(true)
+
val DRIVER_LOG_CLEANER_ENABLED = ConfigBuilder("spark.history.fs.driverlog.cleaner.enabled")
.version("3.0.0")
.doc("Specifies whether the History Server should periodically clean up driver logs from " +
@@ -221,12 +228,19 @@ private[spark] object History {
.intConf
.createWithDefaultFunction(() => Math.ceil(Runtime.getRuntime.availableProcessors() / 4f).toInt)
+ val NUM_COMPACT_THREADS = ConfigBuilder("spark.history.fs.numCompactThreads")
+ .version("4.1.0")
+ .doc("Number of threads that will be used by history server to compact event logs.")
+ .intConf
+ .createWithDefaultFunction(() => Math.ceil(Runtime.getRuntime.availableProcessors() / 4f).toInt)
+
val RETAINED_APPLICATIONS = ConfigBuilder("spark.history.retainedApplications")
.version("1.0.0")
.doc("The number of applications to retain UI data for in the cache. If this cap is " +
"exceeded, then the oldest applications will be removed from the cache. If an application " +
"is not in the cache, it will have to be loaded from disk if it is accessed from the UI.")
.intConf
+ .checkValue(v => v > 0, "The number of applications to retain should be a positive integer.")
.createWithDefault(50)
val PROVIDER = ConfigBuilder("spark.history.provider")
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Python.scala b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
index 46d54be92f3d6..de95e2fa1f7a2 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Python.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
@@ -117,4 +117,25 @@ private[spark] object Python {
.version("4.1.0")
.booleanConf
.createWithDefault(false)
+
+ val PYTHON_WORKER_TRACEBACK_DUMP_INTERVAL_SECONDS =
+ ConfigBuilder("spark.python.worker.tracebackDumpIntervalSeconds")
+ .doc("The interval (in seconds) for Python workers to dump their tracebacks. " +
+ "If it's positive, the Python worker will periodically dump the traceback into " +
+ "its `stderr`. The default is `0` that means it is disabled.")
+ .version("4.1.0")
+ .timeConf(TimeUnit.SECONDS)
+ .checkValue(_ >= 0, "The interval should be 0 or positive.")
+ .createWithDefault(0)
+
+ val PYTHON_FACTORY_IDLE_WORKER_MAX_POOL_SIZE =
+ ConfigBuilder("spark.python.factory.idleWorkerMaxPoolSize")
+ .doc("Maximum number of idle Python workers to keep. " +
+ "If unset, the number is unbounded. " +
+ "If set to a positive integer N, at most N idle workers are retained; " +
+ "least-recently used workers are evicted first.")
+ .version("4.1.0")
+ .intConf
+ .checkValue(_ > 0, "If set, the idle worker max size must be > 0.")
+ .createOptional
}
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 7cb3d068b676f..9876848f654a4 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -224,6 +224,14 @@ package object config {
.booleanConf
.createWithDefault(false)
+ private[spark] val EVENT_LOG_EXCLUDED_PATTERNS =
+ ConfigBuilder("spark.eventLog.excludedPatterns")
+ .doc("Specifies comma-separated event names to be excluded from the event logs.")
+ .version("4.1.0")
+ .stringConf
+ .toSequence
+ .createWithDefault(Nil)
+
private[spark] val EVENT_LOG_ALLOW_EC =
ConfigBuilder("spark.eventLog.erasureCoding.enabled")
.version("3.0.0")
@@ -283,14 +291,6 @@ package object config {
.booleanConf
.createWithDefault(true)
- private[spark] val EVENT_LOG_READER_MAX_STRING_LENGTH =
- ConfigBuilder("spark.eventLog.readerMaxStringLength")
- .doc("Limit the maximum string size an eventlog item can have when deserializing it.")
- .version("4.1.0")
- .intConf
- .checkValue(_ > 0, "Maximum string size of an eventLog item should be positive.")
- .createWithDefault(Int.MaxValue)
-
private[spark] val EVENT_LOG_OVERWRITE =
ConfigBuilder("spark.eventLog.overwrite")
.version("1.0.0")
@@ -317,8 +317,8 @@ package object config {
" to be rolled over.")
.version("3.0.0")
.bytesConf(ByteUnit.BYTE)
- .checkValue(_ >= ByteUnit.MiB.toBytes(10), "Max file size of event log should be " +
- "configured to be at least 10 MiB.")
+ .checkValue(_ >= ByteUnit.MiB.toBytes(2), "Max file size of event log should be " +
+ "configured to be at least 2 MiB.")
.createWithDefaultString("128m")
private[spark] val EXECUTOR_ID =
@@ -492,6 +492,16 @@ package object config {
.doubleConf
.createWithDefault(0.6)
+ private[spark] val UNMANAGED_MEMORY_POLLING_INTERVAL =
+ ConfigBuilder("spark.memory.unmanagedMemoryPollingInterval")
+ .doc("Interval for polling unmanaged memory users to track their memory usage. " +
+ "Unmanaged memory users are components that manage their own memory outside of " +
+ "Spark's core memory management, such as RocksDB for Streaming State Store. " +
+ "Setting this to 0 disables unmanaged memory polling.")
+ .version("4.1.0")
+ .timeConf(TimeUnit.MILLISECONDS)
+ .createWithDefaultString("0s")
+
private[spark] val STORAGE_UNROLL_MEMORY_THRESHOLD =
ConfigBuilder("spark.storage.unrollMemoryThreshold")
.doc("Initial memory to request before unrolling any block")
@@ -594,23 +604,27 @@ package object config {
"cache block replication should be positive.")
.createWithDefaultString("30s")
+ private[spark] val STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP =
+ ConfigBuilder("spark.storage.decommission.fallbackStorage.cleanUp")
+ .doc("If true, Spark cleans up its fallback storage data once individual shuffles are " +
+ "freed (interval configured via spark.cleaner.periodicGC.interval), and during " +
+ "shutting down.")
+ .version("3.2.0")
+ .booleanConf
+ .createWithDefault(false)
+
private[spark] val STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH =
ConfigBuilder("spark.storage.decommission.fallbackStorage.path")
.doc("The location for fallback storage during block manager decommissioning. " +
"For example, `s3a://spark-storage/`. In case of empty, fallback storage is disabled. " +
- "The storage should be managed by TTL because Spark will not clean it up.")
+ "The storage will not be cleaned up by Spark unless " +
+ s"${STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP.key} is true. " +
+ "Use an external clean up mechanism when false, for instance a TTL.")
.version("3.1.0")
.stringConf
.checkValue(_.endsWith(java.io.File.separator), "Path should end with separator.")
.createOptional
- private[spark] val STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP =
- ConfigBuilder("spark.storage.decommission.fallbackStorage.cleanUp")
- .doc("If true, Spark cleans up its fallback storage data during shutting down.")
- .version("3.2.0")
- .booleanConf
- .createWithDefault(false)
-
private[spark] val STORAGE_DECOMMISSION_SHUFFLE_MAX_DISK_SIZE =
ConfigBuilder("spark.storage.decommission.shuffleBlocks.maxDiskSize")
.doc("Maximum disk space to use to store shuffle blocks before rejecting remote " +
@@ -932,7 +946,6 @@ package object config {
private[spark] val EXCLUDE_ON_FAILURE_ENABLED =
ConfigBuilder("spark.excludeOnFailure.enabled")
.version("3.1.0")
- .withAlternative("spark.blacklist.enabled")
.booleanConf
.createOptional
@@ -951,56 +964,48 @@ package object config {
private[spark] val MAX_TASK_ATTEMPTS_PER_EXECUTOR =
ConfigBuilder("spark.excludeOnFailure.task.maxTaskAttemptsPerExecutor")
.version("3.1.0")
- .withAlternative("spark.blacklist.task.maxTaskAttemptsPerExecutor")
.intConf
.createWithDefault(1)
private[spark] val MAX_TASK_ATTEMPTS_PER_NODE =
ConfigBuilder("spark.excludeOnFailure.task.maxTaskAttemptsPerNode")
.version("3.1.0")
- .withAlternative("spark.blacklist.task.maxTaskAttemptsPerNode")
.intConf
.createWithDefault(2)
private[spark] val MAX_FAILURES_PER_EXEC =
ConfigBuilder("spark.excludeOnFailure.application.maxFailedTasksPerExecutor")
.version("3.1.0")
- .withAlternative("spark.blacklist.application.maxFailedTasksPerExecutor")
.intConf
.createWithDefault(2)
private[spark] val MAX_FAILURES_PER_EXEC_STAGE =
ConfigBuilder("spark.excludeOnFailure.stage.maxFailedTasksPerExecutor")
.version("3.1.0")
- .withAlternative("spark.blacklist.stage.maxFailedTasksPerExecutor")
.intConf
.createWithDefault(2)
private[spark] val MAX_FAILED_EXEC_PER_NODE =
ConfigBuilder("spark.excludeOnFailure.application.maxFailedExecutorsPerNode")
.version("3.1.0")
- .withAlternative("spark.blacklist.application.maxFailedExecutorsPerNode")
.intConf
.createWithDefault(2)
private[spark] val MAX_FAILED_EXEC_PER_NODE_STAGE =
ConfigBuilder("spark.excludeOnFailure.stage.maxFailedExecutorsPerNode")
.version("3.1.0")
- .withAlternative("spark.blacklist.stage.maxFailedExecutorsPerNode")
.intConf
.createWithDefault(2)
private[spark] val EXCLUDE_ON_FAILURE_TIMEOUT_CONF =
ConfigBuilder("spark.excludeOnFailure.timeout")
.version("3.1.0")
- .withAlternative("spark.blacklist.timeout")
.timeConf(TimeUnit.MILLISECONDS)
.createOptional
private[spark] val EXCLUDE_ON_FAILURE_KILL_ENABLED =
ConfigBuilder("spark.excludeOnFailure.killExcludedExecutors")
.version("3.1.0")
- .withAlternative("spark.blacklist.killBlacklistedExecutors")
.booleanConf
.createWithDefault(false)
@@ -1015,14 +1020,12 @@ package object config {
ConfigBuilder("spark.scheduler.executorTaskExcludeOnFailureTime")
.internal()
.version("3.1.0")
- .withAlternative("spark.scheduler.executorTaskBlacklistTime")
.timeConf(TimeUnit.MILLISECONDS)
.createOptional
private[spark] val EXCLUDE_ON_FAILURE_FETCH_FAILURE_ENABLED =
ConfigBuilder("spark.excludeOnFailure.application.fetchFailure.enabled")
.version("3.1.0")
- .withAlternative("spark.blacklist.application.fetchFailure.enabled")
.booleanConf
.createWithDefault(false)
@@ -1191,6 +1194,14 @@ package object config {
.checkValue(v => v >= 0, "The value should be a non-negative time value.")
.createWithDefaultString("0min")
+ private[spark] val DRIVER_METRICS_POLLING_INTERVAL =
+ ConfigBuilder("spark.driver.metrics.pollingInterval")
+ .doc("How often to collect driver metrics (in milliseconds). " +
+ "If unset, the polling is done at the executor heartbeat interval. " +
+ "If set, the polling is done at this interval.")
+ .version("4.1.0")
+ .fallbackConf(EXECUTOR_HEARTBEAT_INTERVAL)
+
private[spark] val DRIVER_BIND_ADDRESS = ConfigBuilder("spark.driver.bindAddress")
.doc("Address where to bind network listen sockets on the driver.")
.version("2.1.0")
@@ -1588,6 +1599,18 @@ package object config {
.intConf
.createWithDefault(Integer.MAX_VALUE)
+ private[spark] val SHUFFLE_SPILL_MAX_SIZE_FORCE_SPILL_THRESHOLD =
+ ConfigBuilder("spark.shuffle.spill.maxSizeInBytesForSpillThreshold")
+ .internal()
+ .doc("The maximum in memory size in bytes before forcing the shuffle sorter to spill. " +
+ "By default it is Long.MAX_VALUE, which means we never force the sorter to spill, " +
+ "until we reach some limitations, like the max page size limitation for the pointer " +
+ "array in the sorter.")
+ .version("4.1.0")
+ .bytesConf(ByteUnit.BYTE)
+ .checkValue(v => v > 0, "The threshold should be positive.")
+ .createWithDefault(Long.MaxValue)
+
private[spark] val SHUFFLE_MAP_OUTPUT_PARALLEL_AGGREGATION_THRESHOLD =
ConfigBuilder("spark.shuffle.mapOutput.parallelAggregationThreshold")
.internal()
@@ -1836,7 +1859,6 @@ package object config {
"before aborting a TaskSet which is unschedulable because all executors are " +
"excluded due to failures.")
.version("3.1.0")
- .withAlternative("spark.scheduler.blacklist.unschedulableTaskSetTimeout")
.timeConf(TimeUnit.SECONDS)
.checkValue(v => v >= 0, "The value should be a non negative time value.")
.createWithDefault(120)
@@ -2105,12 +2127,21 @@ package object config {
.intConf
.createWithDefault(1)
+ private[spark] val IO_COMPRESSION_ZSTD_STRATEGY =
+ ConfigBuilder("spark.io.compression.zstd.strategy")
+ .doc("Compression strategy for Zstd compression codec. The higher the value is, the more " +
+ "complex it becomes, usually resulting stronger but slower compression or higher CPU " +
+ "cost.")
+ .version("4.1.0")
+ .intConf
+ .createOptional
+
private[spark] val IO_COMPRESSION_LZF_PARALLEL =
ConfigBuilder("spark.io.compression.lzf.parallel.enabled")
.doc("When true, LZF compression will use multiple threads to compress data in parallel.")
.version("4.0.0")
.booleanConf
- .createWithDefault(false)
+ .createWithDefault(true)
private[spark] val IO_WARNING_LARGEFILETHRESHOLD =
ConfigBuilder("spark.io.warning.largeFileThreshold")
@@ -2280,6 +2311,15 @@ package object config {
.toSequence
.createWithDefault(Nil)
+ private[spark] val SUBMIT_CALL_SYSTEM_EXIT_ON_MAIN_EXIT =
+ ConfigBuilder("spark.submit.callSystemExitOnMainExit")
+ .doc("If true, SparkSubmit will call System.exit() to initiate JVM shutdown once the " +
+ "user's main method has exited. This can be useful in cases where non-daemon JVM " +
+ "threads might otherwise prevent the JVM from shutting down on its own.")
+ .version("4.1.0")
+ .booleanConf
+ .createWithDefault(false)
+
private[spark] val SCHEDULER_ALLOCATION_FILE =
ConfigBuilder("spark.scheduler.allocation.file")
.version("0.8.1")
@@ -2838,4 +2878,30 @@ package object config {
.checkValues(Set("connect", "classic"))
.createWithDefault(
if (sys.env.get("SPARK_CONNECT_MODE").contains("1")) "connect" else "classic")
+
+ private[spark] val DRIVER_REDIRECT_CONSOLE_OUTPUTS =
+ ConfigBuilder("spark.driver.log.redirectConsoleOutputs")
+ .doc("Comma-separated list of the console output kind for driver that needs to redirect " +
+ "to logging system. Supported values are `stdout`, `stderr`. It only takes affect when " +
+ s"`${PLUGINS.key}` is configured with `org.apache.spark.deploy.RedirectConsolePlugin`.")
+ .version("4.1.0")
+ .stringConf
+ .transform(_.toLowerCase(Locale.ROOT))
+ .toSequence
+ .checkValue(v => v.forall(Set("stdout", "stderr").contains),
+ "The value only can be one or more of 'stdout, stderr'.")
+ .createWithDefault(Seq("stdout", "stderr"))
+
+ private[spark] val EXEC_REDIRECT_CONSOLE_OUTPUTS =
+ ConfigBuilder("spark.executor.logs.redirectConsoleOutputs")
+ .doc("Comma-separated list of the console output kind for executor that needs to redirect " +
+ "to logging system. Supported values are `stdout`, `stderr`. It only takes affect when " +
+ s"`${PLUGINS.key}` is configured with `org.apache.spark.deploy.RedirectConsolePlugin`.")
+ .version("4.1.0")
+ .stringConf
+ .transform(_.toLowerCase(Locale.ROOT))
+ .toSequence
+ .checkValue(v => v.forall(Set("stdout", "stderr").contains),
+ "The value only can be one or more of 'stdout, stderr'.")
+ .createWithDefault(Seq("stdout", "stderr"))
}
diff --git a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
index e2a96267082b8..651895bf1f7a2 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
@@ -20,6 +20,7 @@ package org.apache.spark.internal.io
import org.apache.hadoop.fs._
import org.apache.hadoop.mapreduce._
+import org.apache.spark.SparkException
import org.apache.spark.annotation.Unstable
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
@@ -96,7 +97,9 @@ abstract class FileCommitProtocol extends Logging {
* guarantees that files written by different tasks will not conflict.
*/
@deprecated("use newTaskTempFile(..., spec: FileNameSpec) instead", "3.3.0")
- def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], ext: String): String
+ def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], ext: String): String = {
+ throw SparkException.mustOverrideOneMethodError("newTaskTempFile")
+ }
/**
* Notifies the commit protocol to add a new file, and gets back the full path that should be
@@ -135,7 +138,9 @@ abstract class FileCommitProtocol extends Logging {
*/
@deprecated("use newTaskTempFileAbsPath(..., spec: FileNameSpec) instead", "3.3.0")
def newTaskTempFileAbsPath(
- taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String
+ taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String = {
+ throw SparkException.mustOverrideOneMethodError("newTaskTempFileAbsPath")
+ }
/**
* Similar to newTaskTempFile(), but allows files to committed to an absolute output location.
diff --git a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapRedCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapRedCommitProtocol.scala
index 44f8d7cd63635..358424abefba5 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapRedCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapRedCommitProtocol.scala
@@ -21,7 +21,6 @@ import org.apache.hadoop.mapred._
import org.apache.hadoop.mapreduce.{TaskAttemptContext => NewTaskAttemptContext}
import org.apache.spark.internal.LogKeys
-import org.apache.spark.internal.MDC
/**
* An [[FileCommitProtocol]] implementation backed by an underlying Hadoop OutputCommitter
diff --git a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
index 476cddc643954..d7bda5bbe721a 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
@@ -29,7 +29,7 @@ import org.apache.hadoop.mapreduce._
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.mapred.SparkHadoopMapRedUtil
@@ -117,11 +117,6 @@ class HadoopMapReduceCommitProtocol(
format.getOutputCommitter(context)
}
- override def newTaskTempFile(
- taskContext: TaskAttemptContext, dir: Option[String], ext: String): String = {
- newTaskTempFile(taskContext, dir, FileNameSpec("", ext))
- }
-
override def newTaskTempFile(
taskContext: TaskAttemptContext, dir: Option[String], spec: FileNameSpec): String = {
val filename = getFilename(taskContext, spec)
@@ -145,11 +140,6 @@ class HadoopMapReduceCommitProtocol(
}
}
- override def newTaskTempFileAbsPath(
- taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String = {
- newTaskTempFileAbsPath(taskContext, absoluteDir, FileNameSpec("", ext))
- }
-
override def newTaskTempFileAbsPath(
taskContext: TaskAttemptContext, absoluteDir: String, spec: FileNameSpec): String = {
val filename = getFilename(taskContext, spec)
diff --git a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
index db961b3c42f4c..de136a86a2cc3 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
@@ -32,7 +32,7 @@ import org.apache.hadoop.mapreduce.task.{TaskAttemptContextImpl => NewTaskAttemp
import org.apache.spark.{SerializableWritable, SparkConf, SparkException, TaskContext}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{DURATION, JOB_ID, TASK_ATTEMPT_ID}
import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage
import org.apache.spark.rdd.{HadoopRDD, RDD}
diff --git a/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala b/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala
index a0c07bd75f885..51b29e1a0ec68 100644
--- a/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala
+++ b/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala
@@ -22,7 +22,7 @@ import scala.util.{Either, Left, Right}
import org.apache.spark.{SparkContext, SparkEnv, TaskFailedReason}
import org.apache.spark.api.plugin._
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config._
import org.apache.spark.resource.ResourceInformation
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/internal/plugin/PluginEndpoint.scala b/core/src/main/scala/org/apache/spark/internal/plugin/PluginEndpoint.scala
index 6ff918979c9ed..2c2f2f022a7a3 100644
--- a/core/src/main/scala/org/apache/spark/internal/plugin/PluginEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/internal/plugin/PluginEndpoint.scala
@@ -18,7 +18,7 @@
package org.apache.spark.internal.plugin
import org.apache.spark.api.plugin.DriverPlugin
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rpc.{IsolatedThreadSafeRpcEndpoint, RpcCallContext, RpcEnv}
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
index 233228a9c6d4c..b81e46667323e 100644
--- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
+++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
@@ -228,6 +228,7 @@ class ZStdCompressionCodec(conf: SparkConf) extends CompressionCodec {
// Default compression level for zstd compression to 1 because it is
// fastest of all with reasonably high compression ratio.
private val level = conf.get(IO_COMPRESSION_ZSTD_LEVEL)
+ private val strategy = conf.get(IO_COMPRESSION_ZSTD_STRATEGY)
private val bufferPool = if (conf.get(IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED)) {
RecyclingBufferPool.INSTANCE
@@ -241,6 +242,7 @@ class ZStdCompressionCodec(conf: SparkConf) extends CompressionCodec {
// Wrap the zstd output stream in a buffered output stream, so that we can
// avoid overhead excessive of JNI call while trying to compress small amount of data.
val os = new ZstdOutputStreamNoFinalizer(s, bufferPool).setLevel(level).setWorkers(workers)
+ strategy.foreach(os.setStrategy)
new BufferedOutputStream(os, bufferSize)
}
diff --git a/core/src/main/scala/org/apache/spark/io/HadoopCodecStreams.scala b/core/src/main/scala/org/apache/spark/io/HadoopCodecStreams.scala
new file mode 100644
index 0000000000000..cba0e7ffd73bf
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/io/HadoopCodecStreams.scala
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.io
+
+import java.io.InputStream
+import java.util.Locale
+
+import scala.collection.Seq
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.io.compress._
+
+import org.apache.spark.{SparkConf, SparkEnv}
+import org.apache.spark.io.{CompressionCodec => SparkCompressionCodec}
+
+/**
+ * An utility object to look up Hadoop compression codecs and create input streams.
+ * In addition to standard Hadoop codecs, it also supports Spark's Zstandard codec
+ * if Hadopp is not compiled with Zstandard support. Additionally, it supports
+ * non-standard file extensions like `.zstd` and `.gzip` for Zstandard and Gzip codecs.
+ */
+object HadoopCodecStreams {
+ private val ZSTD_EXTENSIONS = Seq(".zstd", ".zst")
+
+ // get codec based on file name extension
+ def getDecompressionCodec(
+ config: Configuration,
+ file: Path): Option[CompressionCodec] = {
+ val factory = new CompressionCodecFactory(config)
+ Option(factory.getCodec(file)).orElse {
+ // Try some non-standards extensions for Zstandard and Gzip
+ file.getName.toLowerCase() match {
+ case name if name.endsWith(".zstd") =>
+ Option(factory.getCodecByName(classOf[ZStandardCodec].getName))
+ case name if name.endsWith(".gzip") =>
+ Option(factory.getCodecByName(classOf[GzipCodec].getName))
+ case _ => None
+ }
+ }
+ }
+
+ def createZstdInputStream(
+ file: Path,
+ inputStream: InputStream): Option[InputStream] = {
+ val sparkConf = Option(SparkEnv.get).map(_.conf).getOrElse(new SparkConf)
+ val fileName = file.getName.toLowerCase(Locale.ROOT)
+
+ val isOpt = if (ZSTD_EXTENSIONS.exists(fileName.endsWith)) {
+ Some(
+ SparkCompressionCodec
+ .createCodec(sparkConf, SparkCompressionCodec.ZSTD)
+ .compressedInputStream(inputStream)
+ )
+ } else {
+ None
+ }
+ isOpt
+ }
+
+ def createInputStream(
+ config: Configuration,
+ file: Path): InputStream = {
+ val fs = file.getFileSystem(config)
+ val inputStream: InputStream = fs.open(file)
+
+ getDecompressionCodec(config, file)
+ .map { codec =>
+ try {
+ codec.createInputStream(inputStream)
+ } catch {
+ case e: RuntimeException =>
+ // createInputStream may fail for ZSTD if hadoop is not already compiled with ZSTD
+ // support. In that case, we try to use Spark's Zstandard codec.
+ createZstdInputStream(file, inputStream).getOrElse(throw e)
+ }
+ }.getOrElse(inputStream)
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala b/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala
index 0aaa222e6195e..e9745a7c2ba67 100644
--- a/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala
+++ b/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala
@@ -24,7 +24,7 @@ import org.apache.hadoop.mapreduce.{OutputCommitter => MapReduceOutputCommitter}
import org.apache.spark.{SparkEnv, TaskContext}
import org.apache.spark.executor.CommitDeniedException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{TASK_ATTEMPT_ID, TOTAL_TIME}
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala b/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala
index 7098961d1649a..8b31d02cf97e3 100644
--- a/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala
+++ b/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala
@@ -21,7 +21,7 @@ import javax.annotation.concurrent.GuardedBy
import scala.collection.mutable
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
/**
diff --git a/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala b/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala
index 24fcb5b17f388..7cde44b553321 100644
--- a/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala
+++ b/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala
@@ -20,7 +20,7 @@ package org.apache.spark.memory
import javax.annotation.concurrent.GuardedBy
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.storage.BlockId
import org.apache.spark.storage.memory.MemoryStore
diff --git a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala
index d4ec6ed8495af..db51f14415e1a 100644
--- a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala
+++ b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala
@@ -17,11 +17,19 @@
package org.apache.spark.memory
+import java.util.concurrent.{ConcurrentHashMap, ScheduledExecutorService, TimeUnit}
+import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong}
+
+import scala.jdk.CollectionConverters._
+import scala.util.control.NonFatal
+
import org.apache.spark.{SparkConf, SparkIllegalArgumentException}
-import org.apache.spark.internal.{config, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.Tests._
+import org.apache.spark.internal.config.UNMANAGED_MEMORY_POLLING_INTERVAL
import org.apache.spark.storage.BlockId
+import org.apache.spark.util.{ThreadUtils, Utils}
/**
* A [[MemoryManager]] that enforces a soft boundary between execution and storage such that
@@ -56,7 +64,47 @@ private[spark] class UnifiedMemoryManager(
conf,
numCores,
onHeapStorageRegionSize,
- maxHeapMemory - onHeapStorageRegionSize) {
+ maxHeapMemory - onHeapStorageRegionSize) with Logging {
+
+ /**
+ * Unmanaged memory tracking infrastructure.
+ *
+ * Unmanaged memory refers to memory consumed by components that manage their own memory
+ * outside of Spark's unified memory management system. Examples include:
+ * - RocksDB state stores used in structured streaming
+ * - Native libraries with their own memory management
+ * - Off-heap caches managed by unmanaged systems
+ *
+ * We track this memory to:
+ * 1. Provide visibility into total memory usage on executors
+ * 2. Prevent OOM errors by accounting for it in memory allocation decisions
+ * 3. Enable better debugging and monitoring of memory-intensive applications
+ *
+ * The polling mechanism periodically queries registered unmanaged memory consumers
+ * to detect inactive consumers and handle cleanup.
+ */
+ // Configuration for polling interval (in milliseconds)
+ private val unmanagedMemoryPollingIntervalMs = conf.get(UNMANAGED_MEMORY_POLLING_INTERVAL)
+ // Initialize background polling if enabled
+ if (unmanagedMemoryPollingIntervalMs > 0) {
+ UnifiedMemoryManager.startPollingIfNeeded(unmanagedMemoryPollingIntervalMs)
+ }
+
+ /**
+ * Get the current unmanaged memory usage in bytes for a specific memory mode.
+ * @param memoryMode The memory mode (ON_HEAP or OFF_HEAP) to get usage for
+ * @return The current unmanaged memory usage in bytes
+ */
+ private def getUnmanagedMemoryUsed(memoryMode: MemoryMode): Long = {
+ // Only consider unmanaged memory if polling is enabled
+ if (unmanagedMemoryPollingIntervalMs <= 0) {
+ return 0L
+ }
+ memoryMode match {
+ case MemoryMode.ON_HEAP => UnifiedMemoryManager.unmanagedOnHeapUsed.get()
+ case MemoryMode.OFF_HEAP => UnifiedMemoryManager.unmanagedOffHeapUsed.get()
+ }
+ }
private def assertInvariants(): Unit = {
assert(onHeapExecutionMemoryPool.poolSize + onHeapStorageMemoryPool.poolSize == maxHeapMemory)
@@ -140,9 +188,15 @@ private[spark] class UnifiedMemoryManager(
* in execution memory allocation across tasks, Otherwise, a task may occupy more than
* its fair share of execution memory, mistakenly thinking that other tasks can acquire
* the portion of storage memory that cannot be evicted.
+ *
+ * This also factors in unmanaged memory usage to ensure we don't over-allocate memory
+ * when unmanaged components are consuming significant memory.
*/
def computeMaxExecutionPoolSize(): Long = {
- maxMemory - math.min(storagePool.memoryUsed, storageRegionSize)
+ val unmanagedMemory = getUnmanagedMemoryUsed(memoryMode)
+ val availableMemory = maxMemory - math.min(storagePool.memoryUsed, storageRegionSize)
+ // Reduce available memory by unmanaged memory usage to prevent over-allocation
+ math.max(0L, availableMemory - unmanagedMemory)
}
executionPool.acquireMemory(
@@ -165,11 +219,21 @@ private[spark] class UnifiedMemoryManager(
offHeapStorageMemoryPool,
maxOffHeapStorageMemory)
}
- if (numBytes > maxMemory) {
+
+ // Factor in unmanaged memory usage for the specific memory mode
+ val unmanagedMemory = getUnmanagedMemoryUsed(memoryMode)
+ val effectiveMaxMemory = math.max(0L, maxMemory - unmanagedMemory)
+
+ if (numBytes > effectiveMaxMemory) {
// Fail fast if the block simply won't fit
logInfo(log"Will not store ${MDC(BLOCK_ID, blockId)} as the required space" +
log" (${MDC(NUM_BYTES, numBytes)} bytes) exceeds our" +
- log" memory limit (${MDC(NUM_BYTES_MAX, maxMemory)} bytes)")
+ log" memory limit (${MDC(NUM_BYTES_MAX, effectiveMaxMemory)} bytes)" +
+ (if (unmanagedMemory > 0) {
+ log" (unmanaged memory usage: ${MDC(NUM_BYTES, unmanagedMemory)} bytes)"
+ } else {
+ log""
+ }))
return false
}
if (numBytes > storagePool.memoryFree) {
@@ -191,7 +255,7 @@ private[spark] class UnifiedMemoryManager(
}
}
-object UnifiedMemoryManager {
+object UnifiedMemoryManager extends Logging {
// Set aside a fixed amount of memory for non-storage, non-execution purposes.
// This serves a function similar to `spark.memory.fraction`, but guarantees that we reserve
@@ -199,6 +263,181 @@ object UnifiedMemoryManager {
// the memory used for execution and storage will be (1024 - 300) * 0.6 = 434MB by default.
private val RESERVED_SYSTEM_MEMORY_BYTES = 300 * 1024 * 1024
+ private val unmanagedMemoryConsumers =
+ new ConcurrentHashMap[UnmanagedMemoryConsumerId, UnmanagedMemoryConsumer]
+
+ // Cached unmanaged memory usage values updated by polling
+ private val unmanagedOnHeapUsed = new AtomicLong(0L)
+ private val unmanagedOffHeapUsed = new AtomicLong(0L)
+
+ // Atomic flag to ensure polling is only started once per JVM
+ private val pollingStarted = new AtomicBoolean(false)
+
+ /**
+ * Register an unmanaged memory consumer to track its memory usage.
+ *
+ * Unmanaged memory consumers are components that manage their own memory outside
+ * of Spark's unified memory management system. By registering, their memory usage
+ * will be periodically polled and factored into Spark's memory allocation decisions.
+ *
+ * @param unmanagedMemoryConsumer The consumer to register for memory tracking
+ */
+ def registerUnmanagedMemoryConsumer(
+ unmanagedMemoryConsumer: UnmanagedMemoryConsumer): Unit = {
+ val id = unmanagedMemoryConsumer.unmanagedMemoryConsumerId
+ unmanagedMemoryConsumers.put(id, unmanagedMemoryConsumer)
+ }
+
+ /**
+ * Unregister an unmanaged memory consumer.
+ * This should be called when a component is shutting down to prevent memory leaks
+ * and ensure accurate memory tracking.
+ *
+ * @param unmanagedMemoryConsumer The consumer to unregister. Only used in tests
+ */
+ private[spark] def unregisterUnmanagedMemoryConsumer(
+ unmanagedMemoryConsumer: UnmanagedMemoryConsumer): Unit = {
+ val id = unmanagedMemoryConsumer.unmanagedMemoryConsumerId
+ unmanagedMemoryConsumers.remove(id)
+ }
+
+
+ /**
+ * Get the current memory usage in bytes for a specific component type.
+ * @param componentType The type of component to filter by (e.g., "RocksDB")
+ * @return Total memory usage in bytes for the specified component type
+ */
+ def getMemoryByComponentType(componentType: String): Long = {
+ unmanagedMemoryConsumers.asScala.values.toSeq
+ .filter(_.unmanagedMemoryConsumerId.componentType == componentType)
+ .map { memoryUser =>
+ try {
+ memoryUser.getMemBytesUsed
+ } catch {
+ case e: Exception =>
+ 0L
+ }
+ }
+ .sum
+ }
+
+ /**
+ * Clear all unmanaged memory users.
+ * This is useful during executor shutdown or cleanup.
+ * Since each executor runs in its own JVM, this clears all users for this executor.
+ */
+ def clearUnmanagedMemoryUsers(): Unit = {
+ unmanagedMemoryConsumers.clear()
+ // Reset cached values when clearing consumers
+ unmanagedOnHeapUsed.set(0L)
+ unmanagedOffHeapUsed.set(0L)
+ }
+
+ // Shared polling infrastructure - only one polling thread per JVM
+ @volatile private var unmanagedMemoryPoller: ScheduledExecutorService = _
+
+ /**
+ * Start unmanaged memory polling if not already started.
+ * This ensures only one polling thread is created per JVM, regardless of how many
+ * UnifiedMemoryManager instances are created.
+ */
+ private[memory] def startPollingIfNeeded(pollingIntervalMs: Long): Unit = {
+ if (pollingStarted.compareAndSet(false, true)) {
+ unmanagedMemoryPoller = ThreadUtils.newDaemonSingleThreadScheduledExecutor(
+ "unmanaged-memory-poller")
+
+ val pollingTask = new Runnable {
+ override def run(): Unit = Utils.tryLogNonFatalError {
+ pollUnmanagedMemoryUsers()
+ }
+ }
+
+ unmanagedMemoryPoller.scheduleAtFixedRate(
+ pollingTask,
+ 0L, // initial delay
+ pollingIntervalMs,
+ TimeUnit.MILLISECONDS)
+
+ logInfo(log"Unmanaged memory polling started with interval " +
+ log"${MDC(LogKeys.TIME, pollingIntervalMs)}ms")
+ }
+ }
+
+ private def pollUnmanagedMemoryUsers(): Unit = {
+ val consumers = unmanagedMemoryConsumers.asScala.toMap
+
+ // Get memory usage for each consumer, handling failures gracefully
+ val memoryUsages = consumers.map { case (userId, memoryUser) =>
+ try {
+ val memoryUsed = memoryUser.getMemBytesUsed
+ if (memoryUsed == -1L) {
+ logDebug(log"Unmanaged memory consumer ${MDC(LogKeys.OBJECT_ID, userId.toString)} " +
+ log"is no longer active, marking for removal")
+ (userId, memoryUser, None) // Mark for removal
+ } else if (memoryUsed < 0L) {
+ logWarning(log"Invalid memory usage value ${MDC(LogKeys.NUM_BYTES, memoryUsed)} " +
+ log"from unmanaged memory user ${MDC(LogKeys.OBJECT_ID, userId.toString)}")
+ (userId, memoryUser, Some(0L)) // Treat as 0
+ } else {
+ (userId, memoryUser, Some(memoryUsed))
+ }
+ } catch {
+ case NonFatal(e) =>
+ logWarning(log"Failed to get memory usage for unmanaged memory user " +
+ log"${MDC(LogKeys.OBJECT_ID, userId.toString)} ${MDC(LogKeys.EXCEPTION, e)}")
+ (userId, memoryUser, Some(0L)) // Treat as 0 on error
+ }
+ }
+
+ // Remove inactive consumers
+ memoryUsages.filter(_._3.isEmpty).foreach { case (userId, _, _) =>
+ unmanagedMemoryConsumers.remove(userId)
+ logInfo(log"Removed inactive unmanaged memory consumer " +
+ log"${MDC(LogKeys.OBJECT_ID, userId.toString)}")
+ }
+ // Calculate total memory usage by mode
+ val activeUsages = memoryUsages.filter(_._3.isDefined)
+ val onHeapTotal = activeUsages
+ .filter(_._2.memoryMode == MemoryMode.ON_HEAP)
+ .map(_._3.get)
+ .sum
+ val offHeapTotal = activeUsages
+ .filter(_._2.memoryMode == MemoryMode.OFF_HEAP)
+ .map(_._3.get)
+ .sum
+ // Update cached values atomically
+ unmanagedOnHeapUsed.set(onHeapTotal)
+ unmanagedOffHeapUsed.set(offHeapTotal)
+ // Log polling results for monitoring
+ val totalMemoryUsed = onHeapTotal + offHeapTotal
+ val numConsumers = activeUsages.size
+ logDebug(s"Unmanaged memory polling completed: $numConsumers consumers, " +
+ s"total memory used: ${totalMemoryUsed} bytes " +
+ s"(on-heap: ${onHeapTotal}, off-heap: ${offHeapTotal})")
+ }
+
+ /**
+ * Shutdown the unmanaged memory polling thread. Only used in tests
+ */
+ private[spark] def shutdownUnmanagedMemoryPoller(): Unit = {
+ synchronized {
+ if (unmanagedMemoryPoller != null) {
+ unmanagedMemoryPoller.shutdown()
+ try {
+ if (!unmanagedMemoryPoller.awaitTermination(5, TimeUnit.SECONDS)) {
+ unmanagedMemoryPoller.shutdownNow()
+ }
+ } catch {
+ case _: InterruptedException =>
+ Thread.currentThread().interrupt()
+ }
+ unmanagedMemoryPoller = null
+ pollingStarted.set(false)
+ logInfo(log"Unmanaged memory poller shutdown complete")
+ }
+ }
+ }
+
def apply(conf: SparkConf, numCores: Int): UnifiedMemoryManager = {
val maxMemory = getMaxMemory(conf)
new UnifiedMemoryManager(
diff --git a/core/src/main/scala/org/apache/spark/memory/UnmanagedMemoryConsumer.scala b/core/src/main/scala/org/apache/spark/memory/UnmanagedMemoryConsumer.scala
new file mode 100644
index 0000000000000..8351918282156
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/memory/UnmanagedMemoryConsumer.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.memory
+
+/**
+ * Identifier for an unmanaged memory consumer.
+ *
+ * @param componentType The type of component (e.g., "RocksDB", "NativeLibrary")
+ * @param instanceKey A unique key to identify this specific instance of the component.
+ * For shared memory consumers, this should be a common key across
+ * all instances to avoid double counting.
+ */
+case class UnmanagedMemoryConsumerId(
+ componentType: String,
+ instanceKey: String
+)
+
+/**
+ * Interface for components that consume memory outside of Spark's unified memory management.
+ *
+ * Components implementing this trait can register themselves with the memory manager
+ * to have their memory usage tracked and factored into memory allocation decisions.
+ * This helps prevent OOM errors when unmanaged components use significant memory.
+ *
+ * Examples of unmanaged memory consumers:
+ * - RocksDB state stores in structured streaming
+ * - Native libraries with custom memory allocation
+ * - Off-heap caches managed outside of Spark
+ */
+trait UnmanagedMemoryConsumer {
+ /**
+ * Returns the unique identifier for this memory consumer.
+ * The identifier is used to track and manage the consumer in the memory tracking system.
+ */
+ def unmanagedMemoryConsumerId: UnmanagedMemoryConsumerId
+
+ /**
+ * Returns the memory mode (ON_HEAP or OFF_HEAP) that this consumer uses.
+ * This is used to ensure unmanaged memory usage only affects the correct memory pool.
+ */
+ def memoryMode: MemoryMode
+
+ /**
+ * Returns the current memory usage in bytes.
+ *
+ * This method is called periodically by the memory polling mechanism to track
+ * memory usage over time. Implementations should return the current total memory
+ * consumed by this component.
+ *
+ * @return Current memory usage in bytes. Should return 0 if no memory is currently used.
+ * Return -1L to indicate this consumer is no longer active and should be
+ * automatically removed from tracking.
+ * @throws Exception if memory usage cannot be determined. The polling mechanism
+ * will handle exceptions gracefully and log warnings.
+ */
+ def getMemBytesUsed: Long
+}
diff --git a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
index 965468ac2418f..b192a9f9babd1 100644
--- a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
@@ -24,7 +24,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.SparkEnv
import org.apache.spark.executor.ProcfsMetricsGetter
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.memory.MemoryManager
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
index a845feeb67ff0..b975cefb336c2 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import scala.util.matching.Regex
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PATH
import org.apache.spark.internal.config.METRICS_CONF
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
index 709ce0060e150..af1f54007248d 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
@@ -26,7 +26,7 @@ import com.codahale.metrics.{Metric, MetricRegistry}
import org.eclipse.jetty.servlet.ServletContextHandler
import org.apache.spark.{SecurityManager, SparkConf}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.metrics.sink.{MetricsServlet, PrometheusServlet, Sink}
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala
index 30b10d64882ac..4b74354aa15e3 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit
import com.codahale.metrics.{Metric, MetricFilter, MetricRegistry}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PREFIX
import org.apache.spark.metrics.MetricsSystem
diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
index a922eb336c28f..03810292bd05c 100644
--- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
+++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
@@ -23,7 +23,7 @@ import scala.jdk.CollectionConverters._
import scala.reflect.ClassTag
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.BlockDataManager
import org.apache.spark.network.buffer.NioManagedBuffer
diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
index 7ceb50db5966a..fe55518d80004 100644
--- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
+++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
@@ -30,7 +30,7 @@ import com.codahale.metrics.{Metric, MetricSet}
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.ExecutorDeadException
-import org.apache.spark.internal.{config, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.network._
import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer}
import org.apache.spark.network.client.{RpcResponseCallback, TransportClientBootstrap}
@@ -57,7 +57,7 @@ private[spark] class NettyBlockTransferService(
_port: Int,
numCores: Int,
driverEndPointRef: RpcEndpointRef = null)
- extends BlockTransferService {
+ extends BlockTransferService with Logging {
// TODO: Don't use Java serialization, use a more cross-version compatible serialization format.
private val serializer = serializerManager.getSerializer(scala.reflect.classTag[Any], false)
@@ -196,9 +196,9 @@ private[spark] class NettyBlockTransferService(
override def onFailure(e: Throwable): Unit = {
if (asStream) {
- logger.error(s"Error while uploading {} as stream", e, MDC.of(LogKeys.BLOCK_ID, blockId))
+ logger.error(s"Error while uploading {} as stream", e, MDC(LogKeys.BLOCK_ID, blockId))
} else {
- logger.error(s"Error while uploading {}", e, MDC.of(LogKeys.BLOCK_ID, blockId))
+ logger.error(s"Error while uploading {}", e, MDC(LogKeys.BLOCK_ID, blockId))
}
result.failure(e)
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
index 44ce1bfa8c5c8..e594b086bfda2 100644
--- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
@@ -24,7 +24,7 @@ import java.util.{Date, Locale}
import scala.reflect.ClassTag
-import org.apache.hadoop.conf.{Configurable, Configuration}
+import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hdfs.BlockMissingException
import org.apache.hadoop.io.compress.CompressionCodecFactory
import org.apache.hadoop.mapred._
@@ -39,7 +39,7 @@ import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.rdd.HadoopRDD.HadoopMapPartitionsWithSplitRDD
@@ -214,14 +214,13 @@ class HadoopRDD[K, V](
}
}
- protected def getInputFormat(conf: JobConf): InputFormat[K, V] = {
- val newInputFormat = ReflectionUtils.newInstance(inputFormatClass.asInstanceOf[Class[_]], conf)
+ protected def getInputFormat(conf: JobConf): InputFormat[K, V] = try {
+ ReflectionUtils.newInstance(inputFormatClass.asInstanceOf[Class[_]], conf)
.asInstanceOf[InputFormat[K, V]]
- newInputFormat match {
- case c: Configurable => c.setConf(conf)
- case _ =>
- }
- newInputFormat
+ } catch {
+ case r: RuntimeException
+ if r.getCause != null && r.getCause.isInstanceOf[InstantiationException] =>
+ throw new RuntimeException(s"Failed to instantiate ${inputFormatClass.getName}", r.getCause)
}
override def getPartitions: Array[Partition] = {
diff --git a/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala b/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
index 8c10bcbc25a86..e71cf4d002a74 100644
--- a/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
@@ -25,7 +25,7 @@ import org.apache.spark.{Partition, SparkContext, TaskContext}
import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
import org.apache.spark.api.java.function.{Function => JFunction}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.NextIterator
@@ -54,7 +54,9 @@ private[spark] class JdbcPartition(idx: Int, val lower: Long, val upper: Long) e
* @param mapRow a function from a ResultSet to a single row of the desired result type(s).
* This should only call getInt, getString, etc; the RDD takes care of calling next.
* The default maps a ResultSet to an array of Object.
+ * @deprecated Jdbc RDD is deprecated, consider using JDBC data source instead.
*/
+@deprecated("Jdbc RDD is deprecated, consider using JDBC data source instead.", "4.1.0")
class JdbcRDD[T: ClassTag](
sc: SparkContext,
getConnection: () => Connection,
diff --git a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
index 27e4948ce1297..cdf90a03a554a 100644
--- a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
@@ -39,7 +39,7 @@ import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.rdd.NewHadoopRDD.NewHadoopMapPartitionsWithSplitRDD
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
index c0966dd5ede14..956620e0b2c23 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -35,7 +35,7 @@ import org.apache.hadoop.mapreduce.{Job => NewAPIHadoopJob, OutputFormat => NewO
import org.apache.spark._
import org.apache.spark.Partitioner.defaultPartitioner
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.SPECULATION_ENABLED
import org.apache.spark.internal.io._
diff --git a/core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala b/core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala
index 965618ee827d1..99b1546f858c1 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala
@@ -47,23 +47,12 @@ class PartitionerAwareUnionRDDPartition(
}
}
-/**
- * Class representing an RDD that can take multiple RDDs partitioned by the same partitioner and
- * unify them into a single RDD while preserving the partitioner. So m RDDs with p partitions each
- * will be unified to a single RDD with p partitions and the same partitioner. The preferred
- * location for each partition of the unified RDD will be the most common preferred location
- * of the corresponding partitions of the parent RDDs. For example, location of partition 0
- * of the unified RDD will be where most of partition 0 of the parent RDDs are located.
- */
private[spark]
-class PartitionerAwareUnionRDD[T: ClassTag](
+abstract class PartitionerAwareUnionRDDBase[T: ClassTag](
sc: SparkContext,
var rdds: Seq[RDD[T]]
) extends RDD[T](sc, rdds.map(x => new OneToOneDependency(x))) {
- require(rdds.nonEmpty)
- require(rdds.forall(_.partitioner.isDefined))
- require(rdds.flatMap(_.partitioner).toSet.size == 1,
- "Parent RDDs have different partitioners: " + rdds.flatMap(_.partitioner))
+ require(rdds.nonEmpty, "RDDs cannot be empty")
override val partitioner = rdds.head.partitioner
@@ -111,3 +100,49 @@ class PartitionerAwareUnionRDD[T: ClassTag](
rdd.context.getPreferredLocs(rdd, part.index).map(tl => tl.host)
}
}
+
+/**
+ * Class representing an RDD that can take multiple RDDs partitioned by the same partitioner and
+ * unify them into a single RDD while preserving the partitioner. So m RDDs with p partitions each
+ * will be unified to a single RDD with p partitions and the same partitioner. The preferred
+ * location for each partition of the unified RDD will be the most common preferred location
+ * of the corresponding partitions of the parent RDDs. For example, location of partition 0
+ * of the unified RDD will be where most of partition 0 of the parent RDDs are located.
+ */
+private[spark]
+class PartitionerAwareUnionRDD[T: ClassTag](
+ sc: SparkContext,
+ var _rdds: Seq[RDD[T]]
+ ) extends PartitionerAwareUnionRDDBase(sc, _rdds) {
+ require(_rdds.forall(_.partitioner.isDefined))
+ require(_rdds.flatMap(_.partitioner).toSet.size == 1,
+ "Parent RDDs have different partitioners: " + _rdds.flatMap(_.partitioner))
+}
+
+/**
+ * This is similar to [[PartitionerAwareUnionRDD]], but it doesn't require the parent RDDs
+ * to have defined partitioner and have the same partitioner if defined.
+ * It is because SQL's shuffle RDD's partitioner is not defined in `ShuffledRowRDD`.
+ * The actual partitioning is implemented in `ShuffleExchangeExec.prepareShuffleDependency`.
+ *
+ * Thus, this RDD doesn't check the partitioner of parent RDDs. Its correctness relies on the
+ * fact that the given RDDs are partitioned in the same way. So before using this RDD, you must
+ * ensure that all parent RDDs are partitioned correctly by checking their SQL output partitioning.
+ */
+private[spark]
+class SQLPartitioningAwareUnionRDD[T: ClassTag](
+ sc: SparkContext,
+ var _rdds: Seq[RDD[T]],
+ val numPartitions: Int
+ ) extends PartitionerAwareUnionRDDBase(sc, _rdds) {
+ require(partitioner.isEmpty || partitioner.get.numPartitions == numPartitions,
+ "Partitioner of parent RDDs does not match the number of partitions: " +
+ s"expected $numPartitions, but got ${partitioner.map(_.numPartitions).getOrElse("none")}")
+
+ override def getPartitions: Array[Partition] = {
+ (0 until numPartitions).map { index =>
+ new PartitionerAwareUnionRDDPartition(_rdds, index)
+ }.toArray
+ }
+}
+
diff --git a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
index 126c92e4cb656..5a9e51026ba98 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
@@ -35,7 +35,6 @@ import scala.reflect.ClassTag
import org.apache.spark.{Partition, TaskContext}
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.internal.LogKeys.{COMMAND, ERROR, PATH}
-import org.apache.spark.internal.MDC
import org.apache.spark.util.Utils
@@ -90,7 +89,7 @@ private[spark] class PipedRDD[T: ClassTag](
val currentDir = new File(".")
logDebug("currentDir = " + currentDir.getAbsolutePath())
val taskDirFile = new File(taskDirectory)
- taskDirFile.mkdirs()
+ Utils.createDirectory(taskDirFile)
try {
val tasksDirFilter = new NotEqualsFileNameFilter("tasks")
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 80db818b77e42..d1408ee774ce8 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -36,7 +36,7 @@ import org.apache.spark.Partitioner._
import org.apache.spark.annotation.{DeveloperApi, Experimental, Since}
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.RDD_LIMIT_SCALE_UP_FACTOR
@@ -1773,7 +1773,7 @@ abstract class RDD[T: ClassTag](
/**
* Return whether this RDD is reliably checkpointed and materialized.
*/
- private[rdd] def isReliablyCheckpointed: Boolean = {
+ private[spark] def isReliablyCheckpointed: Boolean = {
checkpointData match {
case Some(reliable: ReliableRDDCheckpointData[_]) if reliable.isCheckpointed => true
case _ => false
@@ -2016,7 +2016,7 @@ abstract class RDD[T: ClassTag](
def firstDebugString(rdd: RDD[_]): Seq[String] = {
val partitionStr = "(" + rdd.partitions.length + ")"
val leftOffset = (partitionStr.length - 1) / 2
- val nextPrefix = (" " * leftOffset) + "|" + (" " * (partitionStr.length - leftOffset))
+ val nextPrefix = " ".repeat(leftOffset) + "|" + " ".repeat(partitionStr.length - leftOffset)
debugSelf(rdd).zipWithIndex.map{
case (desc: String, 0) => s"$partitionStr $desc"
@@ -2030,7 +2030,7 @@ abstract class RDD[T: ClassTag](
val nextPrefix = (
thisPrefix
+ (if (isLastChild) " " else "| ")
- + (" " * leftOffset) + "|" + (" " * (partitionStr.length - leftOffset)))
+ + (" ".repeat(leftOffset)) + "|" + (" ".repeat(partitionStr.length - leftOffset)))
debugSelf(rdd).zipWithIndex.map{
case (desc: String, 0) => s"$thisPrefix+-$partitionStr $desc"
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
index 3abb2d8a11f35..49c259999a471 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
@@ -17,13 +17,13 @@
package org.apache.spark.rdd
+import java.util.Objects
import java.util.concurrent.atomic.AtomicInteger
import com.fasterxml.jackson.annotation.{JsonIgnore, JsonInclude, JsonPropertyOrder}
import com.fasterxml.jackson.annotation.JsonInclude.Include
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
-import com.google.common.base.Objects
import org.apache.spark.SparkContext
import org.apache.spark.internal.Logging
@@ -69,7 +69,7 @@ private[spark] class RDDOperationScope(
}
}
- override def hashCode(): Int = Objects.hashCode(id, name, parent)
+ override def hashCode(): Int = Objects.hash(id, name, parent)
override def toString: String = toJson
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala b/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
index ee13acc650337..fd42cea795d60 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path
import org.apache.spark._
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{BUFFER_SIZE, CACHE_CHECKPOINT_PREFERRED_LOCS_EXPIRE_TIME, CHECKPOINT_COMPRESS}
import org.apache.spark.io.CompressionCodec
diff --git a/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala b/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala
index b468a38fcf229..9f80d53d2c0bf 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NEW_RDD_ID, RDD_CHECKPOINT_DIR, RDD_ID}
import org.apache.spark.internal.config.CLEANER_REFERENCE_TRACKING_CLEAN_CHECKPOINTS
diff --git a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
index 118660ef69476..14e8bade2200a 100644
--- a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
@@ -23,7 +23,7 @@ import org.apache.hadoop.io.compress.CompressionCodec
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapred.SequenceFileOutputFormat
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
/**
* Extra functions available on RDDs of (key, value) pairs to create a Hadoop SequenceFile,
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
index 51de7e2b9ac70..b11c88828e065 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
@@ -23,7 +23,7 @@ import java.util.Optional
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.api.resource.ResourceDiscoveryPlugin
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.util.Utils.executeAndGetOutput
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
index 0baa578764d08..1b4b4f61016a4 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
@@ -26,7 +26,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.{SparkConf, SparkContext, SparkEnv, SparkException}
import org.apache.spark.annotation.{Evolving, Since}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.Python.PYSPARK_EXECUTOR_MEMORY
@@ -106,6 +106,14 @@ class ResourceProfile(
executorResources.get(ResourceProfile.PYSPARK_MEM).map(_.amount)
}
+ private[spark] def getOverheadMemory: Option[Long] = {
+ executorResources.get(ResourceProfile.OVERHEAD_MEM).map(_.amount)
+ }
+
+ private[spark] def getExecutorOffHeap: Option[Long] = {
+ executorResources.get(ResourceProfile.OFFHEAP_MEM).map(_.amount)
+ }
+
private[spark] def getExecutorMemory: Option[Long] = {
executorResources.get(ResourceProfile.MEMORY).map(_.amount)
}
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala
index 6a6b5067f70f2..10121f6ef2667 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala
@@ -23,7 +23,7 @@ import scala.collection.mutable.HashMap
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.annotation.Evolving
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config.Tests._
import org.apache.spark.scheduler.{LiveListenerBus, SparkListenerResourceProfileAdded}
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
index 78c45cdc75418..8e11923a9e8f0 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
@@ -28,7 +28,7 @@ import org.json4s.jackson.JsonMethods._
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.api.resource.ResourceDiscoveryPlugin
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{EXECUTOR_CORES, RESOURCES_DISCOVERY_PLUGIN, SPARK_TASK_PREFIX}
import org.apache.spark.internal.config.Tests.RESOURCES_WARNING_TESTING
diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala b/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
index 770ae2f1dd22f..da23f2abd3ac9 100644
--- a/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
@@ -28,7 +28,7 @@ import org.apache.spark.util.{ThreadUtils, Utils}
/**
* An exception thrown if RpcTimeout modifies a `TimeoutException`.
*/
-private[rpc] class RpcTimeoutException(message: String, cause: TimeoutException)
+private[spark] class RpcTimeoutException(message: String, cause: TimeoutException)
extends TimeoutException(message) { initCause(cause) }
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala b/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala
index 8acfef38659c0..faa263c7db6a7 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import scala.util.control.NonFatal
import org.apache.spark.{SparkEnv, SparkException}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.client.RpcResponseCallback
import org.apache.spark.rpc._
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
index 0de67a65593b1..dfaa9ea436458 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
@@ -22,7 +22,7 @@ import javax.annotation.concurrent.GuardedBy
import scala.util.control.NonFatal
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rpc.{RpcAddress, RpcEndpoint, ThreadSafeRpcEndpoint}
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/MessageLoop.scala b/core/src/main/scala/org/apache/spark/rpc/netty/MessageLoop.scala
index 2fd1c6d7fe71e..cce455270df43 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/MessageLoop.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/MessageLoop.scala
@@ -22,7 +22,7 @@ import java.util.concurrent._
import scala.util.control.NonFatal
import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.ERROR
import org.apache.spark.internal.config.EXECUTOR_ID
import org.apache.spark.internal.config.Network._
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
index c2688610fe8b1..a4f6d5438bf3d 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
@@ -30,7 +30,7 @@ import scala.util.{DynamicVariable, Failure, Success, Try}
import scala.util.control.NonFatal
import org.apache.spark.{SecurityManager, SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.EXECUTOR_ID
import org.apache.spark.internal.config.Network._
diff --git a/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala b/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
index 16e9211b54851..3931b8ff5190e 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong}
import com.codahale.metrics.{Gauge, Timer}
import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index baf0ed4df5309..7d77628c3f088 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -37,7 +37,7 @@ import org.apache.spark._
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{LEGACY_ABORT_STAGE_AFTER_KILL_TASKS, RDD_CACHE_VISIBILITY_TRACKING_ENABLED}
import org.apache.spark.internal.config.Tests.TEST_NO_STAGE_RETRY
@@ -49,6 +49,7 @@ import org.apache.spark.rdd.{RDD, RDDCheckpointData}
import org.apache.spark.resource.{ResourceProfile, TaskResourceProfile}
import org.apache.spark.resource.ResourceProfile.{DEFAULT_RESOURCE_PROFILE_ID, EXECUTOR_CORES_LOCAL_PROPERTY, PYSPARK_MEMORY_LOCAL_PROPERTY}
import org.apache.spark.rpc.RpcTimeout
+import org.apache.spark.rpc.RpcTimeoutException
import org.apache.spark.storage._
import org.apache.spark.storage.BlockManagerMessages.BlockManagerHeartbeat
import org.apache.spark.util._
@@ -771,7 +772,14 @@ private[spark] class DAGScheduler(
def visit(rdd: RDD[_]): Unit = {
if (!visited(rdd)) {
visited += rdd
- val rddHasUncachedPartitions = getCacheLocs(rdd).contains(Nil)
+ val rddHasUncachedPartitions = try {
+ getCacheLocs(rdd).contains(Nil)
+ } catch {
+ case e: RpcTimeoutException =>
+ logWarning(log"Failed to get cache locations for RDD ${MDC(RDD_ID, rdd.id)} due " +
+ log"to rpc timeout, assuming not fully cached.", e)
+ true
+ }
if (rddHasUncachedPartitions) {
for (dep <- rdd.dependencies) {
dep match {
@@ -1463,7 +1471,7 @@ private[spark] class DAGScheduler(
abortStage(stage, reason, None)
} else {
val missing = getMissingParentStages(stage).sortBy(_.id)
- logDebug("missing: " + missing)
+ logInfo(log"Missing parents found for ${MDC(STAGE, stage)}: ${MDC(MISSING_PARENT_STAGES, missing)}")
if (missing.isEmpty) {
logInfo(log"Submitting ${MDC(STAGE, stage)} (${MDC(RDD_ID, stage.rdd)}), " +
log"which has no missing parents")
@@ -1551,29 +1559,46 @@ private[spark] class DAGScheduler(
// The operation here can make sure for the partially completed intermediate stage,
// `findMissingPartitions()` returns all partitions every time.
stage match {
- case sms: ShuffleMapStage if stage.isIndeterminate && !sms.isAvailable =>
- // already executed at least once
- if (sms.getNextAttemptId > 0) {
- // While we previously validated possible rollbacks during the handling of a FetchFailure,
- // where we were fetching from an indeterminate source map stages, this later check
- // covers additional cases like recalculating an indeterminate stage after an executor
- // loss. Moreover, because this check occurs later in the process, if a result stage task
- // has successfully completed, we can detect this and abort the job, as rolling back a
- // result stage is not possible.
- val stagesToRollback = collectSucceedingStages(sms)
- abortStageWithInvalidRollBack(stagesToRollback)
- // stages which cannot be rolled back were aborted which leads to removing the
- // the dependant job(s) from the active jobs set
- val numActiveJobsWithStageAfterRollback =
- activeJobs.count(job => stagesToRollback.contains(job.finalStage))
- if (numActiveJobsWithStageAfterRollback == 0) {
- logInfo(log"All jobs depending on the indeterminate stage " +
- log"(${MDC(STAGE_ID, stage.id)}) were aborted so this stage is not needed anymore.")
- return
+ case sms: ShuffleMapStage if !sms.isAvailable =>
+ val needFullStageRetry = if (sms.shuffleDep.checksumMismatchFullRetryEnabled) {
+ // When the parents of this stage are indeterminate (e.g., some parents are not
+ // checkpointed and checksum mismatches are detected), the output data of the parents
+ // may have changed due to task retries. For correctness reason, we need to
+ // retry all tasks of the current stage. The legacy way of using current stage's
+ // deterministic level to trigger full stage retry is not accurate.
+ stage.isParentIndeterminate
+ } else {
+ if (stage.isIndeterminate) {
+ // already executed at least once
+ if (sms.getNextAttemptId > 0) {
+ // While we previously validated possible rollbacks during the handling of a FetchFailure,
+ // where we were fetching from an indeterminate source map stages, this later check
+ // covers additional cases like recalculating an indeterminate stage after an executor
+ // loss. Moreover, because this check occurs later in the process, if a result stage task
+ // has successfully completed, we can detect this and abort the job, as rolling back a
+ // result stage is not possible.
+ val stagesToRollback = collectSucceedingStages(sms)
+ abortStageWithInvalidRollBack(stagesToRollback)
+ // stages which cannot be rolled back were aborted which leads to removing the
+ // the dependant job(s) from the active jobs set
+ val numActiveJobsWithStageAfterRollback =
+ activeJobs.count(job => stagesToRollback.contains(job.finalStage))
+ if (numActiveJobsWithStageAfterRollback == 0) {
+ logInfo(log"All jobs depending on the indeterminate stage " +
+ log"(${MDC(STAGE_ID, stage.id)}) were aborted so this stage is not needed anymore.")
+ return
+ }
+ }
+ true
+ } else {
+ false
}
}
- mapOutputTracker.unregisterAllMapAndMergeOutput(sms.shuffleDep.shuffleId)
- sms.shuffleDep.newShuffleMergeState()
+
+ if (needFullStageRetry) {
+ mapOutputTracker.unregisterAllMapAndMergeOutput(sms.shuffleDep.shuffleId)
+ sms.shuffleDep.newShuffleMergeState()
+ }
case _ =>
}
@@ -1886,6 +1911,20 @@ private[spark] class DAGScheduler(
}
}
+ /**
+ * If a map stage is non-deterministic, the map tasks of the stage may return different result
+ * when re-try. To make sure data correctness, we need to re-try all the tasks of its succeeding
+ * stages, as the input data may be changed after the map tasks are re-tried. For stages where
+ * rollback and retry all tasks are not possible, we will need to abort the stages.
+ */
+ private[scheduler] def abortUnrollbackableStages(mapStage: ShuffleMapStage): Unit = {
+ val stagesToRollback = collectSucceedingStages(mapStage)
+ val rollingBackStages = abortStageWithInvalidRollBack(stagesToRollback)
+ logInfo(log"The shuffle map stage ${MDC(SHUFFLE_ID, mapStage)} with indeterminate output " +
+ log"was failed, we will roll back and rerun below stages which include itself and all its " +
+ log"indeterminate child stages: ${MDC(STAGES, rollingBackStages)}")
+ }
+
/**
* Responds to a task finishing. This is called inside the event loop so it assumes that it can
* modify the scheduler's internal state. Use taskEnded() to post a task end event from outside.
@@ -2022,8 +2061,26 @@ private[spark] class DAGScheduler(
// The epoch of the task is acceptable (i.e., the task was launched after the most
// recent failure we're aware of for the executor), so mark the task's output as
// available.
- mapOutputTracker.registerMapOutput(
+ val isChecksumMismatched = mapOutputTracker.registerMapOutput(
shuffleStage.shuffleDep.shuffleId, smt.partitionId, status)
+ if (isChecksumMismatched) {
+ shuffleStage.isChecksumMismatched = isChecksumMismatched
+ // There could be multiple checksum mismatches detected for a single stage attempt.
+ // We check for stage abortion once and only once when we first detect checksum
+ // mismatch for each stage attempt. For example, assume that we have
+ // stage1 -> stage2, and we encounter checksum mismatch during the retry of stage1.
+ // In this case, we need to call abortUnrollbackableStages() for the succeeding
+ // stages. Assume that when stage2 is retried, some tasks finish and some tasks
+ // failed again with FetchFailed. In case that we encounter checksum mismatch again
+ // during the retry of stage1, we need to call abortUnrollbackableStages() again.
+ if (shuffleStage.maxChecksumMismatchedId < smt.stageAttemptId) {
+ shuffleStage.maxChecksumMismatchedId = smt.stageAttemptId
+ if (shuffleStage.shuffleDep.checksumMismatchFullRetryEnabled
+ && shuffleStage.isStageIndeterminate) {
+ abortUnrollbackableStages(shuffleStage)
+ }
+ }
+ }
}
} else {
logInfo(log"Ignoring ${MDC(TASK_NAME, smt)} completion from an older attempt of indeterminate stage")
@@ -2148,12 +2205,8 @@ private[spark] class DAGScheduler(
// Note that, if map stage is UNORDERED, we are fine. The shuffle partitioner is
// guaranteed to be determinate, so the input data of the reducers will not change
// even if the map tasks are re-tried.
- if (mapStage.isIndeterminate) {
- val stagesToRollback = collectSucceedingStages(mapStage)
- val rollingBackStages = abortStageWithInvalidRollBack(stagesToRollback)
- logInfo(log"The shuffle map stage ${MDC(SHUFFLE_ID, mapStage)} with indeterminate output was failed, " +
- log"we will roll back and rerun below stages which include itself and all its " +
- log"indeterminate child stages: ${MDC(STAGES, rollingBackStages)}")
+ if (mapStage.isIndeterminate && !mapStage.shuffleDep.checksumMismatchFullRetryEnabled) {
+ abortUnrollbackableStages(mapStage)
}
// We expect one executor failure to trigger many FetchFailures in rapid succession,
@@ -2762,7 +2815,7 @@ private[spark] class DAGScheduler(
hostToUnregisterOutputs.foreach(
host => blockManagerMaster.removeShufflePushMergerLocation(host))
}
- blockManagerMaster.removeExecutor(execId)
+ blockManagerMaster.removeExecutorAsync(execId)
clearCacheLocs()
}
if (fileLost) {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 1a47d5ed17158..1e46142fab255 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -31,7 +31,7 @@ import org.apache.spark.deploy.history.EventLogFileWriter
import org.apache.spark.executor.ExecutorMetrics
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config._
-import org.apache.spark.util.{JsonProtocol, Utils}
+import org.apache.spark.util.{JsonProtocol, JsonProtocolOptions, Utils}
/**
* A SparkListener that logs events to persistent storage.
@@ -74,7 +74,7 @@ private[spark] class EventLoggingListener(
private val liveStageExecutorMetrics =
mutable.HashMap.empty[(Int, Int), mutable.HashMap[String, ExecutorMetrics]]
- private[this] val jsonProtocol = new JsonProtocol(sparkConf)
+ private[this] val jsonProtocolOptions = new JsonProtocolOptions(sparkConf)
/**
* Creates the log file in the configured log directory.
@@ -86,7 +86,7 @@ private[spark] class EventLoggingListener(
private def initEventLog(): Unit = {
val metadata = SparkListenerLogStart(SPARK_VERSION)
- val eventJson = jsonProtocol.sparkEventToJsonString(metadata)
+ val eventJson = JsonProtocol.sparkEventToJsonString(metadata, jsonProtocolOptions)
logWriter.writeEvent(eventJson, flushLogger = true)
if (testing && loggedEvents != null) {
loggedEvents += eventJson
@@ -95,7 +95,7 @@ private[spark] class EventLoggingListener(
/** Log the event as JSON. */
private def logEvent(event: SparkListenerEvent, flushLogger: Boolean = false): Unit = {
- val eventJson = jsonProtocol.sparkEventToJsonString(event)
+ val eventJson = JsonProtocol.sparkEventToJsonString(event, jsonProtocolOptions)
logWriter.writeEvent(eventJson, flushLogger)
if (testing) {
loggedEvents += eventJson
diff --git a/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala b/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala
index 82ec0ef91f4fc..fe62e39432250 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicReference
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
import org.apache.spark.{ExecutorAllocationClient, SparkConf, SparkContext}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.{Clock, SystemClock, Utils}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
index 7251eb2c86ea1..656f9875de2e2 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
@@ -30,7 +30,7 @@ import com.codahale.metrics.{Counter, MetricRegistry, Timer}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLASS_NAME, MAX_SIZE}
import org.apache.spark.internal.config._
import org.apache.spark.metrics.MetricsSystem
diff --git a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
index 113521453ad7b..e348b6a5f1493 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
@@ -58,6 +58,12 @@ private[spark] sealed trait MapStatus extends ShuffleOutputStatus {
* partitionId of the task or taskContext.taskAttemptId is used.
*/
def mapId: Long
+
+ /**
+ * The checksum value of this shuffle map task, which can be used to evaluate whether the
+ * output data has changed across different map task retries.
+ */
+ def checksumValue: Long = 0
}
@@ -74,11 +80,12 @@ private[spark] object MapStatus {
def apply(
loc: BlockManagerId,
uncompressedSizes: Array[Long],
- mapTaskId: Long): MapStatus = {
+ mapTaskId: Long,
+ checksumVal: Long = 0): MapStatus = {
if (uncompressedSizes.length > minPartitionsToUseHighlyCompressMapStatus) {
- HighlyCompressedMapStatus(loc, uncompressedSizes, mapTaskId)
+ HighlyCompressedMapStatus(loc, uncompressedSizes, mapTaskId, checksumVal)
} else {
- new CompressedMapStatus(loc, uncompressedSizes, mapTaskId)
+ new CompressedMapStatus(loc, uncompressedSizes, mapTaskId, checksumVal)
}
}
@@ -119,18 +126,24 @@ private[spark] object MapStatus {
* @param loc location where the task is being executed.
* @param compressedSizes size of the blocks, indexed by reduce partition id.
* @param _mapTaskId unique task id for the task
+ * @param _checksumVal the checksum value for the task
*/
private[spark] class CompressedMapStatus(
private[this] var loc: BlockManagerId,
private[this] var compressedSizes: Array[Byte],
- private[this] var _mapTaskId: Long)
+ private[this] var _mapTaskId: Long,
+ private[this] var _checksumVal: Long = 0)
extends MapStatus with Externalizable {
// For deserialization only
- protected def this() = this(null, null.asInstanceOf[Array[Byte]], -1)
+ protected def this() = this(null, null.asInstanceOf[Array[Byte]], -1, 0)
- def this(loc: BlockManagerId, uncompressedSizes: Array[Long], mapTaskId: Long) = {
- this(loc, uncompressedSizes.map(MapStatus.compressSize), mapTaskId)
+ def this(
+ loc: BlockManagerId,
+ uncompressedSizes: Array[Long],
+ mapTaskId: Long,
+ checksumVal: Long) = {
+ this(loc, uncompressedSizes.map(MapStatus.compressSize), mapTaskId, checksumVal)
}
override def location: BlockManagerId = loc
@@ -145,11 +158,14 @@ private[spark] class CompressedMapStatus(
override def mapId: Long = _mapTaskId
+ override def checksumValue: Long = _checksumVal
+
override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException {
loc.writeExternal(out)
out.writeInt(compressedSizes.length)
out.write(compressedSizes)
out.writeLong(_mapTaskId)
+ out.writeLong(_checksumVal)
}
override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException {
@@ -158,6 +174,7 @@ private[spark] class CompressedMapStatus(
compressedSizes = new Array[Byte](len)
in.readFully(compressedSizes)
_mapTaskId = in.readLong()
+ _checksumVal = in.readLong()
}
}
@@ -172,6 +189,7 @@ private[spark] class CompressedMapStatus(
* @param avgSize average size of the non-empty and non-huge blocks
* @param hugeBlockSizes sizes of huge blocks by their reduceId.
* @param _mapTaskId unique task id for the task
+ * @param _checksumVal checksum value for the task
*/
private[spark] class HighlyCompressedMapStatus private (
private[this] var loc: BlockManagerId,
@@ -179,7 +197,8 @@ private[spark] class HighlyCompressedMapStatus private (
private[this] var emptyBlocks: RoaringBitmap,
private[this] var avgSize: Long,
private[this] var hugeBlockSizes: scala.collection.Map[Int, Byte],
- private[this] var _mapTaskId: Long)
+ private[this] var _mapTaskId: Long,
+ private[this] var _checksumVal: Long = 0)
extends MapStatus with Externalizable {
// loc could be null when the default constructor is called during deserialization
@@ -187,7 +206,7 @@ private[spark] class HighlyCompressedMapStatus private (
|| numNonEmptyBlocks == 0 || _mapTaskId > 0,
"Average size can only be zero for map stages that produced no output")
- protected def this() = this(null, -1, null, -1, null, -1) // For deserialization only
+ protected def this() = this(null, -1, null, -1, null, -1, 0) // For deserialization only
override def location: BlockManagerId = loc
@@ -209,6 +228,8 @@ private[spark] class HighlyCompressedMapStatus private (
override def mapId: Long = _mapTaskId
+ override def checksumValue: Long = _checksumVal
+
override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException {
loc.writeExternal(out)
emptyBlocks.serialize(out)
@@ -219,6 +240,7 @@ private[spark] class HighlyCompressedMapStatus private (
out.writeByte(kv._2)
}
out.writeLong(_mapTaskId)
+ out.writeLong(_checksumVal)
}
override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException {
@@ -236,6 +258,7 @@ private[spark] class HighlyCompressedMapStatus private (
}
hugeBlockSizes = hugeBlockSizesImpl
_mapTaskId = in.readLong()
+ _checksumVal = in.readLong()
}
}
@@ -243,7 +266,8 @@ private[spark] object HighlyCompressedMapStatus {
def apply(
loc: BlockManagerId,
uncompressedSizes: Array[Long],
- mapTaskId: Long): HighlyCompressedMapStatus = {
+ mapTaskId: Long,
+ checksumVal: Long = 0): HighlyCompressedMapStatus = {
// We must keep track of which blocks are empty so that we don't report a zero-sized
// block as being non-empty (or vice-versa) when using the average block size.
var i = 0
@@ -310,6 +334,6 @@ private[spark] object HighlyCompressedMapStatus {
emptyBlocks.trim()
emptyBlocks.runOptimize()
new HighlyCompressedMapStatus(loc, numNonEmptyBlocks, emptyBlocks, avgSize,
- hugeBlockSizes, mapTaskId)
+ hugeBlockSizes, mapTaskId, checksumVal)
}
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
index a769c3fa14b62..3e6f0fe3bfc44 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
@@ -20,7 +20,7 @@ package org.apache.spark.scheduler
import scala.collection.mutable
import org.apache.spark._
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.rpc.{RpcCallContext, RpcEndpoint, RpcEndpointRef, RpcEnv}
import org.apache.spark.util.{RpcUtils, ThreadUtils}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ReplayListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/ReplayListenerBus.scala
index 0aa74190d9f3d..2e6cfa98ff373 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ReplayListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ReplayListenerBus.scala
@@ -24,7 +24,7 @@ import scala.io.{Codec, Source}
import com.fasterxml.jackson.core.JsonParseException
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.scheduler.ReplayListenerBus._
import org.apache.spark.util.JsonProtocol
@@ -32,8 +32,7 @@ import org.apache.spark.util.JsonProtocol
/**
* A SparkListenerBus that can be used to replay events from serialized event data.
*/
-private[spark] class ReplayListenerBus(
- jsonProtocol: JsonProtocol) extends SparkListenerBus with Logging {
+private[spark] class ReplayListenerBus extends SparkListenerBus with Logging {
/**
* Replay each event in the order maintained in the given stream. The stream is expected to
@@ -87,7 +86,7 @@ private[spark] class ReplayListenerBus(
currentLine = entry._1
lineNumber = entry._2 + 1
- postToAll(jsonProtocol.sparkEventFromJson(currentLine))
+ postToAll(JsonProtocol.sparkEventFromJson(currentLine))
} catch {
case e: ClassNotFoundException =>
// Ignore unknown events, parse through the event log file.
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala
index bea49fb279ee3..06b6045cccd99 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala
@@ -26,7 +26,7 @@ import scala.xml.{Node, XML}
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{SCHEDULER_ALLOCATION_FILE, SCHEDULER_MODE}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/Stage.scala b/core/src/main/scala/org/apache/spark/scheduler/Stage.scala
index f35beafd87480..9bf604e9a83cf 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/Stage.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/Stage.scala
@@ -72,6 +72,18 @@ private[scheduler] abstract class Stage(
private var nextAttemptId: Int = 0
private[scheduler] def getNextAttemptId: Int = nextAttemptId
+ /**
+ * Whether checksum mismatches have been detected across different attempt of the stage, where
+ * checksum mismatches typically indicates that different stage attempts have produced different
+ * data.
+ */
+ private[scheduler] var isChecksumMismatched: Boolean = false
+
+ /**
+ * The maximum of task attempt id where checksum mismatches are detected.
+ */
+ private[scheduler] var maxChecksumMismatchedId: Int = nextAttemptId
+
val name: String = callSite.shortForm
val details: String = callSite.longForm
@@ -131,4 +143,14 @@ private[scheduler] abstract class Stage(
def isIndeterminate: Boolean = {
rdd.outputDeterministicLevel == DeterministicLevel.INDETERMINATE
}
+
+ // Returns true if any parents of this stage are indeterminate.
+ def isParentIndeterminate: Boolean = {
+ parents.exists(_.isStageIndeterminate)
+ }
+
+ // Returns true if the stage itself is indeterminate.
+ def isStageIndeterminate: Boolean = {
+ !rdd.isReliablyCheckpointed && isChecksumMismatched
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala b/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala
index e46dde5561a26..c8ffaefba3d64 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.util.{Distribution, Utils}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala
index e21ec77ce69ec..0e5e2c9faa05a 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala
@@ -22,7 +22,7 @@ import java.util.Properties
import org.apache.spark._
import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.APP_CALLER_CONTEXT
import org.apache.spark.internal.plugin.PluginContainer
import org.apache.spark.memory.{MemoryMode, TaskMemoryManager}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala
index 97c539bb05a58..70f58747506cc 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala
@@ -25,7 +25,7 @@ import scala.util.control.NonFatal
import org.apache.spark._
import org.apache.spark.TaskState.TaskState
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.CLASS_LOADER
import org.apache.spark.serializer.{SerializerHelper, SerializerInstance}
import org.apache.spark.util.{LongAccumulator, ThreadUtils, Utils}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 13018da5bc274..1351d8c778b58 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -33,7 +33,7 @@ import org.apache.spark.InternalAccumulator.{input, shuffleRead}
import org.apache.spark.TaskState.TaskState
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.ExecutorMetrics
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.resource.ResourceProfile
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala
index 3637305293107..c1b09567013f0 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala
@@ -19,7 +19,7 @@ package org.apache.spark.scheduler
import scala.collection.mutable.{HashMap, HashSet}
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.util.Clock
/**
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index 0eaf138d3eb8d..69e0a10a34b28 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -30,7 +30,7 @@ import org.apache.spark.InternalAccumulator
import org.apache.spark.InternalAccumulator.{input, shuffleRead}
import org.apache.spark.TaskState.TaskState
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.scheduler.SchedulingMode._
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 4b3a16b4d3f60..49590f0940f76 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -32,7 +32,7 @@ import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.security.HadoopDelegationTokenManager
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.ExecutorLogUrlHandler
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
index eb408a95589f7..061b54914c839 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
@@ -27,7 +27,7 @@ import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.deploy.{ApplicationDescription, Command}
import org.apache.spark.deploy.client.{StandaloneAppClient, StandaloneAppClientListener}
import org.apache.spark.executor.ExecutorExitCode
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.config.EXECUTOR_REMOVE_DELAY
import org.apache.spark.internal.config.Tests.IS_TESTING
import org.apache.spark.launcher.{LauncherBackend, SparkAppHandle}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
index 57505c87f879e..a98672fd7db4a 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config._
import org.apache.spark.resource.ResourceProfile.UNKNOWN_RESOURCE_PROFILE_ID
import org.apache.spark.scheduler._
diff --git a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
index 1ee46d51ce70b..b230df93e1fac 100644
--- a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
+++ b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
@@ -26,12 +26,11 @@ import javax.crypto.spec.{IvParameterSpec, SecretKeySpec}
import scala.jdk.CollectionConverters._
-import com.google.common.io.ByteStreams
import org.apache.commons.crypto.random._
import org.apache.commons.crypto.stream._
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.network.util.{CryptoUtils, JavaUtils}
@@ -84,7 +83,7 @@ private[spark] object CryptoStreamUtils extends Logging {
sparkConf: SparkConf,
key: Array[Byte]): InputStream = {
val iv = new Array[Byte](IV_LENGTH_IN_BYTES)
- ByteStreams.readFully(is, iv)
+ JavaUtils.readFully(is, iv, 0, IV_LENGTH_IN_BYTES)
val params = new CryptoParams(key, sparkConf)
new CryptoInputStream(params.transformation, params.conf, is, params.keySpec,
new IvParameterSpec(iv))
diff --git a/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala
index d09abff2773b8..23a5965d6aead 100644
--- a/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala
@@ -28,7 +28,6 @@ import com.esotericsoftware.kryo.io.{Input => KryoInput, Output => KryoOutput}
import org.apache.avro.{Schema, SchemaNormalization}
import org.apache.avro.generic.{GenericContainer, GenericData}
import org.apache.avro.io._
-import org.apache.commons.io.IOUtils
import org.apache.spark.{SparkEnv, SparkException}
import org.apache.spark.io.CompressionCodec
@@ -93,7 +92,7 @@ private[serializer] class GenericAvroSerializer[D <: GenericContainer]
schemaBytes.remaining())
val in = codec.compressedInputStream(bis)
val bytes = Utils.tryWithSafeFinally {
- IOUtils.toByteArray(in)
+ in.readAllBytes()
} {
in.close()
}
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 000ba8d79bc02..28261a4520a28 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -40,7 +40,7 @@ import org.roaringbitmap.RoaringBitmap
import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.api.python.PythonBroadcast
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.CLASS_NAME
import org.apache.spark.internal.config.Kryo._
import org.apache.spark.internal.io.FileCommitProtocol._
diff --git a/core/src/main/scala/org/apache/spark/serializer/SerializerManager.scala b/core/src/main/scala/org/apache/spark/serializer/SerializerManager.scala
index 640396a695269..d53a4d5497821 100644
--- a/core/src/main/scala/org/apache/spark/serializer/SerializerManager.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/SerializerManager.scala
@@ -172,9 +172,8 @@ private[spark] class SerializerManager(
outputStream: OutputStream,
values: Iterator[T]): Unit = {
val byteStream = new BufferedOutputStream(outputStream)
- val autoPick = !blockId.isInstanceOf[StreamBlockId]
- val ser = getSerializer(implicitly[ClassTag[T]], autoPick).newInstance()
- ser.serializeStream(wrapForCompression(blockId, byteStream)).writeAll(values).close()
+ blockSerializationStream[T](blockId, byteStream)(implicitly[ClassTag[T]])
+ .writeAll(values).close()
}
/** Serializes into a chunked byte buffer. */
@@ -212,4 +211,14 @@ private[spark] class SerializerManager(
.deserializeStream(wrapForCompression(blockId, stream))
.asIterator.asInstanceOf[Iterator[T]]
}
+
+ /** Generate a `SerializationStream` for a block. */
+ private[spark] def blockSerializationStream[T](
+ blockId: BlockId,
+ outputStream: OutputStream)
+ (classTag: ClassTag[T]): SerializationStream = {
+ val autoPick = !blockId.isInstanceOf[StreamBlockId]
+ val ser = getSerializer(classTag, autoPick).newInstance()
+ ser.serializeStream(wrapForCompression(blockId, outputStream))
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
index bf3117a9a9b12..a46c23447f84f 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
@@ -21,7 +21,7 @@ import java.io._
import java.nio.ByteBuffer
import java.nio.channels.Channels
import java.nio.file.Files
-import java.util.{Collections, Map => JMap}
+import java.util.concurrent.{ConcurrentHashMap, ConcurrentMap}
import scala.collection.mutable.ArrayBuffer
@@ -29,7 +29,7 @@ import com.google.common.cache.CacheBuilder
import org.apache.spark.{SecurityManager, SparkConf, SparkEnv, SparkException}
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.io.NioBufferedFileInputStream
import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer}
@@ -58,19 +58,19 @@ private[spark] class IndexShuffleBlockResolver(
conf: SparkConf,
// var for testing
var _blockManager: BlockManager,
- val taskIdMapsForShuffle: JMap[Int, OpenHashSet[Long]])
+ val taskIdMapsForShuffle: ConcurrentMap[Int, OpenHashSet[Long]])
extends ShuffleBlockResolver
with Logging with MigratableResolver {
def this(conf: SparkConf) = {
- this(conf, null, Collections.emptyMap())
+ this(conf, null, new ConcurrentHashMap[Int, OpenHashSet[Long]]())
}
def this(conf: SparkConf, _blockManager: BlockManager) = {
- this(conf, _blockManager, Collections.emptyMap())
+ this(conf, _blockManager, new ConcurrentHashMap[Int, OpenHashSet[Long]]())
}
- def this(conf: SparkConf, taskIdMapsForShuffle: JMap[Int, OpenHashSet[Long]]) = {
+ def this(conf: SparkConf, taskIdMapsForShuffle: ConcurrentHashMap[Int, OpenHashSet[Long]]) = {
this(conf, null, taskIdMapsForShuffle)
}
@@ -310,13 +310,13 @@ private[spark] class IndexShuffleBlockResolver(
val mapTaskIds = taskIdMapsForShuffle.computeIfAbsent(
shuffleId, _ => new OpenHashSet[Long](8)
)
- mapTaskIds.add(mapId)
+ mapTaskIds.synchronized { mapTaskIds.add(mapId) }
case ShuffleDataBlockId(shuffleId, mapId, _) =>
val mapTaskIds = taskIdMapsForShuffle.computeIfAbsent(
shuffleId, _ => new OpenHashSet[Long](8)
)
- mapTaskIds.add(mapId)
+ mapTaskIds.synchronized { mapTaskIds.add(mapId) }
case _ => // Unreachable
}
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala
index 4e3191e44fbdf..548ecb399d5ff 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala
@@ -27,7 +27,7 @@ import scala.util.control.NonFatal
import org.apache.spark.{SecurityManager, ShuffleDependency, SparkConf, SparkContext, SparkEnv}
import org.apache.spark.annotation.Since
import org.apache.spark.executor.{CoarseGrainedExecutorBackend, ExecutorBackend}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.launcher.SparkLauncher
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala
index be42af092f24a..47d54ae4f10b0 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala
@@ -18,7 +18,7 @@
package org.apache.spark.shuffle
import org.apache.spark.{ShuffleDependency, SparkEnv, TaskContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NUM_MERGER_LOCATIONS, SHUFFLE_ID, STAGE_ID}
import org.apache.spark.scheduler.MapStatus
diff --git a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala
index 6902fb6d236de..ca5f5bc28ae92 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleManager.scala
@@ -178,8 +178,10 @@ private[spark] class SortShuffleManager(conf: SparkConf) extends ShuffleManager
/** Remove a shuffle's metadata from the ShuffleManager. */
override def unregisterShuffle(shuffleId: Int): Boolean = {
Option(taskIdMapsForShuffle.remove(shuffleId)).foreach { mapTaskIds =>
- mapTaskIds.iterator.foreach { mapTaskId =>
- shuffleBlockResolver.removeDataByMap(shuffleId, mapTaskId)
+ mapTaskIds.synchronized {
+ mapTaskIds.iterator.foreach { mapTaskId =>
+ shuffleBlockResolver.removeDataByMap(shuffleId, mapTaskId)
+ }
}
}
true
diff --git a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala
index 3be7d24f7e4ec..a7ac20016a0ec 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala
@@ -23,6 +23,7 @@ import org.apache.spark.scheduler.MapStatus
import org.apache.spark.shuffle.{BaseShuffleHandle, ShuffleWriter}
import org.apache.spark.shuffle.ShuffleWriteMetricsReporter
import org.apache.spark.shuffle.api.ShuffleExecutorComponents
+import org.apache.spark.shuffle.checksum.RowBasedChecksum
import org.apache.spark.util.collection.ExternalSorter
private[spark] class SortShuffleWriter[K, V, C](
@@ -48,17 +49,31 @@ private[spark] class SortShuffleWriter[K, V, C](
private var partitionLengths: Array[Long] = _
+ def getRowBasedChecksums: Array[RowBasedChecksum] = {
+ if (sorter != null) {
+ sorter.getRowBasedChecksums
+ } else {
+ ShuffleDependency.EMPTY_ROW_BASED_CHECKSUMS
+ }
+ }
+
+ def getAggregatedChecksumValue: Long = {
+ if (sorter != null) sorter.getAggregatedChecksumValue else 0
+ }
+
/** Write a bunch of records to this task's output */
override def write(records: Iterator[Product2[K, V]]): Unit = {
sorter = if (dep.mapSideCombine) {
new ExternalSorter[K, V, C](
- context, dep.aggregator, Some(dep.partitioner), dep.keyOrdering, dep.serializer)
+ context, dep.aggregator, Some(dep.partitioner), dep.keyOrdering,
+ dep.serializer, dep.rowBasedChecksums)
} else {
// In this case we pass neither an aggregator nor an ordering to the sorter, because we don't
// care whether the keys get sorted in each partition; that will be done on the reduce side
// if the operation being run is sortByKey.
new ExternalSorter[K, V, V](
- context, aggregator = None, Some(dep.partitioner), ordering = None, dep.serializer)
+ context, aggregator = None, Some(dep.partitioner), ordering = None,
+ dep.serializer, dep.rowBasedChecksums)
}
sorter.insertAll(records)
@@ -69,7 +84,8 @@ private[spark] class SortShuffleWriter[K, V, C](
dep.shuffleId, mapId, dep.partitioner.numPartitions)
sorter.writePartitionedMapOutput(dep.shuffleId, mapId, mapOutputWriter, writeMetrics)
partitionLengths = mapOutputWriter.commitAllPartitions(sorter.getChecksums).getPartitionLengths
- mapStatus = MapStatus(blockManager.shuffleServerId, partitionLengths, mapId)
+ mapStatus =
+ MapStatus(blockManager.shuffleServerId, partitionLengths, mapId, getAggregatedChecksumValue)
}
/** Close this writer, passing along whether the map completed */
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index 5c93bf4bf77a0..52856427cb37a 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -26,7 +26,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark._
import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.CPUS_PER_TASK
import org.apache.spark.internal.config.Status._
import org.apache.spark.resource.ResourceProfile.CPUS
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 87f876467c30e..a7f3fde9e6f6f 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -25,7 +25,7 @@ import scala.collection.mutable.HashMap
import scala.jdk.CollectionConverters._
import org.apache.spark.{JobExecutionStatus, SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PATH
import org.apache.spark.internal.config.Status.LIVE_UI_LOCAL_STORE_DIR
import org.apache.spark.status.AppStatusUtils.getQuantilesValue
diff --git a/core/src/main/scala/org/apache/spark/status/KVUtils.scala b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
index e334626413dc0..76fb654f8da2d 100644
--- a/core/src/main/scala/org/apache/spark/status/KVUtils.scala
+++ b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
@@ -31,7 +31,7 @@ import org.rocksdb.RocksDBException
import org.apache.spark.SparkConf
import org.apache.spark.deploy.history.{FsHistoryProvider, FsHistoryProviderMetadata}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.History
import org.apache.spark.internal.config.History.HYBRID_STORE_DISK_BACKEND
@@ -213,6 +213,20 @@ private[spark] object KVUtils extends Logging {
}
}
+ /**
+ * Maps all values of KVStoreView to new values using a transformation function
+ * and filtered by a filter function.
+ */
+ def mapToSeqWithFilter[T, B](
+ view: KVStoreView[T],
+ max: Int)
+ (mapFunc: T => B)
+ (filterFunc: B => Boolean): Seq[B] = {
+ Utils.tryWithResource(view.closeableIterator()) { iter =>
+ iter.asScala.map(mapFunc).filter(filterFunc).take(max).toList
+ }
+ }
+
def size[T](view: KVStoreView[T]): Int = {
Utils.tryWithResource(view.closeableIterator()) { iter =>
iter.asScala.size
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
index 66fac8a9d105a..61e800844db09 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
@@ -82,6 +82,10 @@ private[spark] trait UIRoot {
def withSparkUI[T](appId: String, attemptId: Option[String])(fn: SparkUI => T): T
def getApplicationInfoList: Iterator[ApplicationInfo]
+
+ def getApplicationInfoList(max: Int)(
+ filter: ApplicationInfo => Boolean): Iterator[ApplicationInfo]
+
def getApplicationInfo(appId: String): Option[ApplicationInfo]
/**
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
index 2d5dd97b501f1..aaaa08b3340b9 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
@@ -38,7 +38,7 @@ private[v1] class ApplicationListResource extends ApiRequestContext {
val includeCompleted = status.isEmpty || status.contains(ApplicationStatus.COMPLETED)
val includeRunning = status.isEmpty || status.contains(ApplicationStatus.RUNNING)
- uiRoot.getApplicationInfoList.filter { app =>
+ uiRoot.getApplicationInfoList(numApps) { app =>
val anyRunning = app.attempts.isEmpty || !app.attempts.head.completed
// if any attempt is still running, we consider the app to also still be running;
// keep the app if *any* attempts fall in the right time window
@@ -46,7 +46,7 @@ private[v1] class ApplicationListResource extends ApiRequestContext {
app.attempts.exists { attempt =>
isAttemptInRange(attempt, minDate, maxDate, minEndDate, maxEndDate, anyRunning)
}
- }.take(numApps)
+ }
}
private def isAttemptInRange(
diff --git a/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala b/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
index d83cff5f23a27..26e5068d15325 100644
--- a/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
@@ -21,12 +21,11 @@ import java.util.Date
import scala.jdk.CollectionConverters._
-import org.apache.commons.collections4.MapUtils
-
import org.apache.spark.status.StageDataWrapper
import org.apache.spark.status.api.v1.{ExecutorMetricsDistributions, ExecutorPeakMetricsDistributions, InputMetricDistributions, InputMetrics, OutputMetricDistributions, OutputMetrics, ShufflePushReadMetricDistributions, ShufflePushReadMetrics, ShuffleReadMetricDistributions, ShuffleReadMetrics, ShuffleWriteMetricDistributions, ShuffleWriteMetrics, SpeculationStageSummary, StageData, TaskData, TaskMetricDistributions, TaskMetrics}
import org.apache.spark.status.protobuf.Utils._
import org.apache.spark.util.Utils.weakIntern
+import org.apache.spark.util.collection.Utils.isNotEmpty
private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDataWrapper] {
@@ -397,11 +396,11 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
val failureReason = getOptional(binary.hasFailureReason, binary.getFailureReason)
val description = getOptional(binary.hasDescription, binary.getDescription)
val accumulatorUpdates = AccumulableInfoSerializer.deserialize(binary.getAccumulatorUpdatesList)
- val tasks = if (MapUtils.isNotEmpty(binary.getTasksMap)) {
+ val tasks = if (isNotEmpty(binary.getTasksMap)) {
Some(binary.getTasksMap.asScala.map(
entry => (entry._1.toLong, deserializeTaskData(entry._2))).toMap)
} else None
- val executorSummary = if (MapUtils.isNotEmpty(binary.getExecutorSummaryMap)) {
+ val executorSummary = if (isNotEmpty(binary.getExecutorSummaryMap)) {
Some(binary.getExecutorSummaryMap.asScala.toMap
.transform((_, v) => ExecutorStageSummarySerializer.deserialize(v)))
} else None
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockId.scala b/core/src/main/scala/org/apache/spark/storage/BlockId.scala
index 6eb015d56b2c7..a15426783ebec 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockId.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockId.scala
@@ -23,6 +23,7 @@ import org.apache.spark.SparkException
import org.apache.spark.annotation.{DeveloperApi, Since}
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.network.shuffle.RemoteBlockPushResolver
+import org.apache.spark.storage.LogBlockType.LogBlockType
/**
* :: DeveloperApi ::
@@ -175,6 +176,42 @@ case class PythonStreamBlockId(streamId: Int, uniqueId: Long) extends BlockId {
override def name: String = "python-stream-" + streamId + "-" + uniqueId
}
+object LogBlockType extends Enumeration {
+ type LogBlockType = Value
+ val TEST = Value
+}
+
+/**
+ * Identifies a block of log data.
+ *
+ * @param lastLogTime the timestamp of the last log entry in this block, used for filtering
+ * and log management.
+ * @param executorId the ID of the executor that produced this log block.
+ */
+abstract sealed class LogBlockId(
+ val lastLogTime: Long,
+ val executorId: String) extends BlockId {
+ def logBlockType: LogBlockType
+}
+
+object LogBlockId {
+ def empty(logBlockType: LogBlockType): LogBlockId = {
+ logBlockType match {
+ case LogBlockType.TEST => TestLogBlockId(0L, "")
+ case _ => throw new SparkException(s"Unsupported log block type: $logBlockType")
+ }
+ }
+}
+
+// Used for test purpose only.
+case class TestLogBlockId(override val lastLogTime: Long, override val executorId: String)
+ extends LogBlockId(lastLogTime, executorId) {
+ override def name: String =
+ "test_log_" + lastLogTime + "_" + executorId
+
+ override def logBlockType: LogBlockType = LogBlockType.TEST
+}
+
/** Id associated with temporary local data managed as blocks. Not serializable. */
private[spark] case class TempLocalBlockId(id: UUID) extends BlockId {
override def name: String = "temp_local_" + id
@@ -222,6 +259,7 @@ object BlockId {
val TEMP_LOCAL = "temp_local_([-A-Fa-f0-9]+)".r
val TEMP_SHUFFLE = "temp_shuffle_([-A-Fa-f0-9]+)".r
val TEST = "test_(.*)".r
+ val TEST_LOG_BLOCK = "test_log_([0-9]+)_(.*)".r
def apply(name: String): BlockId = name match {
case RDD(rddId, splitIndex) =>
@@ -262,6 +300,8 @@ object BlockId {
TempLocalBlockId(UUID.fromString(uuid))
case TEMP_SHUFFLE(uuid) =>
TempShuffleBlockId(UUID.fromString(uuid))
+ case TEST_LOG_BLOCK(lastLogTime, executorId) =>
+ TestLogBlockId(lastLogTime.toLong, executorId)
case TEST(value) =>
TestBlockId(value)
case _ => throw SparkCoreErrors.unrecognizedBlockIdError(name)
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
index d99bc5bf30546..602e8d068fb61 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
@@ -36,12 +36,11 @@ import scala.util.control.NonFatal
import com.codahale.metrics.{MetricRegistry, MetricSet}
import com.esotericsoftware.kryo.KryoException
import com.google.common.cache.CacheBuilder
-import org.apache.commons.io.IOUtils
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.{DataReadMethod, ExecutorExitCode}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{Network, RDD_CACHE_VISIBILITY_TRACKING_ENABLED, Tests}
import org.apache.spark.memory.{MemoryManager, MemoryMode}
@@ -59,6 +58,7 @@ import org.apache.spark.scheduler.ExecutorCacheTaskLocation
import org.apache.spark.serializer.{SerializerInstance, SerializerManager}
import org.apache.spark.shuffle.{IndexShuffleBlockResolver, MigratableResolver, ShuffleManager, ShuffleWriteMetricsReporter}
import org.apache.spark.storage.BlockManagerMessages.{DecommissionBlockManager, ReplicateBlock}
+import org.apache.spark.storage.LogBlockType.LogBlockType
import org.apache.spark.storage.memory._
import org.apache.spark.unsafe.Platform
import org.apache.spark.util._
@@ -295,12 +295,17 @@ private[spark] class BlockManager(
decommissioner.isDefined
}
- @inline final private def checkShouldStore(blockId: BlockId) = {
+ @inline final private def checkShouldStore(blockId: BlockId, level: StorageLevel) = {
// Don't reject broadcast blocks since they may be stored during task exec and
// don't need to be migrated.
if (isDecommissioning() && !blockId.isBroadcast) {
throw SparkCoreErrors.cannotSaveBlockOnDecommissionedExecutorError(blockId)
}
+ if (blockId.isInstanceOf[LogBlockId] && level != StorageLevel.DISK_ONLY) {
+ throw SparkException.internalError(
+ s"Cannot store log block $blockId with storage level $level. " +
+ "Log blocks must be stored with DISK_ONLY.")
+ }
}
// This is a lazy val so someone can migrating RDDs even if they don't have a MigratableResolver
@@ -373,7 +378,7 @@ private[spark] class BlockManager(
logInfo(extendMessageWithBlockDetails(ex.getMessage, blockId))
throw ex
} finally {
- IOUtils.closeQuietly(inputStream)
+ Utils.closeQuietly(inputStream)
}
}
@@ -764,7 +769,7 @@ private[spark] class BlockManager(
level: StorageLevel,
classTag: ClassTag[_]): StreamCallbackWithID = {
- checkShouldStore(blockId)
+ checkShouldStore(blockId, level)
if (blockId.isShuffle) {
logDebug(s"Putting shuffle block ${blockId}")
@@ -1484,6 +1489,26 @@ private[spark] class BlockManager(
syncWrites, writeMetrics, blockId)
}
+ /**
+ * To get a log block writer that can write logs directly to a disk block. Either `save` or
+ * `close` should be called to finish the writing and release opened resources.
+ * `save` would write the block to the block manager, while `close` would just close the writer.
+ */
+ def getLogBlockWriter(
+ logBlockType: LogBlockType): LogBlockWriter = {
+ new LogBlockWriter(this, logBlockType, conf)
+ }
+
+ /**
+ * To get a rolling log writer that can write logs to block manager and split the logs
+ * to multiple blocks if the log size exceeds the threshold.
+ */
+ def getRollingLogWriter(
+ blockIdGenerator: LogBlockIdGenerator,
+ rollingSize: Long = 33554432L): RollingLogWriter = {
+ new RollingLogWriter(this, blockIdGenerator, rollingSize)
+ }
+
/**
* Put a new block of serialized bytes to the block manager.
*
@@ -1541,7 +1566,7 @@ private[spark] class BlockManager(
require(blockId != null, "BlockId is null")
require(level != null && level.isValid, "StorageLevel is null or invalid")
- checkShouldStore(blockId)
+ checkShouldStore(blockId, level)
val putBlockInfo = {
val newInfo = new BlockInfo(level, classTag, tellMaster)
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala
index 19807453ee28c..1a067fab04c1d 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala
@@ -26,7 +26,7 @@ import scala.util.control.NonFatal
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.shuffle.ShuffleBlockInfo
import org.apache.spark.storage.BlockManagerMessages.ReplicateBlock
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
index 276bd63e14237..e77b4c2785745 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
@@ -23,7 +23,7 @@ import scala.concurrent.Future
import org.apache.spark.SparkConf
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rpc.RpcEndpointRef
import org.apache.spark.storage.BlockManagerMessages._
@@ -167,11 +167,12 @@ class BlockManagerMaster(
/**
* Remove the host from the candidate list of shuffle push mergers. This can be
- * triggered if there is a FetchFailedException on the host
+ * triggered if there is a FetchFailedException on the host. Non-blocking.
* @param host
*/
def removeShufflePushMergerLocation(host: String): Unit = {
- driverEndpoint.askSync[Unit](RemoveShufflePushMergerLocation(host))
+ logInfo(log"Request to remove shuffle push merger location ${MDC(HOST, host)}")
+ driverEndpoint.ask[Unit](RemoveShufflePushMergerLocation(host))
}
def getExecutorEndpointRef(executorId: String): Option[RpcEndpointRef] = {
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
index 858db498e83ad..9d6539e09f452 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
@@ -31,7 +31,7 @@ import com.google.common.cache.CacheBuilder
import org.apache.spark.{MapOutputTrackerMaster, SparkConf, SparkContext, SparkEnv}
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.RDD_CACHE_VISIBILITY_TRACKING_ENABLED
import org.apache.spark.network.shuffle.{ExternalBlockStoreClient, RemoteBlockPushResolver}
@@ -530,7 +530,13 @@ class BlockManagerMasterEndpoint(
blockManagerInfo.get(candidateBMId).foreach { bm =>
val remainingLocations = locations.toSeq.filter(bm => bm != candidateBMId)
val replicateMsg = ReplicateBlock(blockId, remainingLocations, maxReplicas)
- bm.storageEndpoint.ask[Boolean](replicateMsg)
+ try {
+ bm.storageEndpoint.ask[Boolean](replicateMsg)
+ } catch {
+ case e: Exception =>
+ logWarning(log"Failed to request replication of ${MDC(BLOCK_ID, blockId)} " +
+ log"from ${MDC(BLOCK_MANAGER_ID, candidateBMId)}", e)
+ }
}
}
}
@@ -554,6 +560,7 @@ class BlockManagerMasterEndpoint(
private def removeExecutor(execId: String): Unit = {
logInfo(log"Trying to remove executor ${MDC(EXECUTOR_ID, execId)} from BlockManagerMaster.")
blockManagerIdByExecutor.get(execId).foreach(removeBlockManager)
+ logInfo(log"Removed ${MDC(EXECUTOR_ID, execId)} successfully in removeExecutor")
}
/**
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala
index f29e8778da037..54329c5b1e514 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala
@@ -20,7 +20,7 @@ package org.apache.spark.storage
import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService, Future}
import org.apache.spark.{MapOutputTracker, SparkEnv}
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys.{BLOCK_ID, BROADCAST_ID, RDD_ID, SHUFFLE_ID}
import org.apache.spark.rpc.{IsolatedThreadSafeRpcEndpoint, RpcCallContext, RpcEnv}
import org.apache.spark.storage.BlockManagerMessages._
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala b/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
index 5186cbfa217cc..9126dc8a83561 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
import scala.util.Random
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
/**
diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
index 72d8dc0b19d21..df29ddf6999f5 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
@@ -30,7 +30,7 @@ import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.ExecutorExitCode
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.{MERGE_DIR_NAME, PATH}
import org.apache.spark.network.shuffle.ExecutorDiskUtils
import org.apache.spark.storage.DiskBlockManager.ATTEMPT_ID_KEY
@@ -316,7 +316,7 @@ private[spark] class DiskBlockManager(
throw SparkCoreErrors.failToCreateDirectoryError(dirToCreate.getAbsolutePath, maxAttempts)
}
try {
- dirToCreate.mkdirs()
+ Utils.createDirectory(dirToCreate)
Files.setPosixFilePermissions(
dirToCreate.toPath, PosixFilePermissions.fromString("rwxrwx---"))
if (dirToCreate.exists()) {
diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
index efcdb7fa8c69e..9964e64724f6b 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
@@ -24,7 +24,7 @@ import java.util.zip.Checksum
import org.apache.spark.SparkException
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.io.MutableCheckedOutputStream
import org.apache.spark.serializer.{SerializationStream, SerializerInstance, SerializerManager}
diff --git a/core/src/main/scala/org/apache/spark/storage/DiskStore.scala b/core/src/main/scala/org/apache/spark/storage/DiskStore.scala
index 3e57094b36a7e..b304d7bc32a84 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskStore.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskStore.scala
@@ -27,10 +27,9 @@ import scala.collection.mutable.ListBuffer
import com.google.common.io.Closeables
import io.netty.channel.DefaultFileRegion
-import org.apache.commons.io.FileUtils
import org.apache.spark.{SecurityManager, SparkConf, SparkException}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.buffer.ManagedBuffer
import org.apache.spark.network.util.{AbstractFileRegion, JavaUtils}
@@ -151,7 +150,7 @@ private[spark] class DiskStore(
blockSizes.put(targetBlockId, blockSize)
val targetFile = diskManager.getFile(targetBlockId.name)
logDebug(s"${sourceFile.getPath()} -> ${targetFile.getPath()}")
- FileUtils.moveFile(sourceFile, targetFile)
+ Utils.moveFile(sourceFile, targetFile)
}
def contains(blockId: BlockId): Boolean = diskManager.containsBlock(blockId)
diff --git a/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala b/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala
index 0f2bfaede4454..19cdebd80ebf9 100644
--- a/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala
+++ b/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP, STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH}
import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer}
@@ -36,6 +36,7 @@ import org.apache.spark.network.util.JavaUtils
import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef, RpcTimeout}
import org.apache.spark.shuffle.{IndexShuffleBlockResolver, ShuffleBlockInfo}
import org.apache.spark.shuffle.IndexShuffleBlockResolver.NOOP_REDUCE_ID
+import org.apache.spark.storage.BlockManagerMessages.RemoveShuffle
import org.apache.spark.util.Utils
/**
@@ -95,7 +96,8 @@ private[storage] class FallbackStorage(conf: SparkConf) extends Logging {
}
}
-private[storage] class NoopRpcEndpointRef(conf: SparkConf) extends RpcEndpointRef(conf) {
+private[storage] class FallbackStorageRpcEndpointRef(conf: SparkConf, hadoopConf: Configuration)
+ extends RpcEndpointRef(conf) {
// scalastyle:off executioncontextglobal
import scala.concurrent.ExecutionContext.Implicits.global
// scalastyle:on executioncontextglobal
@@ -103,6 +105,11 @@ private[storage] class NoopRpcEndpointRef(conf: SparkConf) extends RpcEndpointRe
override def name: String = "fallback"
override def send(message: Any): Unit = {}
override def ask[T: ClassTag](message: Any, timeout: RpcTimeout): Future[T] = {
+ message match {
+ case RemoveShuffle(shuffleId) =>
+ FallbackStorage.cleanUp(conf, hadoopConf, Some(shuffleId))
+ case _ => // no-op
+ }
Future{true.asInstanceOf[T]}
}
}
@@ -120,20 +127,25 @@ private[spark] object FallbackStorage extends Logging {
}
/** Register the fallback block manager and its RPC endpoint. */
- def registerBlockManagerIfNeeded(master: BlockManagerMaster, conf: SparkConf): Unit = {
+ def registerBlockManagerIfNeeded(
+ master: BlockManagerMaster,
+ conf: SparkConf,
+ hadoopConf: Configuration): Unit = {
if (conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).isDefined) {
master.registerBlockManager(
- FALLBACK_BLOCK_MANAGER_ID, Array.empty[String], 0, 0, new NoopRpcEndpointRef(conf))
+ FALLBACK_BLOCK_MANAGER_ID, Array.empty[String], 0, 0,
+ new FallbackStorageRpcEndpointRef(conf, hadoopConf))
}
}
- /** Clean up the generated fallback location for this app. */
- def cleanUp(conf: SparkConf, hadoopConf: Configuration): Unit = {
+ /** Clean up the generated fallback location for this app (and shuffle id if given). */
+ def cleanUp(conf: SparkConf, hadoopConf: Configuration, shuffleId: Option[Int] = None): Unit = {
if (conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).isDefined &&
conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP) &&
conf.contains("spark.app.id")) {
- val fallbackPath =
+ val fallbackPath = shuffleId.foldLeft(
new Path(conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH).get, conf.getAppId)
+ ) { case (path, shuffleId) => new Path(path, shuffleId.toString) }
val fallbackUri = fallbackPath.toUri
val fallbackFileSystem = FileSystem.get(fallbackUri, hadoopConf)
// The fallback directory for this app may not be created yet.
diff --git a/core/src/main/scala/org/apache/spark/storage/LogBlockIdGenerator.scala b/core/src/main/scala/org/apache/spark/storage/LogBlockIdGenerator.scala
new file mode 100644
index 0000000000000..4a2b90677ba3e
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/storage/LogBlockIdGenerator.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import org.apache.spark.SparkException
+import org.apache.spark.storage.LogBlockType.LogBlockType
+
+/**
+ * LogBlockIdGenerator is responsible for generating unique LogBlockIds for log blocks.
+ */
+trait LogBlockIdGenerator {
+ // The log block type that this generator supports.
+ def logBlockType: LogBlockType
+
+ // Generates a unique LogBlockId based on the last log time and executor ID.
+ protected def genUniqueBlockId(lastLogTime: Long, executorId: String): LogBlockId
+
+ /**
+ * Generates a new LogBlockId based on the last log time and executor ID. Make sure that
+ * the generated LogBlockId has the same log block type as this generator.
+ *
+ * @param lastLogTime The timestamp of the last log entry.
+ * @param executorId The ID of the executor generating the log block.
+ */
+ final def nextBlockId(lastLogTime: Long, executorId: String): LogBlockId = {
+ val blockId = genUniqueBlockId(lastLogTime, executorId)
+ if (blockId.logBlockType != this.logBlockType) {
+ throw SparkException.internalError(
+ "BlockId generated by LogBlockIdGenerator does not match " +
+ s"the expected log block type: ${blockId.logBlockType} != ${this.logBlockType}")
+ }
+ blockId
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/storage/LogBlockWriter.scala b/core/src/main/scala/org/apache/spark/storage/LogBlockWriter.scala
new file mode 100644
index 0000000000000..5645f59d383d6
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/storage/LogBlockWriter.scala
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import java.io.BufferedOutputStream
+import java.io.File
+import java.io.FileOutputStream
+
+import org.apache.commons.io.output.CountingOutputStream
+
+import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.internal.Logging
+import org.apache.spark.serializer.SerializationStream
+import org.apache.spark.storage.LogBlockType.LogBlockType
+import org.apache.spark.util.Utils
+
+/**
+ * A class for writing logs directly to a file on disk and save as a block in BlockManager if
+ * there are any logs written.
+ * `save` or `close` must be called to ensure resources are released properly. `save` will add
+ * the log block to BlockManager, while `close` will just release the resources without saving
+ * the log block.
+ *
+ * Notes:
+ * - This class does not support concurrent writes.
+ * - The writer will be automatically closed when failed to write logs or failed to save the
+ * log block.
+ * - Write operations after closing will throw exceptions.
+ */
+private[spark] class LogBlockWriter(
+ blockManager: BlockManager,
+ logBlockType: LogBlockType,
+ sparkConf: SparkConf,
+ bufferSize: Int = 32 * 1024) extends Logging {
+
+ private[storage] var tmpFile: File = null
+
+ private var cos: CountingOutputStream = null
+ private var objOut: SerializationStream = null
+ private var hasBeenClosed = false
+ private var recordsWritten = false
+ private var totalBytesWritten = 0
+
+ initialize()
+
+ private def initialize(): Unit = {
+ try {
+ val dir = new File(Utils.getLocalDir(sparkConf))
+ tmpFile = File.createTempFile(s"spark_log_$logBlockType", "", dir)
+ val fos = new FileOutputStream(tmpFile, false)
+ val bos = new BufferedOutputStream(fos, bufferSize)
+ cos = new CountingOutputStream(bos)
+ val emptyBlockId = LogBlockId.empty(logBlockType)
+ objOut = blockManager
+ .serializerManager
+ .blockSerializationStream(emptyBlockId, cos)(LogLine.getClassTag(logBlockType))
+ } catch {
+ case e: Exception =>
+ logError(log"Failed to initialize LogBlockWriter.", e)
+ close()
+ throw e
+ }
+ }
+
+ def bytesWritten(): Int = {
+ Option(cos)
+ .map(_.getCount)
+ .getOrElse(totalBytesWritten)
+ }
+
+ /**
+ * Write a log entry to the log block. Exception will be thrown if the writer has been closed
+ * or if there is an error during writing. Caller needs to deal with the exception. Suggest to
+ * close the writer when exception is thrown as block data could be corrupted which would lead
+ * to issues when reading the log block later.
+ *
+ * @param logEntry The log entry to write.
+ */
+ def writeLog(logEntry: LogLine): Unit = {
+ if (hasBeenClosed) {
+ throw SparkException.internalError(
+ "Writer already closed. Cannot write more data.",
+ category = "STORAGE"
+ )
+ }
+
+ try {
+ objOut.writeObject(logEntry)
+ recordsWritten = true
+ } catch {
+ case e: Exception =>
+ logError(log"Failed to write log entry.", e)
+ throw e
+ }
+ }
+
+ def save(blockId: LogBlockId): Unit = {
+ if (hasBeenClosed) {
+ throw SparkException.internalError(
+ "Writer already closed. Cannot save.",
+ category = "STORAGE"
+ )
+ }
+
+ try {
+ if (blockId.logBlockType != logBlockType) {
+ throw SparkException.internalError(
+ s"LogBlockWriter is for $logBlockType, but got blockId $blockId")
+ }
+
+ objOut.flush()
+ objOut.close()
+ objOut = null
+
+ if(recordsWritten) {
+ totalBytesWritten = cos.getCount
+ // Save log block to BlockManager and delete the tmpFile.
+ val success = saveToBlockManager(blockId, totalBytesWritten)
+ if (!success) {
+ throw SparkException.internalError(s"Failed to save log block $blockId to BlockManager")
+ }
+ }
+ } finally {
+ close()
+ }
+ }
+
+ def close(): Unit = {
+ if (hasBeenClosed) {
+ return
+ }
+
+ try {
+ if (objOut != null) {
+ objOut.close()
+ }
+ if (tmpFile != null && tmpFile.exists()) {
+ tmpFile.delete()
+ }
+ } catch {
+ case e: Exception =>
+ logWarning(log"Failed to close resources of LogBlockWriter", e)
+ } finally {
+ objOut = null
+ cos = null
+ hasBeenClosed = true
+ }
+ }
+
+ // For test only.
+ private[storage] def flush(): Unit = {
+ if (objOut != null) {
+ objOut.flush()
+ }
+ }
+
+ private[storage] def saveToBlockManager(blockId: LogBlockId, blockSize: Long): Boolean = {
+ blockManager.
+ TempFileBasedBlockStoreUpdater(
+ blockId,
+ StorageLevel.DISK_ONLY,
+ LogLine.getClassTag(logBlockType),
+ tmpFile,
+ blockSize)
+ .save()
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/storage/LogLine.scala b/core/src/main/scala/org/apache/spark/storage/LogLine.scala
new file mode 100644
index 0000000000000..dc646f289e37f
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/storage/LogLine.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import scala.reflect.ClassTag
+import scala.reflect.classTag
+
+/**
+ * Base class representing a log line.
+ *
+ * @param eventTime timestamp in milliseconds when the log is written
+ * @param sequenceId sequence ID of the log line
+ * @param message log message
+ */
+trait LogLine {
+ val eventTime: Long
+ val sequenceId: Long
+ val message: String
+}
+
+object LogLine {
+ def getClassTag(logBlockType: LogBlockType.LogBlockType): ClassTag[_<:LogLine] =
+ logBlockType match {
+ case LogBlockType.TEST =>
+ classTag[TestLogLine]
+ case unsupportedLogBlockType =>
+ throw new RuntimeException("Not supported log type " + unsupportedLogBlockType)
+ }
+}
+
+case class TestLogLine(eventTime: Long, sequenceId: Long, message: String)
+ extends LogLine {
+}
diff --git a/core/src/main/scala/org/apache/spark/storage/PushBasedFetchHelper.scala b/core/src/main/scala/org/apache/spark/storage/PushBasedFetchHelper.scala
index 8a3ca3066961c..16a68bbf50e25 100644
--- a/core/src/main/scala/org/apache/spark/storage/PushBasedFetchHelper.scala
+++ b/core/src/main/scala/org/apache/spark/storage/PushBasedFetchHelper.scala
@@ -28,7 +28,7 @@ import org.roaringbitmap.RoaringBitmap
import org.apache.spark.MapOutputTracker
import org.apache.spark.MapOutputTracker.SHUFFLE_PUSH_MAP_ID
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.shuffle.{BlockStoreClient, MergedBlockMeta, MergedBlocksMetaListener}
import org.apache.spark.shuffle.ShuffleReadMetricsReporter
diff --git a/core/src/main/scala/org/apache/spark/storage/RollingLogWriter.scala b/core/src/main/scala/org/apache/spark/storage/RollingLogWriter.scala
new file mode 100644
index 0000000000000..d925d7b79f464
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/storage/RollingLogWriter.scala
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.internal.LogKeys.{BLOCK_ID, BYTE_SIZE}
+import org.apache.spark.storage.LogBlockType.LogBlockType
+
+/**
+ * Rolling log writer that writes log entries to blocks in a rolling manner. Here we split
+ * log blocks based on size limit.
+ *
+ * @param blockManager BlockManager to manage log blocks.
+ * @param blockIdGenerator BlockId generator to generate unique block IDs for log blocks.
+ * @param rollingSize Size limit for each log block. Default is 32MB (33554432 bytes).
+ */
+private[spark] class RollingLogWriter(
+ blockManager: BlockManager,
+ blockIdGenerator: LogBlockIdGenerator,
+ rollingSize: Long = 33554432L) extends Logging {
+ private var currentBlockWriter: Option[LogBlockWriter] = None
+ private var lastLogTime: Long = 0L
+ private val logBlockType: LogBlockType = blockIdGenerator.logBlockType
+
+ private def shouldRollOver: Boolean = {
+ currentBlockWriter match {
+ case Some(writer) => writer.bytesWritten() >= rollingSize
+ case None => false
+ }
+ }
+
+ /**
+ * Write a log entry. If the current block writer is empty, it will create a new one.
+ * If the current block exceeds the rolling size, it will roll over to a new block for
+ * the next log entry.
+ *
+ * @param logEntry log entry to write.
+ * @param removeBlockOnException if true, current log block will be deleted without saving to
+ * BlockManager. Otherwise, not action will be taken on current
+ * block which might be corrupted.
+ */
+ def writeLog(logEntry: LogLine, removeBlockOnException: Boolean = false): Unit = {
+ // Create a new log writer if it's empty
+ if (currentBlockWriter.isEmpty) {
+ currentBlockWriter = Some(blockManager.getLogBlockWriter(logBlockType))
+ }
+
+ try {
+ currentBlockWriter.foreach { writer =>
+ writer.writeLog(logEntry)
+ lastLogTime = logEntry.eventTime
+ }
+ } catch {
+ case e: Exception =>
+ if (removeBlockOnException) {
+ logError(log"Failed to write log, closing block without saving.", e)
+ currentBlockWriter.foreach(_.close())
+ currentBlockWriter = None
+ }
+
+ throw e
+ }
+
+ if (shouldRollOver) {
+ rollOver()
+ }
+ }
+
+ def rollOver(): Unit = {
+ // Save current block and reset the writer
+ try {
+ saveCurrentBlock()
+ } finally {
+ currentBlockWriter = None
+ }
+ }
+
+ def close(): Unit = {
+ try {
+ saveCurrentBlock()
+ } finally {
+ currentBlockWriter = None
+ lastLogTime = 0L
+ }
+ }
+
+ // For test purpose.
+ private[storage] def flush(): Unit = {
+ currentBlockWriter.foreach(_.flush())
+ }
+
+ private def saveCurrentBlock(): Unit = {
+ currentBlockWriter.foreach { writer =>
+ val blockId = blockIdGenerator.nextBlockId(lastLogTime, blockManager.executorId)
+ logInfo(log"Saving log block ${MDC(BLOCK_ID, blockId)} with " +
+ log"approximate size: ${MDC(BYTE_SIZE, writer.bytesWritten())} bytes.")
+ writer.save(blockId)
+ }
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
index 57f6901a7a735..b2f185bc590fd 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -19,7 +19,7 @@ package org.apache.spark.storage
import java.io.{InputStream, IOException}
import java.nio.channels.ClosedByInterruptException
-import java.util.concurrent.{LinkedBlockingQueue, TimeUnit}
+import java.util.concurrent.{LinkedBlockingDeque, TimeUnit}
import java.util.concurrent.atomic.AtomicBoolean
import java.util.zip.CheckedInputStream
import javax.annotation.concurrent.GuardedBy
@@ -30,13 +30,12 @@ import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Queue}
import scala.util.{Failure, Success}
import io.netty.util.internal.OutOfDirectMemoryError
-import org.apache.commons.io.IOUtils
import org.roaringbitmap.RoaringBitmap
import org.apache.spark.{MapOutputTracker, SparkException, TaskContext}
import org.apache.spark.MapOutputTracker.SHUFFLE_PUSH_MAP_ID
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer}
import org.apache.spark.network.shuffle._
@@ -132,7 +131,7 @@ final class ShuffleBlockFetcherIterator(
* A queue to hold our results. This turns the asynchronous model provided by
* [[org.apache.spark.network.BlockTransferService]] into a synchronous model (iterator).
*/
- private[this] val results = new LinkedBlockingQueue[FetchResult]
+ private[this] val results = new LinkedBlockingDeque[FetchResult]()
/**
* Current [[FetchResult]] being processed. We track this so we can release the current buffer
@@ -358,7 +357,7 @@ final class ShuffleBlockFetcherIterator(
results.put(FallbackOnPushMergedFailureResult(
block, address, infoMap(blockId)._1, remainingBlocks.isEmpty))
} else {
- results.put(FailureFetchResult(block, infoMap(blockId)._2, address, e))
+ results.putFirst(FailureFetchResult(block, infoMap(blockId)._2, address, e))
}
}
}
@@ -594,7 +593,8 @@ final class ShuffleBlockFetcherIterator(
log"Error occurred while fetching local blocks, ${MDC(ERROR, ce.getMessage)}")
case ex: Exception => logError("Error occurred while fetching local blocks", ex)
}
- results.put(FailureFetchResult(blockId, mapIndex, blockManager.blockManagerId, e))
+ results.putFirst(
+ FailureFetchResult(blockId, mapIndex, blockManager.blockManagerId, e))
return
}
}
@@ -615,7 +615,7 @@ final class ShuffleBlockFetcherIterator(
case e: Exception =>
// If we see an exception, stop immediately.
logError(s"Error occurred while fetching local blocks", e)
- results.put(FailureFetchResult(blockId, mapIndex, blockManagerId, e))
+ results.putFirst(FailureFetchResult(blockId, mapIndex, blockManagerId, e))
false
}
}
@@ -668,7 +668,7 @@ final class ShuffleBlockFetcherIterator(
val bmId = bmIds.head
val blockInfoSeq = hostLocalBlocksWithMissingDirs(bmId)
val (blockId, _, mapIndex) = blockInfoSeq.head
- results.put(FailureFetchResult(blockId, mapIndex, bmId, throwable))
+ results.putFirst(FailureFetchResult(blockId, mapIndex, bmId, throwable))
}
}
}
@@ -1407,7 +1407,7 @@ private class BufferReleasingInputStream(
val diagnosisResponse = checkedInOpt.map { checkedIn =>
iterator.diagnoseCorruption(checkedIn, address, blockId)
}
- IOUtils.closeQuietly(this)
+ Utils.closeQuietly(this)
// We'd never retry the block whatever the cause is since the block has been
// partially consumed by downstream RDDs.
iterator.throwFetchFailedException(blockId, mapIndex, address, e, diagnosisResponse)
diff --git a/core/src/main/scala/org/apache/spark/storage/TopologyMapper.scala b/core/src/main/scala/org/apache/spark/storage/TopologyMapper.scala
index f1dca53c7e3b1..45d3a4acb9ca9 100644
--- a/core/src/main/scala/org/apache/spark/storage/TopologyMapper.scala
+++ b/core/src/main/scala/org/apache/spark/storage/TopologyMapper.scala
@@ -19,7 +19,7 @@ package org.apache.spark.storage
import org.apache.spark.SparkConf
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
index 6746bbd490c42..3981006c7ffe6 100644
--- a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
+++ b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
@@ -27,10 +27,8 @@ import scala.jdk.CollectionConverters._
import scala.reflect.ClassTag
import scala.util.control.NonFatal
-import com.google.common.io.ByteStreams
-
import org.apache.spark.{SparkConf, SparkException, TaskContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{STORAGE_UNROLL_MEMORY_THRESHOLD, UNROLL_MEMORY_CHECK_PERIOD, UNROLL_MEMORY_GROWTH_FACTOR}
import org.apache.spark.memory.{MemoryManager, MemoryMode}
@@ -905,7 +903,7 @@ private[storage] class PartiallySerializedBlock[T](
// We want to close the output stream in order to free any resources associated with the
// serializer itself (such as Kryo's internal buffers). close() might cause data to be
// written, so redirect the output stream to discard that data.
- redirectableOutputStream.setOutputStream(ByteStreams.nullOutputStream())
+ redirectableOutputStream.setOutputStream(OutputStream.nullOutputStream())
serializationStream.close()
} finally {
discarded = true
@@ -923,7 +921,7 @@ private[storage] class PartiallySerializedBlock[T](
verifyNotConsumedAndNotDiscarded()
consumed = true
// `unrolled`'s underlying buffers will be freed once this input stream is fully read:
- ByteStreams.copy(unrolledBuffer.toInputStream(dispose = true), os)
+ unrolledBuffer.toInputStream(dispose = true).transferTo(os)
memoryStore.releaseUnrollMemoryForThisTask(memoryMode, unrollMemory)
redirectableOutputStream.setOutputStream(os)
while (rest.hasNext) {
diff --git a/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala b/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
index e95eeddbdace3..0ce0f69b20704 100644
--- a/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
+++ b/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
@@ -38,6 +38,8 @@ private[spark] class ConsoleProgressBar(sc: SparkContext) extends Logging {
private val updatePeriodMSec = sc.conf.get(UI_CONSOLE_PROGRESS_UPDATE_INTERVAL)
// Delay to show up a progress bar, in milliseconds
private val firstDelayMSec = 500L
+ // Get the stderr (which is console for spark-shell) before installing RedirectConsolePlugin
+ private val console = System.err
// The width of terminal
private val TerminalWidth = sys.env.getOrElse("COLUMNS", "80").toInt
@@ -92,7 +94,7 @@ private[spark] class ConsoleProgressBar(sc: SparkContext) extends Logging {
// only refresh if it's changed OR after 1 minute (or the ssh connection will be closed
// after idle some time)
if (bar != lastProgressBar || now - lastUpdateTime > 60 * 1000L) {
- System.err.print(s"$CR$bar$CR")
+ console.print(s"$CR$bar$CR")
lastUpdateTime = now
}
lastProgressBar = bar
@@ -103,7 +105,7 @@ private[spark] class ConsoleProgressBar(sc: SparkContext) extends Logging {
*/
private def clear(): Unit = {
if (!lastProgressBar.isEmpty) {
- System.err.printf(s"$CR${" " * TerminalWidth}$CR")
+ console.printf(s"$CR${" ".repeat(TerminalWidth)}$CR")
lastProgressBar = ""
}
}
diff --git a/core/src/main/scala/org/apache/spark/ui/DriverLogPage.scala b/core/src/main/scala/org/apache/spark/ui/DriverLogPage.scala
index 8b4eebc26b3ba..dca85b53178ad 100644
--- a/core/src/main/scala/org/apache/spark/ui/DriverLogPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/DriverLogPage.scala
@@ -21,7 +21,7 @@ import scala.xml.{Node, Unparsed}
import jakarta.servlet.http.HttpServletRequest
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{LOG_TYPE, PATH}
import org.apache.spark.internal.config.DRIVER_LOG_LOCAL_DIR
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 068f9b0b2cbcb..bd7848c764375 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -40,7 +40,7 @@ import org.json4s.JValue
import org.json4s.jackson.JsonMethods.{pretty, render}
import org.apache.spark.{SecurityManager, SparkConf, SSLOptions}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.UI._
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index b8d422c9d9fbb..a23764f98d49d 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -23,7 +23,7 @@ import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletRespons
import org.eclipse.jetty.servlet.ServletContextHandler
import org.apache.spark.{SecurityManager, SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLASS_NAME, WEB_URL}
import org.apache.spark.internal.config.DRIVER_LOG_LOCAL_DIR
import org.apache.spark.internal.config.UI._
@@ -201,6 +201,11 @@ private[spark] class SparkUI private (
))
}
+ override def getApplicationInfoList(max: Int)(
+ filter: ApplicationInfo => Boolean): Iterator[ApplicationInfo] = {
+ getApplicationInfoList.filter(filter).take(max)
+ }
+
def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
getApplicationInfoList.find(_.id == appId)
}
diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
index 60d4e5db99d7e..c0d0fb50ae36b 100644
--- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
@@ -29,7 +29,7 @@ import org.eclipse.jetty.servlet.{FilterHolder, FilterMapping, ServletContextHan
import org.json4s.JsonAST.{JNothing, JValue}
import org.apache.spark.{SecurityManager, SparkConf, SSLOptions}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.ui.JettyUtils._
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
index ef26a02a406a8..fa10c8937144c 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
@@ -73,6 +73,7 @@ private[ui] class ExecutorThreadDumpPage(
Updated at {UIUtils.formatDate(time)}
+ {threadDumpSummary(threadDump)}
{ if (flamegraphEnabled) {
drawExecutorFlamegraph(request, threadDump) }
else {
@@ -123,10 +124,8 @@ private[ui] class ExecutorThreadDumpPage(
}.getOrElse(Text("Error fetching thread dump"))
UIUtils.headerSparkPage(request, s"Thread dump for executor $executorId", content, parent)
- // scalastyle:on
}
- // scalastyle:off
private def drawExecutorFlamegraph(request: HttpServletRequest, thread: Array[ThreadStackTrace]): Seq[Node] = {
val js =
s"""
@@ -154,5 +153,32 @@ private[ui] class ExecutorThreadDumpPage(
}
- // scalastyle:off
+
+
+ private def threadDumpSummary(threadDump: Array[ThreadStackTrace]): Seq[Node] = {
+ val totalCount = threadDump.length
+
+
+
+
+
+ | Thread State | Count | Percentage |
+
+ {
+ threadDump.groupBy(_.threadState).map { case (state, threads) =>
+
+ | {state} |
+ {threads.length} |
+ {"%.2f%%".format(threads.length * 100.0 / totalCount)} |
+
+ }.toSeq
+ }
+
+
+
+ -
+ Submitted:
+ {StageDataUtil.getFormattedSubmissionTime(stageData)}
+
+ -
+ Duration
+ {StageDataUtil.getFormattedDuration(stageData)}
+
-
Resource Profile Id:
{stageData.resourceProfileId}
diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
index ee7f67233bbd5..cd057ed08c3c3 100644
--- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
+++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
@@ -25,7 +25,7 @@ import scala.xml.Utility
import org.apache.commons.text.StringEscapeUtils
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rdd.DeterministicLevel
import org.apache.spark.scheduler.StageInfo
diff --git a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
index 1745498456213..7e2047526da03 100644
--- a/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
+++ b/core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicLong
import org.apache.spark.{InternalAccumulator, SparkContext, TaskContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.scheduler.AccumulableInfo
import org.apache.spark.util.AccumulatorContext.internOption
diff --git a/core/src/main/scala/org/apache/spark/util/BlockingThreadPoolExecutorService.scala b/core/src/main/scala/org/apache/spark/util/BlockingThreadPoolExecutorService.scala
new file mode 100644
index 0000000000000..506a83f5590e8
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/util/BlockingThreadPoolExecutorService.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.util
+import java.util.concurrent._
+
+import com.google.common.util.concurrent.Futures
+
+// scalastyle:off
+/**
+ * This thread pool executor throttles the submission of new tasks by using a semaphore.
+ * Task submissions require permits, task completions release permits.
+ *
+ * NOTE: [[invoke*]] methods are not supported, you should either use the [[submit]] methods
+ * or the [[execute]] method.
+ *
+ * This is inspired by
+ *
+ * Apache S4 BlockingThreadPoolExecutorService
+ */
+// scalastyle:on
+private[spark] class BlockingThreadPoolExecutorService(
+ nThreads: Int, workQueueSize: Int, threadFactory: ThreadFactory)
+ extends ExecutorService {
+
+ private val permits = new Semaphore(nThreads + workQueueSize)
+
+ private val workQuque = new LinkedBlockingQueue[Runnable](nThreads + workQueueSize)
+
+ private val delegate = new ThreadPoolExecutor(
+ nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, workQuque, threadFactory)
+
+ override def shutdown(): Unit = delegate.shutdown()
+
+ override def shutdownNow(): util.List[Runnable] = delegate.shutdownNow()
+
+ override def isShutdown: Boolean = delegate.isShutdown
+
+ override def isTerminated: Boolean = delegate.isTerminated
+
+ override def awaitTermination(timeout: Long, unit: TimeUnit): Boolean =
+ delegate.awaitTermination(timeout, unit)
+
+ override def submit[T](task: Callable[T]): Future[T] = {
+ try permits.acquire() catch {
+ case e: InterruptedException =>
+ Thread.currentThread.interrupt()
+ return Futures.immediateFailedFuture(e)
+ }
+ delegate.submit(new CallableWithPermitRelease(task))
+ }
+
+ override def submit[T](task: Runnable, result: T): Future[T] = {
+ try permits.acquire() catch {
+ case e: InterruptedException =>
+ Thread.currentThread.interrupt()
+ return Futures.immediateFailedFuture(e)
+ }
+ delegate.submit(new RunnableWithPermitRelease(task), result)
+ }
+
+ override def submit(task: Runnable): Future[_] = {
+ try permits.acquire() catch {
+ case e: InterruptedException =>
+ Thread.currentThread.interrupt()
+ return Futures.immediateFailedFuture(e)
+ }
+ delegate.submit(new RunnableWithPermitRelease(task))
+ }
+
+ override def execute(command: Runnable): Unit = {
+ try permits.acquire() catch {
+ case _: InterruptedException =>
+ Thread.currentThread.interrupt()
+ }
+ delegate.execute(new RunnableWithPermitRelease(command))
+ }
+
+ override def invokeAll[T](
+ tasks: util.Collection[_ <: Callable[T]]): util.List[Future[T]] =
+ throw new UnsupportedOperationException("Not implemented")
+
+ override def invokeAll[T](
+ tasks: util.Collection[_ <: Callable[T]],
+ timeout: Long, unit: TimeUnit): util.List[Future[T]] =
+ throw new UnsupportedOperationException("Not implemented")
+
+ override def invokeAny[T](tasks: util.Collection[_ <: Callable[T]]): T =
+ throw new UnsupportedOperationException("Not implemented")
+
+ override def invokeAny[T](
+ tasks: util.Collection[_ <: Callable[T]], timeout: Long, unit: TimeUnit): T =
+ throw new UnsupportedOperationException("Not implemented")
+
+ /**
+ * Releases a permit after the task is executed.
+ */
+ private class RunnableWithPermitRelease(delegate: Runnable) extends Runnable {
+ override def run(): Unit = try delegate.run() finally permits.release()
+ }
+
+ /**
+ * Releases a permit after the task is completed.
+ */
+ private class CallableWithPermitRelease[T](delegate: Callable[T]) extends Callable[T] {
+ override def call(): T = try delegate.call() finally permits.release()
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/util/CommandLineUtils.scala b/core/src/main/scala/org/apache/spark/util/CommandLineUtils.scala
index 651993b057ab0..debb268ad4847 100644
--- a/core/src/main/scala/org/apache/spark/util/CommandLineUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/CommandLineUtils.scala
@@ -30,7 +30,11 @@ private[spark] trait CommandLineUtils extends CommandLineLoggingUtils {
private[spark] trait CommandLineLoggingUtils {
// Exposed for testing
- private[spark] var exitFn: Int => Unit = (exitCode: Int) => System.exit(exitCode)
+ private[spark] var exitFn: (Int, Option[Throwable]) => Unit =
+ (exitCode: Int, cause: Option[Throwable]) => {
+ cause.foreach(_.printStackTrace(printStream))
+ System.exit(exitCode)
+ }
private[spark] var printStream: PrintStream = System.err
@@ -41,6 +45,6 @@ private[spark] trait CommandLineLoggingUtils {
private[spark] def printErrorAndExit(str: String): Unit = {
printMessage("Error: " + str)
printMessage("Run with --help for usage help or --verbose for debug output")
- exitFn(1)
+ exitFn(1, None)
}
}
diff --git a/core/src/main/scala/org/apache/spark/util/DependencyUtils.scala b/core/src/main/scala/org/apache/spark/util/DependencyUtils.scala
index 8526a21254586..63caa14215774 100644
--- a/core/src/main/scala/org/apache/spark/util/DependencyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/DependencyUtils.scala
@@ -20,13 +20,12 @@ package org.apache.spark.util
import java.io.{File, PrintStream}
import java.net.URI
-import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.deploy.SparkSubmit
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
@@ -83,7 +82,7 @@ private[spark] object DependencyUtils extends Logging {
val (transitive, exclusionList, repos) = MavenUtils.parseQueryParams(uri)
val fullReposList = Seq(ivyProperties.repositories, repos)
- .filter(!StringUtils.isBlank(_))
+ .filter(!SparkStringUtils.isBlank(_))
.mkString(",")
resolveMavenDependencies(
transitive,
@@ -103,7 +102,7 @@ private[spark] object DependencyUtils extends Logging {
ivyRepoPath: String,
ivySettingsPath: Option[String]): Seq[String] = {
val exclusions: Seq[String] =
- if (!StringUtils.isBlank(packagesExclusions)) {
+ if (!SparkStringUtils.isBlank(packagesExclusions)) {
packagesExclusions.split(",").toImmutableArraySeq
} else {
Nil
@@ -237,7 +236,7 @@ private[spark] object DependencyUtils extends Logging {
* no files, into a single comma-separated string.
*/
def mergeFileLists(lists: String*): String = {
- val merged = lists.filterNot(StringUtils.isBlank)
+ val merged = lists.filterNot(SparkStringUtils.isBlank)
.flatMap(Utils.stringToSeq)
if (merged.nonEmpty) merged.mkString(",") else null
}
diff --git a/core/src/main/scala/org/apache/spark/util/EventLoop.scala b/core/src/main/scala/org/apache/spark/util/EventLoop.scala
index b9de661b63c4f..c15598a35651f 100644
--- a/core/src/main/scala/org/apache/spark/util/EventLoop.scala
+++ b/core/src/main/scala/org/apache/spark/util/EventLoop.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicBoolean
import scala.util.control.NonFatal
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.EVENT_LOOP
/**
diff --git a/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala b/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala
index 5e50361b278aa..f84184f7c1faf 100644
--- a/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.viewfs.ViewFileSystem
import org.apache.hadoop.hdfs.DistributedFileSystem
import org.apache.spark._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.metrics.source.HiveCatalogMetrics
import org.apache.spark.util.ArrayImplicits._
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index d4b9b3c2ef3a7..d2e684faf9edd 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -38,6 +38,16 @@ import org.apache.spark.storage._
import org.apache.spark.util.ArrayImplicits._
import org.apache.spark.util.Utils.weakIntern
+/**
+ * Helper class for passing configuration options to JsonProtocol.
+ * We use this instead of passing SparkConf directly because it lets us avoid
+ * repeated re-parsing of configuration values on each read.
+ */
+private[spark] class JsonProtocolOptions(conf: SparkConf) {
+ val includeTaskMetricsAccumulators: Boolean =
+ conf.get(EVENT_LOG_INCLUDE_TASK_METRICS_ACCUMULATORS)
+}
+
/**
* Serializes SparkListener events to/from JSON. This protocol provides strong backwards-
* and forwards-compatibility guarantees: any version of Spark should be able to read JSON output
@@ -53,45 +63,49 @@ import org.apache.spark.util.Utils.weakIntern
* - Any new JSON fields should be optional; use `jsonOption` when reading these fields
* in `*FromJson` methods.
*/
-private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
+private[spark] object JsonProtocol extends JsonUtils {
// TODO: Remove this file and put JSON serialization into each individual class.
- // SPARK-49872 remove limit on string lengths
+ // SPARK-49872: Remove jackson JSON string length limitation.
mapper.getFactory.setStreamReadConstraints(
- StreamReadConstraints.builder().maxStringLength(
- sparkConf.get(EVENT_LOG_READER_MAX_STRING_LENGTH)
- ).build()
+ StreamReadConstraints.builder().maxStringLength(Int.MaxValue).build()
)
- val includeTaskMetricsAccumulators: Boolean =
- sparkConf.get(EVENT_LOG_INCLUDE_TASK_METRICS_ACCUMULATORS)
+ private[util]
+ val defaultOptions: JsonProtocolOptions = new JsonProtocolOptions(new SparkConf(false))
/** ------------------------------------------------- *
* JSON serialization methods for SparkListenerEvents |
* -------------------------------------------------- */
+ // Only for use in tests. Production code should use the two-argument overload defined below.
def sparkEventToJsonString(event: SparkListenerEvent): String = {
+ sparkEventToJsonString(event, defaultOptions)
+ }
+
+ def sparkEventToJsonString(event: SparkListenerEvent, options: JsonProtocolOptions): String = {
toJsonString { generator =>
- writeSparkEventToJson(event, generator)
+ writeSparkEventToJson(event, generator, options)
}
}
def writeSparkEventToJson(
event: SparkListenerEvent,
- g: JsonGenerator): Unit = {
+ g: JsonGenerator,
+ options: JsonProtocolOptions): Unit = {
event match {
case stageSubmitted: SparkListenerStageSubmitted =>
- stageSubmittedToJson(stageSubmitted, g)
+ stageSubmittedToJson(stageSubmitted, g, options)
case stageCompleted: SparkListenerStageCompleted =>
- stageCompletedToJson(stageCompleted, g)
+ stageCompletedToJson(stageCompleted, g, options)
case taskStart: SparkListenerTaskStart =>
- taskStartToJson(taskStart, g)
+ taskStartToJson(taskStart, g, options)
case taskGettingResult: SparkListenerTaskGettingResult =>
- taskGettingResultToJson(taskGettingResult, g)
+ taskGettingResultToJson(taskGettingResult, g, options)
case taskEnd: SparkListenerTaskEnd =>
- taskEndToJson(taskEnd, g)
+ taskEndToJson(taskEnd, g, options)
case jobStart: SparkListenerJobStart =>
- jobStartToJson(jobStart, g)
+ jobStartToJson(jobStart, g, options)
case jobEnd: SparkListenerJobEnd =>
jobEndToJson(jobEnd, g)
case environmentUpdate: SparkListenerEnvironmentUpdate =>
@@ -125,16 +139,15 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
}
}
- import JsonProtocol.SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES
-
def stageSubmittedToJson(
stageSubmitted: SparkListenerStageSubmitted,
- g: JsonGenerator): Unit = {
+ g: JsonGenerator,
+ options: JsonProtocolOptions): Unit = {
g.writeStartObject()
g.writeStringField("Event", SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES.stageSubmitted)
g.writeFieldName("Stage Info")
// SPARK-42205: don't log accumulables in start events:
- stageInfoToJson(stageSubmitted.stageInfo, g, includeAccumulables = false)
+ stageInfoToJson(stageSubmitted.stageInfo, g, options, includeAccumulables = false)
Option(stageSubmitted.properties).foreach { properties =>
g.writeFieldName("Properties")
propertiesToJson(properties, g)
@@ -144,42 +157,46 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
def stageCompletedToJson(
stageCompleted: SparkListenerStageCompleted,
- g: JsonGenerator): Unit = {
+ g: JsonGenerator,
+ options: JsonProtocolOptions): Unit = {
g.writeStartObject()
g.writeStringField("Event", SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES.stageCompleted)
g.writeFieldName("Stage Info")
- stageInfoToJson(stageCompleted.stageInfo, g, includeAccumulables = true)
+ stageInfoToJson(stageCompleted.stageInfo, g, options, includeAccumulables = true)
g.writeEndObject()
}
def taskStartToJson(
taskStart: SparkListenerTaskStart,
- g: JsonGenerator): Unit = {
+ g: JsonGenerator,
+ options: JsonProtocolOptions): Unit = {
g.writeStartObject()
g.writeStringField("Event", SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES.taskStart)
g.writeNumberField("Stage ID", taskStart.stageId)
g.writeNumberField("Stage Attempt ID", taskStart.stageAttemptId)
g.writeFieldName("Task Info")
// SPARK-42205: don't log accumulables in start events:
- taskInfoToJson(taskStart.taskInfo, g, includeAccumulables = false)
+ taskInfoToJson(taskStart.taskInfo, g, options, includeAccumulables = false)
g.writeEndObject()
}
def taskGettingResultToJson(
taskGettingResult: SparkListenerTaskGettingResult,
- g: JsonGenerator): Unit = {
+ g: JsonGenerator,
+ options: JsonProtocolOptions): Unit = {
val taskInfo = taskGettingResult.taskInfo
g.writeStartObject()
g.writeStringField("Event", SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES.taskGettingResult)
g.writeFieldName("Task Info")
// SPARK-42205: don't log accumulables in "task getting result" events:
- taskInfoToJson(taskInfo, g, includeAccumulables = false)
+ taskInfoToJson(taskInfo, g, options, includeAccumulables = false)
g.writeEndObject()
}
def taskEndToJson(
taskEnd: SparkListenerTaskEnd,
- g: JsonGenerator): Unit = {
+ g: JsonGenerator,
+ options: JsonProtocolOptions): Unit = {
g.writeStartObject()
g.writeStringField("Event", SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES.taskEnd)
g.writeNumberField("Stage ID", taskEnd.stageId)
@@ -188,7 +205,7 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
g.writeFieldName("Task End Reason")
taskEndReasonToJson(taskEnd.reason, g)
g.writeFieldName("Task Info")
- taskInfoToJson(taskEnd.taskInfo, g, includeAccumulables = true)
+ taskInfoToJson(taskEnd.taskInfo, g, options, includeAccumulables = true)
g.writeFieldName("Task Executor Metrics")
executorMetricsToJson(taskEnd.taskExecutorMetrics, g)
Option(taskEnd.taskMetrics).foreach { m =>
@@ -200,7 +217,8 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
def jobStartToJson(
jobStart: SparkListenerJobStart,
- g: JsonGenerator): Unit = {
+ g: JsonGenerator,
+ options: JsonProtocolOptions): Unit = {
g.writeStartObject()
g.writeStringField("Event", SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES.jobStart)
g.writeNumberField("Job ID", jobStart.jobId)
@@ -211,7 +229,7 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
// the job was submitted: it is technically possible for a stage to belong to multiple
// concurrent jobs, so this situation can arise even without races occurring between
// event logging and stage completion.
- jobStart.stageInfos.foreach(stageInfoToJson(_, g, includeAccumulables = true))
+ jobStart.stageInfos.foreach(stageInfoToJson(_, g, options, includeAccumulables = true))
g.writeEndArray()
g.writeArrayFieldStart("Stage IDs")
jobStart.stageIds.foreach(g.writeNumber)
@@ -410,6 +428,7 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
def stageInfoToJson(
stageInfo: StageInfo,
g: JsonGenerator,
+ options: JsonProtocolOptions,
includeAccumulables: Boolean): Unit = {
g.writeStartObject()
g.writeNumberField("Stage ID", stageInfo.stageId)
@@ -431,7 +450,7 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
accumulablesToJson(
stageInfo.accumulables.values,
g,
- includeTaskMetricsAccumulators = includeTaskMetricsAccumulators)
+ includeTaskMetricsAccumulators = options.includeTaskMetricsAccumulators)
} else {
g.writeStartArray()
g.writeEndArray()
@@ -445,6 +464,7 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
def taskInfoToJson(
taskInfo: TaskInfo,
g: JsonGenerator,
+ options: JsonProtocolOptions,
includeAccumulables: Boolean): Unit = {
g.writeStartObject()
g.writeNumberField("Task ID", taskInfo.taskId)
@@ -465,7 +485,7 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
accumulablesToJson(
taskInfo.accumulables,
g,
- includeTaskMetricsAccumulators = includeTaskMetricsAccumulators)
+ includeTaskMetricsAccumulators = options.includeTaskMetricsAccumulators)
} else {
g.writeStartArray()
g.writeEndArray()
@@ -891,6 +911,29 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
* JSON deserialization methods for SparkListenerEvents |
* ---------------------------------------------------- */
+ private object SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES {
+ val stageSubmitted = Utils.getFormattedClassName(SparkListenerStageSubmitted)
+ val stageCompleted = Utils.getFormattedClassName(SparkListenerStageCompleted)
+ val taskStart = Utils.getFormattedClassName(SparkListenerTaskStart)
+ val taskGettingResult = Utils.getFormattedClassName(SparkListenerTaskGettingResult)
+ val taskEnd = Utils.getFormattedClassName(SparkListenerTaskEnd)
+ val jobStart = Utils.getFormattedClassName(SparkListenerJobStart)
+ val jobEnd = Utils.getFormattedClassName(SparkListenerJobEnd)
+ val environmentUpdate = Utils.getFormattedClassName(SparkListenerEnvironmentUpdate)
+ val blockManagerAdded = Utils.getFormattedClassName(SparkListenerBlockManagerAdded)
+ val blockManagerRemoved = Utils.getFormattedClassName(SparkListenerBlockManagerRemoved)
+ val unpersistRDD = Utils.getFormattedClassName(SparkListenerUnpersistRDD)
+ val applicationStart = Utils.getFormattedClassName(SparkListenerApplicationStart)
+ val applicationEnd = Utils.getFormattedClassName(SparkListenerApplicationEnd)
+ val executorAdded = Utils.getFormattedClassName(SparkListenerExecutorAdded)
+ val executorRemoved = Utils.getFormattedClassName(SparkListenerExecutorRemoved)
+ val logStart = Utils.getFormattedClassName(SparkListenerLogStart)
+ val metricsUpdate = Utils.getFormattedClassName(SparkListenerExecutorMetricsUpdate)
+ val stageExecutorMetrics = Utils.getFormattedClassName(SparkListenerStageExecutorMetrics)
+ val blockUpdate = Utils.getFormattedClassName(SparkListenerBlockUpdated)
+ val resourceProfileAdded = Utils.getFormattedClassName(SparkListenerResourceProfileAdded)
+ }
+
def sparkEventFromJson(json: String): SparkListenerEvent = {
sparkEventFromJson(mapper.readTree(json))
}
@@ -919,8 +962,14 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
case `stageExecutorMetrics` => stageExecutorMetricsFromJson(json)
case `blockUpdate` => blockUpdateFromJson(json)
case `resourceProfileAdded` => resourceProfileAddedFromJson(json)
- case other => mapper.readValue(json.toString, Utils.classForName(other))
- .asInstanceOf[SparkListenerEvent]
+ case other =>
+ val otherClass = Utils.classForName(other)
+ if (classOf[SparkListenerEvent].isAssignableFrom(otherClass)) {
+ mapper.readValue(json.toString, otherClass)
+ .asInstanceOf[SparkListenerEvent]
+ } else {
+ throw new SparkException(s"Unknown event type: $other")
+ }
}
}
@@ -1298,12 +1347,12 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
jsonOption(readJson.get("Total Records Read")).map(_.extractLong).getOrElse(0L))
readMetrics.incRemoteReqsDuration(jsonOption(readJson.get("Remote Requests Duration"))
.map(_.extractLong).getOrElse(0L))
- jsonOption(readJson.get("Shuffle Push Read Metrics")).foreach { shufflePushReadJson =>
+ jsonOption(readJson.get("Push Based Shuffle")).foreach { shufflePushReadJson =>
readMetrics.incCorruptMergedBlockChunks(jsonOption(
shufflePushReadJson.get("Corrupt Merged Block Chunks"))
.map(_.extractLong).getOrElse(0L))
readMetrics.incMergedFetchFallbackCount(jsonOption(
- shufflePushReadJson.get("Merged Fallback Count")).map(_.extractLong).getOrElse(0L))
+ shufflePushReadJson.get("Merged Fetch Fallback Count")).map(_.extractLong).getOrElse(0L))
readMetrics.incRemoteMergedBlocksFetched(jsonOption(shufflePushReadJson
.get("Merged Remote Blocks Fetched")).map(_.extractLong).getOrElse(0L))
readMetrics.incLocalMergedBlocksFetched(jsonOption(shufflePushReadJson
@@ -1360,8 +1409,20 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
metrics
}
+ private object TASK_END_REASON_FORMATTED_CLASS_NAMES {
+ val success = Utils.getFormattedClassName(Success)
+ val resubmitted = Utils.getFormattedClassName(Resubmitted)
+ val fetchFailed = Utils.getFormattedClassName(FetchFailed)
+ val exceptionFailure = Utils.getFormattedClassName(ExceptionFailure)
+ val taskResultLost = Utils.getFormattedClassName(TaskResultLost)
+ val taskKilled = Utils.getFormattedClassName(TaskKilled)
+ val taskCommitDenied = Utils.getFormattedClassName(TaskCommitDenied)
+ val executorLostFailure = Utils.getFormattedClassName(ExecutorLostFailure)
+ val unknownReason = Utils.getFormattedClassName(UnknownReason)
+ }
+
def taskEndReasonFromJson(json: JsonNode): TaskEndReason = {
- import JsonProtocol.TASK_END_REASON_FORMATTED_CLASS_NAMES._
+ import TASK_END_REASON_FORMATTED_CLASS_NAMES._
json.get("Reason").extractString match {
case `success` => Success
@@ -1434,8 +1495,13 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
BlockManagerId(executorId, host, port)
}
+ private object JOB_RESULT_FORMATTED_CLASS_NAMES {
+ val jobSucceeded = Utils.getFormattedClassName(JobSucceeded)
+ val jobFailed = Utils.getFormattedClassName(JobFailed)
+ }
+
def jobResultFromJson(json: JsonNode): JobResult = {
- import JsonProtocol.JOB_RESULT_FORMATTED_CLASS_NAMES._
+ import JOB_RESULT_FORMATTED_CLASS_NAMES._
json.get("Result").extractString match {
case `jobSucceeded` => JobSucceeded
@@ -1637,45 +1703,3 @@ private[spark] class JsonProtocol(sparkConf: SparkConf) extends JsonUtils {
}
}
}
-
-private[spark] object JsonProtocol {
- private object JOB_RESULT_FORMATTED_CLASS_NAMES {
- val jobSucceeded = Utils.getFormattedClassName(JobSucceeded)
- val jobFailed = Utils.getFormattedClassName(JobFailed)
- }
-
- private object SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES {
- val stageSubmitted = Utils.getFormattedClassName(SparkListenerStageSubmitted)
- val stageCompleted = Utils.getFormattedClassName(SparkListenerStageCompleted)
- val taskStart = Utils.getFormattedClassName(SparkListenerTaskStart)
- val taskGettingResult = Utils.getFormattedClassName(SparkListenerTaskGettingResult)
- val taskEnd = Utils.getFormattedClassName(SparkListenerTaskEnd)
- val jobStart = Utils.getFormattedClassName(SparkListenerJobStart)
- val jobEnd = Utils.getFormattedClassName(SparkListenerJobEnd)
- val environmentUpdate = Utils.getFormattedClassName(SparkListenerEnvironmentUpdate)
- val blockManagerAdded = Utils.getFormattedClassName(SparkListenerBlockManagerAdded)
- val blockManagerRemoved = Utils.getFormattedClassName(SparkListenerBlockManagerRemoved)
- val unpersistRDD = Utils.getFormattedClassName(SparkListenerUnpersistRDD)
- val applicationStart = Utils.getFormattedClassName(SparkListenerApplicationStart)
- val applicationEnd = Utils.getFormattedClassName(SparkListenerApplicationEnd)
- val executorAdded = Utils.getFormattedClassName(SparkListenerExecutorAdded)
- val executorRemoved = Utils.getFormattedClassName(SparkListenerExecutorRemoved)
- val logStart = Utils.getFormattedClassName(SparkListenerLogStart)
- val metricsUpdate = Utils.getFormattedClassName(SparkListenerExecutorMetricsUpdate)
- val stageExecutorMetrics = Utils.getFormattedClassName(SparkListenerStageExecutorMetrics)
- val blockUpdate = Utils.getFormattedClassName(SparkListenerBlockUpdated)
- val resourceProfileAdded = Utils.getFormattedClassName(SparkListenerResourceProfileAdded)
- }
-
- private object TASK_END_REASON_FORMATTED_CLASS_NAMES {
- val success = Utils.getFormattedClassName(Success)
- val resubmitted = Utils.getFormattedClassName(Resubmitted)
- val fetchFailed = Utils.getFormattedClassName(FetchFailed)
- val exceptionFailure = Utils.getFormattedClassName(ExceptionFailure)
- val taskResultLost = Utils.getFormattedClassName(TaskResultLost)
- val taskKilled = Utils.getFormattedClassName(TaskKilled)
- val taskCommitDenied = Utils.getFormattedClassName(TaskCommitDenied)
- val executorLostFailure = Utils.getFormattedClassName(ExecutorLostFailure)
- val unknownReason = Utils.getFormattedClassName(UnknownReason)
- }
-}
diff --git a/core/src/main/scala/org/apache/spark/util/ListenerBus.scala b/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
index 4f01cd6ac2136..0c0c932ac797c 100644
--- a/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
@@ -26,7 +26,7 @@ import scala.util.control.NonFatal
import com.codahale.metrics.Timer
import org.apache.spark.SparkEnv
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.{EVENT, LISTENER, TOTAL_TIME}
import org.apache.spark.scheduler.EventLoggingListener
import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate
diff --git a/core/src/main/scala/org/apache/spark/util/PeriodicCheckpointer.scala b/core/src/main/scala/org/apache/spark/util/PeriodicCheckpointer.scala
index 7a98c4830db92..5c9c3d3564d00 100644
--- a/core/src/main/scala/org/apache/spark/util/PeriodicCheckpointer.scala
+++ b/core/src/main/scala/org/apache/spark/util/PeriodicCheckpointer.scala
@@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
diff --git a/core/src/main/scala/org/apache/spark/util/RpcUtils.scala b/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
index 30f5fced5a8bf..58fa4df3b72dc 100644
--- a/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/RpcUtils.scala
@@ -36,6 +36,15 @@ private[spark] object RpcUtils {
rpcEnv.setupEndpointRef(RpcAddress(driverHost, driverPort), name)
}
+ def makeDriverRef(
+ name: String,
+ driverHost: String,
+ driverPort: Int,
+ rpcEnv: RpcEnv): RpcEndpointRef = {
+ Utils.checkHost(driverHost)
+ rpcEnv.setupEndpointRef(RpcAddress(driverHost, driverPort), name)
+ }
+
/** Returns the default Spark timeout to use for RPC ask operations. */
def askRpcTimeout(conf: SparkConf): RpcTimeout = {
RpcTimeout(conf, Seq(RPC_ASK_TIMEOUT.key, NETWORK_TIMEOUT.key), "120s")
diff --git a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
index ad9c3d3f10cfe..af93f781343d2 100644
--- a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
+++ b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
@@ -26,7 +26,7 @@ import scala.util.Try
import org.apache.hadoop.fs.FileSystem
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.LogKeys.PATH
import org.apache.spark.internal.config.SPARK_SHUTDOWN_TIMEOUT_MS
diff --git a/core/src/main/scala/org/apache/spark/util/SignalUtils.scala b/core/src/main/scala/org/apache/spark/util/SignalUtils.scala
index b41166a50efd2..e2f615b44115b 100644
--- a/core/src/main/scala/org/apache/spark/util/SignalUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/SignalUtils.scala
@@ -21,11 +21,10 @@ import java.util.Collections
import scala.jdk.CollectionConverters._
-import org.apache.commons.lang3.SystemUtils
import org.slf4j.Logger
import sun.misc.{Signal, SignalHandler}
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys._
/**
@@ -58,7 +57,7 @@ private[spark] object SignalUtils extends Logging {
* All actions for a given signal are run in a separate thread.
*/
def register(signal: String)(action: => Boolean): Unit = {
- if (SystemUtils.IS_OS_UNIX) {
+ if (Utils.isUnix) {
register(signal, log"Failed to register signal handler for ${MDC(SIGNAL, signal)}",
logStackTrace = true)(action)
}
diff --git a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
index 88fe64859a214..12a417fee3a21 100644
--- a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
+++ b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala
@@ -29,6 +29,7 @@ import com.google.common.collect.MapMaker
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config.Tests.TEST_USE_COMPRESSED_OOPS_KEY
+import org.apache.spark.util.Utils
import org.apache.spark.util.collection.OpenHashSet
/**
@@ -106,7 +107,7 @@ object SizeEstimator extends Logging {
// Sets object size, pointer size based on architecture and CompressedOops settings
// from the JVM.
private def initialize(): Unit = {
- val arch = System.getProperty("os.arch")
+ val arch = Utils.osArch
is64bit = arch.contains("64") || arch.contains("s390x")
isCompressedOops = getIsCompressedOops
diff --git a/core/src/main/scala/org/apache/spark/util/SparkExitCode.scala b/core/src/main/scala/org/apache/spark/util/SparkExitCode.scala
index e8f8788243cd9..0ffc2afd96356 100644
--- a/core/src/main/scala/org/apache/spark/util/SparkExitCode.scala
+++ b/core/src/main/scala/org/apache/spark/util/SparkExitCode.scala
@@ -45,6 +45,9 @@ private[spark] object SparkExitCode {
OutOfMemoryError. */
val OOM = 52
+ /** Exit due to ClassNotFoundException or NoClassDefFoundError. */
+ val CLASS_NOT_FOUND = 101
+
/** Exit because the driver is running over the given threshold. */
val DRIVER_TIMEOUT = 124
diff --git a/core/src/main/scala/org/apache/spark/util/SparkUncaughtExceptionHandler.scala b/core/src/main/scala/org/apache/spark/util/SparkUncaughtExceptionHandler.scala
index 25a6322743baa..a1aeee1d877c7 100644
--- a/core/src/main/scala/org/apache/spark/util/SparkUncaughtExceptionHandler.scala
+++ b/core/src/main/scala/org/apache/spark/util/SparkUncaughtExceptionHandler.scala
@@ -19,7 +19,7 @@ package org.apache.spark.util
import org.apache.spark.SparkEnv
import org.apache.spark.executor.{ExecutorExitCode, KilledByTaskReaperException}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.THREAD
import org.apache.spark.internal.config.KILL_ON_FATAL_ERROR_DEPTH
diff --git a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
index e9d14f904db45..d22e14d992655 100644
--- a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
@@ -188,6 +188,23 @@ private[spark] object ThreadUtils {
rejectedExecutionHandler)
}
+ /**
+ * Simliar to newDaemonFixedThreadPool, but with a bound workQueue, task submission will
+ * be blocked when queue is full.
+ *
+ * @param nThreads the number of threads in the pool
+ * @param workQueueSize the capacity of the queue to use for holding tasks before they are
+ * executed. Task submission will be blocked when queue is full.
+ * @param prefix thread names are formatted as prefix-ID, where ID is a unique, sequentially
+ * assigned integer.
+ * @return BlockingThreadPoolExecutorService
+ */
+ def newDaemonBlockingThreadPoolExecutorService(
+ nThreads: Int, workQueueSize: Int, prefix: String): ExecutorService = {
+ val threadFactory = namedThreadFactory(prefix)
+ new BlockingThreadPoolExecutorService(nThreads, workQueueSize, threadFactory)
+ }
+
/**
* Wrapper over ScheduledThreadPoolExecutor the pool with daemon threads.
*/
diff --git a/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala b/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala
index 8fba5ed944c67..3d504856f5660 100644
--- a/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala
+++ b/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala
@@ -107,8 +107,11 @@ private[spark] class UninterruptibleThread(
// super.interrupt() is called. In this case to prevent runUninterruptibly() from being
// interrupted, we use awaitInterruptThread flag. We need to set it only if
// runUninterruptibly() is not yet set uninterruptible to true (!shouldInterruptThread) and
- // there is no other threads that called interrupt (awaitInterruptThread is already true)
- if (!shouldInterruptThread && !awaitInterruptThread) {
+ // there is no other threads that called interrupt (awaitInterruptThread is already true or
+ // isInterrupted is true. (SPARK-53394) Otherwise, the state of shouldInterruptThread would
+ // become inconsistent between isInterruptible() and isInterruptPending(), leading to
+ // UninterruptibleThread be interruptible under runUninterruptibly.)
+ if (!shouldInterruptThread && !awaitInterruptThread && !isInterrupted) {
awaitInterruptThread = true
true
} else {
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index ea9b742fb2e1b..98dd5dc1e16d9 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -28,7 +28,7 @@ import java.nio.channels.Channels
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.security.SecureRandom
-import java.util.{Locale, Properties, Random, UUID}
+import java.util.{HexFormat, Locale, Properties, Random, UUID}
import java.util.concurrent._
import java.util.concurrent.TimeUnit.NANOSECONDS
import java.util.zip.{GZIPInputStream, ZipInputStream}
@@ -46,12 +46,8 @@ import scala.util.matching.Regex
import _root_.io.netty.channel.unix.Errors.NativeIoException
import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
import com.google.common.collect.Interners
-import com.google.common.io.{ByteStreams, Files => GFiles}
import com.google.common.net.InetAddresses
import jakarta.ws.rs.core.UriBuilder
-import org.apache.commons.codec.binary.Hex
-import org.apache.commons.io.IOUtils
-import org.apache.commons.lang3.{JavaVersion, SystemUtils}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, FileUtil, Path}
import org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext
@@ -59,7 +55,7 @@ import org.apache.hadoop.io.compress.{CompressionCodecFactory, SplittableCompres
import org.apache.hadoop.ipc.{CallerContext => HadoopCallerContext}
import org.apache.hadoop.ipc.CallerContext.{Builder => HadoopCallerContextBuilder}
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.util.{RunJar, StringUtils}
+import org.apache.hadoop.util.RunJar
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.logging.log4j.{Level, LogManager}
import org.apache.logging.log4j.core.LoggerContext
@@ -67,9 +63,9 @@ import org.apache.logging.log4j.core.config.LoggerConfig
import org.eclipse.jetty.util.MultiException
import org.slf4j.Logger
-import org.apache.spark._
+import org.apache.spark.{SPARK_VERSION, _}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
@@ -105,7 +101,8 @@ private[spark] object Utils
with SparkFileUtils
with SparkSerDeUtils
with SparkStreamUtils
- with SparkStringUtils {
+ with SparkStringUtils
+ with SparkSystemUtils {
private val sparkUncaughtExceptionHandler = new SparkUncaughtExceptionHandler
@volatile private var cachedLocalDir: String = ""
@@ -351,7 +348,7 @@ private[spark] object Utils
def fetchFile(
url: String,
targetDir: File,
- conf: SparkConf,
+ conf: ReadOnlySparkConf,
hadoopConf: Configuration,
timestamp: Long,
useCache: Boolean,
@@ -435,7 +432,7 @@ private[spark] object Utils
if (!source.exists()) {
throw new FileNotFoundException(source.getAbsolutePath)
}
- val lowerSrc = StringUtils.toLowerCase(source.getName)
+ val lowerSrc = source.getName.toLowerCase(Locale.ENGLISH)
if (lowerSrc.endsWith(".jar")) {
RunJar.unJar(source, dest, RunJar.MATCH_ANY)
} else if (lowerSrc.endsWith(".zip")) {
@@ -458,7 +455,7 @@ private[spark] object Utils
* to work around a security issue, see also SPARK-38631.
*/
private def unTarUsingJava(source: File, dest: File): Unit = {
- if (!dest.mkdirs && !dest.isDirectory) {
+ if (!Utils.createDirectory(dest) && !dest.isDirectory) {
throw new IOException(s"Mkdirs failed to create $dest")
} else {
try {
@@ -594,7 +591,7 @@ private[spark] object Utils
case (f1, f2) => filesEqualRecursive(f1, f2)
}
} else if (file1.isFile && file2.isFile) {
- GFiles.equal(file1, file2)
+ contentEquals(file1, file2)
} else {
false
}
@@ -625,7 +622,7 @@ private[spark] object Utils
url: String,
targetDir: File,
filename: String,
- conf: SparkConf,
+ conf: ReadOnlySparkConf,
hadoopConf: Configuration): File = {
val targetFile = new File(targetDir, filename)
val uri = new URI(url)
@@ -672,7 +669,7 @@ private[spark] object Utils
path: Path,
targetDir: File,
fs: FileSystem,
- conf: SparkConf,
+ conf: ReadOnlySparkConf,
hadoopConf: Configuration,
fileOverwrite: Boolean,
filename: Option[String] = None): Unit = {
@@ -727,7 +724,7 @@ private[spark] object Utils
* always return a single directory. The return directory is chosen randomly from the array
* of directories it gets from getOrCreateLocalRootDirs.
*/
- def getLocalDir(conf: SparkConf): String = {
+ def getLocalDir(conf: ReadOnlySparkConf): String = {
val localRootDirs = getOrCreateLocalRootDirs(conf)
if (localRootDirs.isEmpty) {
val configuredLocalDirs = getConfiguredLocalDirs(conf)
@@ -738,7 +735,7 @@ private[spark] object Utils
}
}
- private[spark] def isRunningInYarnContainer(conf: SparkConf): Boolean = {
+ private[spark] def isRunningInYarnContainer(conf: ReadOnlySparkConf): Boolean = {
// These environment variables are set by YARN.
conf.getenv("CONTAINER_ID") != null
}
@@ -758,7 +755,7 @@ private[spark] object Utils
* So calling it multiple times with a different configuration will always return the same
* set of directories.
*/
- private[spark] def getOrCreateLocalRootDirs(conf: SparkConf): Array[String] = {
+ private[spark] def getOrCreateLocalRootDirs(conf: ReadOnlySparkConf): Array[String] = {
if (localRootDirs == null) {
this.synchronized {
if (localRootDirs == null) {
@@ -774,7 +771,7 @@ private[spark] object Utils
* method does not create any directories on its own, it only encapsulates the
* logic of locating the local directories according to deployment mode.
*/
- def getConfiguredLocalDirs(conf: SparkConf): Array[String] = {
+ def getConfiguredLocalDirs(conf: ReadOnlySparkConf): Array[String] = {
if (isRunningInYarnContainer(conf)) {
// If we are in yarn mode, systems can have different disk layouts so we must set it
// to what Yarn on this system said was available. Note this assumes that Yarn has
@@ -793,7 +790,7 @@ private[spark] object Utils
}
}
- private def getOrCreateLocalRootDirsImpl(conf: SparkConf): Array[String] = {
+ private def getOrCreateLocalRootDirsImpl(conf: ReadOnlySparkConf): Array[String] = {
val configuredLocalDirs = getConfiguredLocalDirs(conf)
val uris = configuredLocalDirs.filter { root =>
// Here, we guess if the given value is a URI at its best - check if scheme is set.
@@ -810,26 +807,25 @@ private[spark] object Utils
configuredLocalDirs.flatMap { root =>
try {
val rootDir = new File(root)
- if (rootDir.exists || rootDir.mkdirs()) {
+ if (rootDir.exists || Utils.createDirectory(rootDir)) {
val dir = createTempDir(root)
chmod700(dir)
Some(dir.getAbsolutePath)
} else {
logError(log"Failed to create dir in ${MDC(PATH, root)}. Ignoring this directory.")
-
None
}
} catch {
case e: IOException =>
logError(
- log"Failed to create local root dir in ${MDC(PATH, root)}. Ignoring this directory.")
+ log"Failed to create local root dir in ${MDC(PATH, root)}. Ignoring this directory.", e)
None
}
}
}
/** Get the Yarn approved local directories. */
- private def getYarnLocalDirs(conf: SparkConf): String = {
+ private def getYarnLocalDirs(conf: ReadOnlySparkConf): String = {
val localDirs = Option(conf.getenv("LOCAL_DIRS")).getOrElse("")
if (localDirs.isEmpty) {
@@ -1523,7 +1519,7 @@ private[spark] object Utils
private var compressedLogFileLengthCache: LoadingCache[String, java.lang.Long] = null
private def getCompressedLogFileLengthCache(
- sparkConf: SparkConf): LoadingCache[String, java.lang.Long] = this.synchronized {
+ sparkConf: ReadOnlySparkConf): LoadingCache[String, java.lang.Long] = this.synchronized {
if (compressedLogFileLengthCache == null) {
val compressedLogFileLengthCacheSize = sparkConf.get(
UNCOMPRESSED_LOG_FILE_LENGTH_CACHE_SIZE_CONF)
@@ -1543,7 +1539,7 @@ private[spark] object Utils
* It also caches the uncompressed file size to avoid repeated decompression. The cache size is
* read from workerConf.
*/
- def getFileLength(file: File, workConf: SparkConf): Long = {
+ def getFileLength(file: File, workConf: ReadOnlySparkConf): Long = {
if (file.getName.endsWith(".gz")) {
getCompressedLogFileLengthCache(workConf).get(file.getAbsolutePath)
} else {
@@ -1560,10 +1556,10 @@ private[spark] object Utils
gzInputStream = new GZIPInputStream(new FileInputStream(file))
val bufSize = 1024
val buf = new Array[Byte](bufSize)
- var numBytes = ByteStreams.read(gzInputStream, buf, 0, bufSize)
+ var numBytes = gzInputStream.readNBytes(buf, 0, bufSize)
while (numBytes > 0) {
fileSize += numBytes
- numBytes = ByteStreams.read(gzInputStream, buf, 0, bufSize)
+ numBytes = gzInputStream.readNBytes(buf, 0, bufSize)
}
fileSize
} catch {
@@ -1590,8 +1586,8 @@ private[spark] object Utils
}
try {
- ByteStreams.skipFully(stream, effectiveStart)
- ByteStreams.readFully(stream, buff)
+ stream.skipNBytes(effectiveStart)
+ readFully(stream, buff, 0, buff.length)
} finally {
stream.close()
}
@@ -1859,24 +1855,14 @@ private[spark] object Utils
}
/**
- * Whether the underlying operating system is Windows.
- */
- val isWindows = SystemUtils.IS_OS_WINDOWS
-
- /**
- * Whether the underlying operating system is Mac OS X.
+ * Whether the underlying Java version is at most 17.
*/
- val isMac = SystemUtils.IS_OS_MAC_OSX
+ val isJavaVersionAtMost17 = Runtime.version().feature() <= 17
/**
* Whether the underlying Java version is at least 21.
*/
- val isJavaVersionAtLeast21 = SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_21)
-
- /**
- * Whether the underlying operating system is Mac OS X and processor is Apple Silicon.
- */
- val isMacOnAppleSilicon = SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64")
+ val isJavaVersionAtLeast21 = Runtime.version().feature() >= 21
/**
* Whether the underlying JVM prefer IPv6 addresses.
@@ -2191,7 +2177,9 @@ private[spark] object Utils
/**
* Convert all spark properties set in the given SparkConf to a sequence of java options.
*/
- def sparkJavaOpts(conf: SparkConf, filterKey: (String => Boolean) = _ => true): Seq[String] = {
+ def sparkJavaOpts(
+ conf: ReadOnlySparkConf,
+ filterKey: (String => Boolean) = _ => true): Seq[String] = {
conf.getAll
.filter { case (k, _) => filterKey(k) }
.map { case (k, v) => s"-D$k=$v" }
@@ -2201,7 +2189,7 @@ private[spark] object Utils
/**
* Maximum number of retries when binding to a port before giving up.
*/
- def portMaxRetries(conf: SparkConf): Int = {
+ def portMaxRetries(conf: ReadOnlySparkConf): Int = {
val maxRetries = conf.getOption("spark.port.maxRetries").map(_.toInt)
if (conf.contains(IS_TESTING)) {
// Set a higher number of retries for tests...
@@ -2226,7 +2214,7 @@ private[spark] object Utils
def startServiceOnPort[T](
startPort: Int,
startService: Int => (T, Int),
- conf: SparkConf,
+ conf: ReadOnlySparkConf,
serviceName: String = ""): (T, Int) = {
startServiceOnPort(startPort, startService, portMaxRetries(conf), serviceName)
}
@@ -2457,7 +2445,7 @@ private[spark] object Utils
val EMPTY_USER_GROUPS = Set.empty[String]
// Returns the groups to which the current user belongs.
- def getCurrentUserGroups(sparkConf: SparkConf, username: String): Set[String] = {
+ def getCurrentUserGroups(sparkConf: ReadOnlySparkConf, username: String): Set[String] = {
val groupProviderClassName = sparkConf.get(USER_GROUPS_MAPPING)
if (groupProviderClassName != "") {
try {
@@ -2620,7 +2608,7 @@ private[spark] object Utils
(!isLocalMaster(conf) || conf.get(DYN_ALLOCATION_TESTING))
}
- def isStreamingDynamicAllocationEnabled(conf: SparkConf): Boolean = {
+ def isStreamingDynamicAllocationEnabled(conf: ReadOnlySparkConf): Boolean = {
val streamingDynamicAllocationEnabled = conf.get(STREAMING_DYN_ALLOCATION_ENABLED)
streamingDynamicAllocationEnabled &&
(!isLocalMaster(conf) || conf.get(STREAMING_DYN_ALLOCATION_TESTING))
@@ -2629,7 +2617,7 @@ private[spark] object Utils
/**
* Return the initial number of executors for dynamic allocation.
*/
- def getDynamicAllocationInitialExecutors(conf: SparkConf): Int = {
+ def getDynamicAllocationInitialExecutors(conf: ReadOnlySparkConf): Int = {
if (conf.get(DYN_ALLOCATION_INITIAL_EXECUTORS) < conf.get(DYN_ALLOCATION_MIN_EXECUTORS)) {
logWarning(log"${MDC(CONFIG, DYN_ALLOCATION_INITIAL_EXECUTORS.key)} less than " +
log"${MDC(CONFIG2, DYN_ALLOCATION_MIN_EXECUTORS.key)} is invalid, ignoring its setting, " +
@@ -2696,7 +2684,7 @@ private[spark] object Utils
* This is designed for a code path which logging system may be initilized before
* loading SparkConf.
*/
- def resetStructuredLogging(sparkConf: SparkConf): Unit = {
+ def resetStructuredLogging(sparkConf: ReadOnlySparkConf): Unit = {
if (sparkConf.get(STRUCTURED_LOGGING_ENABLED)) {
Logging.enableStructuredLogging()
} else {
@@ -2709,7 +2697,7 @@ private[spark] object Utils
* these jars through file server. In the YARN mode, it will return an empty list, since YARN
* has its own mechanism to distribute jars.
*/
- def getUserJars(conf: SparkConf): Seq[String] = {
+ def getUserJars(conf: ReadOnlySparkConf): Seq[String] = {
conf.get(JARS).filter(_.nonEmpty)
}
@@ -2718,7 +2706,7 @@ private[spark] object Utils
* specified by --jars (spark.jars) or --packages, remote jars will be downloaded to local by
* SparkSubmit at first.
*/
- def getLocalUserJarsForShell(conf: SparkConf): Seq[String] = {
+ def getLocalUserJarsForShell(conf: ReadOnlySparkConf): Seq[String] = {
val localJars = conf.getOption("spark.repl.local.jars")
localJars.map(_.split(",")).map(_.filter(_.nonEmpty)).toSeq.flatten
}
@@ -2729,7 +2717,7 @@ private[spark] object Utils
* Redact the sensitive values in the given map. If a map key matches the redaction pattern then
* its value is replaced with a dummy text.
*/
- def redact(conf: SparkConf,
+ def redact(conf: ReadOnlySparkConf,
kvs: scala.collection.Seq[(String, String)]): scala.collection.Seq[(String, String)] = {
val redactionPattern = conf.get(SECRET_REDACTION_PATTERN)
redact(redactionPattern, kvs)
@@ -2806,7 +2794,7 @@ private[spark] object Utils
redact(redactionPattern, kvs.toArray)
}
- def redactCommandLineArgs(conf: SparkConf, commands: Seq[String]): Seq[String] = {
+ def redactCommandLineArgs(conf: ReadOnlySparkConf, commands: Seq[String]): Seq[String] = {
val redactionPattern = conf.get(SECRET_REDACTION_PATTERN)
commands.map {
case PATTERN_FOR_COMMAND_LINE_ARG(key, value) =>
@@ -2920,12 +2908,19 @@ private[spark] object Utils
opt.replace("{{APP_ID}}", appId)
}
- def createSecret(conf: SparkConf): String = {
+ /**
+ * Replaces all the {{SPARK_VERSION}} occurrences with the Spark version.
+ */
+ def substituteSparkVersion(opt: String): String = {
+ opt.replace("{{SPARK_VERSION}}", SPARK_VERSION)
+ }
+
+ def createSecret(conf: ReadOnlySparkConf): String = {
val bits = conf.get(AUTH_SECRET_BIT_LENGTH)
val rnd = new SecureRandom()
val secretBytes = new Array[Byte](bits / JByte.SIZE)
rnd.nextBytes(secretBytes)
- Hex.encodeHexString(secretBytes)
+ HexFormat.of().formatHex(secretBytes)
}
/**
@@ -2971,7 +2966,7 @@ private[spark] object Utils
}
}
- def isClientMode(conf: SparkConf): Boolean = {
+ def isClientMode(conf: ReadOnlySparkConf): Boolean = {
"client".equals(conf.get(SparkLauncher.DEPLOY_MODE, "client"))
}
@@ -3037,7 +3032,7 @@ private[spark] object Utils
/**
* Convert MEMORY_OFFHEAP_SIZE to MB Unit, return 0 if MEMORY_OFFHEAP_ENABLED is false.
*/
- def executorOffHeapMemorySizeAsMb(sparkConf: SparkConf): Int = {
+ def executorOffHeapMemorySizeAsMb(sparkConf: ReadOnlySparkConf): Int = {
val sizeInMB = Utils.memoryStringToMb(sparkConf.get(MEMORY_OFFHEAP_SIZE).toString)
checkOffHeapEnabled(sparkConf, sizeInMB).toInt
}
@@ -3045,7 +3040,7 @@ private[spark] object Utils
/**
* return 0 if MEMORY_OFFHEAP_ENABLED is false.
*/
- def checkOffHeapEnabled(sparkConf: SparkConf, offHeapSize: Long): Long = {
+ def checkOffHeapEnabled(sparkConf: ReadOnlySparkConf, offHeapSize: Long): Long = {
if (sparkConf.get(MEMORY_OFFHEAP_ENABLED)) {
require(offHeapSize > 0,
s"${MEMORY_OFFHEAP_SIZE.key} must be > 0 when ${MEMORY_OFFHEAP_ENABLED.key} == true")
@@ -3056,9 +3051,9 @@ private[spark] object Utils
}
/** Returns a string message about delegation token generation failure */
- def createFailedToGetTokenMessage(serviceName: String, e: scala.Throwable): MessageWithContext = {
- log"Failed to get token from service ${MDC(SERVICE_NAME, serviceName)} " +
- log"due to ${MDC(ERROR, e)}. If ${MDC(SERVICE_NAME, serviceName)} is not used, " +
+ def createFailedToGetTokenMessage(serviceName: String): MessageWithContext = {
+ log"Failed to get token from service ${MDC(SERVICE_NAME, serviceName)}. " +
+ log"If ${MDC(SERVICE_NAME, serviceName)} is not used, " +
log"set spark.security.credentials.${MDC(SERVICE_NAME, serviceName)}.enabled to false."
}
@@ -3078,7 +3073,7 @@ private[spark] object Utils
val outFile = new File(localDir, fileName)
files += outFile
out = new FileOutputStream(outFile)
- IOUtils.copy(in, out)
+ in.transferTo(out)
out.close()
in.closeEntry()
}
@@ -3088,8 +3083,8 @@ private[spark] object Utils
logDebug(log"Unzipped from ${MDC(PATH, dfsZipFile)}\n\t${MDC(PATHS, files.mkString("\n\t"))}")
} finally {
// Close everything no matter what happened
- IOUtils.closeQuietly(in)
- IOUtils.closeQuietly(out)
+ Utils.closeQuietly(in)
+ Utils.closeQuietly(out)
}
files.toSeq
}
@@ -3143,7 +3138,19 @@ private[spark] object Utils
/**
* Return whether we are using G1GC or not
*/
- lazy val isG1GC: Boolean = {
+ lazy val isG1GC: Boolean = checkUseGC("UseG1GC")
+
+ /**
+ * Return whether we are using ZGC or not
+ */
+ lazy val isZGC: Boolean = checkUseGC("UseZGC")
+
+ /**
+ * Return whether we are using ShenandoahGC or not
+ */
+ lazy val isShenandoahGC: Boolean = checkUseGC("UseShenandoahGC")
+
+ def checkUseGC(useGCObjectStr: String): Boolean = {
Try {
val clazz = Utils.classForName("com.sun.management.HotSpotDiagnosticMXBean")
.asInstanceOf[Class[_ <: PlatformManagedObject]]
@@ -3152,11 +3159,22 @@ private[spark] object Utils
val vmOptionMethod = clazz.getMethod("getVMOption", classOf[String])
val valueMethod = vmOptionClazz.getMethod("getValue")
- val useG1GCObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC")
- val useG1GC = valueMethod.invoke(useG1GCObject).asInstanceOf[String]
- "true".equals(useG1GC)
+ val useGCObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, useGCObjectStr)
+ val useGC = valueMethod.invoke(useGCObject).asInstanceOf[String]
+ "true".equals(useGC)
}.getOrElse(false)
}
+
+ /**
+ * Return a string of printStackTrace result.
+ */
+ def stringifyException(e: Throwable): String = {
+ val stm = new StringWriter()
+ val wrt = new PrintWriter(stm)
+ e.printStackTrace(wrt)
+ wrt.close()
+ stm.toString()
+ }
}
private[util] object CallerContext extends Logging {
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
index 16a2f4fb6cad9..d892fa0e47060 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -29,7 +29,7 @@ import com.google.common.io.ByteStreams
import org.apache.spark.{SparkEnv, TaskContext}
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.executor.ShuffleWriteMetrics
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.serializer.{DeserializationStream, Serializer, SerializerManager}
import org.apache.spark.storage.{BlockId, BlockManager}
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
index 393cdbbef0a5a..4da89a94201a9 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
@@ -28,12 +28,12 @@ import com.google.common.io.ByteStreams
import org.apache.spark._
import org.apache.spark.executor.ShuffleWriteMetrics
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.{NUM_BYTES, TASK_ATTEMPT_ID}
import org.apache.spark.serializer._
import org.apache.spark.shuffle.{ShufflePartitionPairsWriter, ShuffleWriteMetricsReporter}
import org.apache.spark.shuffle.api.{ShuffleMapOutputWriter, ShufflePartitionWriter}
-import org.apache.spark.shuffle.checksum.ShuffleChecksumSupport
+import org.apache.spark.shuffle.checksum.{RowBasedChecksum, ShuffleChecksumSupport}
import org.apache.spark.storage.{BlockId, DiskBlockObjectWriter, ShuffleBlockId}
import org.apache.spark.util.{CompletionIterator, Utils => TryUtils}
@@ -97,7 +97,8 @@ private[spark] class ExternalSorter[K, V, C](
aggregator: Option[Aggregator[K, V, C]] = None,
partitioner: Option[Partitioner] = None,
ordering: Option[Ordering[K]] = None,
- serializer: Serializer = SparkEnv.get.serializer)
+ serializer: Serializer = SparkEnv.get.serializer,
+ rowBasedChecksums: Array[RowBasedChecksum] = Array.empty)
extends Spillable[WritablePartitionedPairCollection[K, C]](context.taskMemoryManager())
with Logging with ShuffleChecksumSupport {
@@ -142,10 +143,16 @@ private[spark] class ExternalSorter[K, V, C](
private val forceSpillFiles = new ArrayBuffer[SpilledFile]
@volatile private var readingIterator: SpillableIterator = null
+ /** Checksum calculator for each partition. Empty when shuffle checksum disabled. */
private val partitionChecksums = createPartitionChecksums(numPartitions, conf)
def getChecksums: Array[Long] = getChecksumValues(partitionChecksums)
+ def getRowBasedChecksums: Array[RowBasedChecksum] = rowBasedChecksums
+
+ def getAggregatedChecksumValue: Long =
+ RowBasedChecksum.getAggregatedChecksumValue(rowBasedChecksums)
+
// A comparator for keys K that orders them within a partition to allow aggregation or sorting.
// Can be a partial ordering by hash code if a total ordering is not provided through by the
// user. (A partial ordering means that equal keys have comparator.compare(k, k) = 0, but some
@@ -197,16 +204,24 @@ private[spark] class ExternalSorter[K, V, C](
while (records.hasNext) {
addElementsRead()
kv = records.next()
- map.changeValue((actualPartitioner.getPartition(kv._1), kv._1), update)
+ val partitionId = actualPartitioner.getPartition(kv._1)
+ map.changeValue((partitionId, kv._1), update)
maybeSpillCollection(usingMap = true)
+ if (rowBasedChecksums.nonEmpty) {
+ rowBasedChecksums(partitionId).update(kv._1, kv._2)
+ }
}
} else {
// Stick values into our buffer
while (records.hasNext) {
addElementsRead()
val kv = records.next()
- buffer.insert(actualPartitioner.getPartition(kv._1), kv._1, kv._2.asInstanceOf[C])
+ val partitionId = actualPartitioner.getPartition(kv._1)
+ buffer.insert(partitionId, kv._1, kv._2.asInstanceOf[C])
maybeSpillCollection(usingMap = false)
+ if (rowBasedChecksums.nonEmpty) {
+ rowBasedChecksums(partitionId).update(kv._1, kv._2)
+ }
}
}
}
diff --git a/core/src/main/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMap.scala b/core/src/main/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMap.scala
deleted file mode 100644
index 69665aaeac4db..0000000000000
--- a/core/src/main/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMap.scala
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.util.collection
-
-import scala.reflect._
-
-/**
- * A fast hash map implementation for primitive, non-null keys. This hash map supports
- * insertions and updates, but not deletions. This map is about an order of magnitude
- * faster than java.util.HashMap, while using much less space overhead.
- *
- * Under the hood, it uses our OpenHashSet implementation.
- */
-private[spark]
-class PrimitiveKeyOpenHashMap[@specialized(Long, Int) K: ClassTag,
- @specialized(Long, Int, Double) V: ClassTag](
- initialCapacity: Int)
- extends Iterable[(K, V)]
- with Serializable {
-
- def this() = this(64)
-
- require(classTag[K] == classTag[Long] || classTag[K] == classTag[Int])
-
- // Init in constructor (instead of in declaration) to work around a Scala compiler specialization
- // bug that would generate two arrays (one for Object and one for specialized T).
- protected var _keySet: OpenHashSet[K] = _
- private var _values: Array[V] = _
- _keySet = new OpenHashSet[K](initialCapacity)
- _values = new Array[V](_keySet.capacity)
-
- private var _oldValues: Array[V] = null
-
- override def size: Int = _keySet.size
-
- /** Tests whether this map contains a binding for a key. */
- def contains(k: K): Boolean = {
- _keySet.getPos(k) != OpenHashSet.INVALID_POS
- }
-
- /** Get the value for a given key */
- def apply(k: K): V = {
- val pos = _keySet.getPos(k)
- _values(pos)
- }
-
- /** Get the value for a given key, or returns elseValue if it doesn't exist. */
- def getOrElse(k: K, elseValue: V): V = {
- val pos = _keySet.getPos(k)
- if (pos >= 0) _values(pos) else elseValue
- }
-
- /** Set the value for a key */
- def update(k: K, v: V): Unit = {
- val pos = _keySet.addWithoutResize(k) & OpenHashSet.POSITION_MASK
- _values(pos) = v
- _keySet.rehashIfNeeded(k, grow, move)
- _oldValues = null
- }
-
- /**
- * If the key doesn't exist yet in the hash map, set its value to defaultValue; otherwise,
- * set its value to mergeValue(oldValue).
- *
- * @return the newly updated value.
- */
- def changeValue(k: K, defaultValue: => V, mergeValue: (V) => V): V = {
- val pos = _keySet.addWithoutResize(k)
- if ((pos & OpenHashSet.NONEXISTENCE_MASK) != 0) {
- val newValue = defaultValue
- _values(pos & OpenHashSet.POSITION_MASK) = newValue
- _keySet.rehashIfNeeded(k, grow, move)
- newValue
- } else {
- _values(pos) = mergeValue(_values(pos))
- _values(pos)
- }
- }
-
- override def iterator: Iterator[(K, V)] = new Iterator[(K, V)] {
- var pos = 0
- var nextPair: (K, V) = computeNextPair()
-
- /** Get the next value we should return from next(), or null if we're finished iterating */
- def computeNextPair(): (K, V) = {
- pos = _keySet.nextPos(pos)
- if (pos >= 0) {
- val ret = (_keySet.getValue(pos), _values(pos))
- pos += 1
- ret
- } else {
- null
- }
- }
-
- def hasNext: Boolean = nextPair != null
-
- def next(): (K, V) = {
- val pair = nextPair
- nextPair = computeNextPair()
- pair
- }
- }
-
- private def grow(newCapacity: Int): Unit = {
- _oldValues = _values
- _values = new Array[V](newCapacity)
- }
-
- private def move(oldPos: Int, newPos: Int): Unit = {
- _values(newPos) = _oldValues(oldPos)
- }
-}
diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
index 7f2a1a8419a71..17713e83bb83e 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
@@ -18,7 +18,7 @@
package org.apache.spark.util.collection
import org.apache.spark.SparkEnv
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config._
import org.apache.spark.memory.{MemoryConsumer, MemoryMode, TaskMemoryManager}
@@ -58,6 +58,10 @@ private[spark] abstract class Spillable[C](taskMemoryManager: TaskMemoryManager)
private[this] val numElementsForceSpillThreshold: Int =
SparkEnv.get.conf.get(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD)
+ // Force this collection to spill when its size is greater than this threshold
+ private[this] val maxSizeForceSpillThreshold: Long =
+ SparkEnv.get.conf.get(SHUFFLE_SPILL_MAX_SIZE_FORCE_SPILL_THRESHOLD)
+
// Threshold for this collection's size in bytes before we start tracking its memory usage
// To avoid a large number of small spills, initialize this to a value orders of magnitude > 0
@volatile private[this] var myMemoryThreshold = initialMemoryThreshold
@@ -80,21 +84,25 @@ private[spark] abstract class Spillable[C](taskMemoryManager: TaskMemoryManager)
* @return true if `collection` was spilled to disk; false otherwise
*/
protected def maybeSpill(collection: C, currentMemory: Long): Boolean = {
- var shouldSpill = false
- if (elementsRead % 32 == 0 && currentMemory >= myMemoryThreshold) {
+ val shouldSpill = if (_elementsRead > numElementsForceSpillThreshold
+ || currentMemory > maxSizeForceSpillThreshold) {
+ // Check number of elements or memory usage limits, whichever is hit first
+ true
+ } else if (_elementsRead % 32 == 0 && currentMemory >= myMemoryThreshold) {
// Claim up to double our current memory from the shuffle memory pool
val amountToRequest = 2 * currentMemory - myMemoryThreshold
val granted = acquireMemory(amountToRequest)
myMemoryThreshold += granted
// If we were granted too little memory to grow further (either tryToAcquire returned 0,
// or we already had more memory than myMemoryThreshold), spill the current collection
- shouldSpill = currentMemory >= myMemoryThreshold
+ currentMemory >= myMemoryThreshold
+ } else {
+ false
}
- shouldSpill = shouldSpill || _elementsRead > numElementsForceSpillThreshold
// Actually spill
if (shouldSpill) {
_spillCount += 1
- logSpillage(currentMemory)
+ logSpillage(currentMemory, _elementsRead)
spill(collection)
_elementsRead = 0
_memoryBytesSpilled += currentMemory
@@ -140,12 +148,14 @@ private[spark] abstract class Spillable[C](taskMemoryManager: TaskMemoryManager)
* Prints a standard log message detailing spillage.
*
* @param size number of bytes spilled
+ * @param elements number of elements read from input since last spill
*/
- @inline private def logSpillage(size: Long): Unit = {
+ @inline private def logSpillage(size: Long, elements: Int): Unit = {
val threadId = Thread.currentThread().getId
logInfo(log"Thread ${MDC(LogKeys.THREAD_ID, threadId)} " +
log"spilling in-memory map of ${MDC(LogKeys.BYTE_SIZE,
- org.apache.spark.util.Utils.bytesToString(size))} to disk " +
+ org.apache.spark.util.Utils.bytesToString(size))} " +
+ log"(elements: ${MDC(LogKeys.NUM_ELEMENTS_SPILL_RECORDS, elements)}) to disk " +
log"(${MDC(LogKeys.NUM_SPILLS, _spillCount)} times so far)")
}
}
diff --git a/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala b/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
index 88bd117ba22bf..0f4cf79c0fe91 100644
--- a/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
+++ b/core/src/main/scala/org/apache/spark/util/io/ChunkedByteBuffer.scala
@@ -21,10 +21,8 @@ import java.io.{Externalizable, File, FileInputStream, InputStream, ObjectInput,
import java.nio.ByteBuffer
import java.nio.channels.WritableByteChannel
-import com.google.common.io.ByteStreams
import com.google.common.primitives.UnsignedBytes
import io.netty.handler.stream.ChunkedStream
-import org.apache.commons.io.IOUtils
import org.apache.spark.SparkEnv
import org.apache.spark.internal.config
@@ -246,12 +244,12 @@ private[spark] object ChunkedByteBuffer {
// and spark currently is not expecting memory-mapped buffers in the memory store, it conflicts
// with other parts that manage the lifecycle of buffers and dispose them. See SPARK-25422.
val is = new FileInputStream(file)
- ByteStreams.skipFully(is, offset)
+ is.skipNBytes(offset)
val in = new LimitedInputStream(is, length)
val chunkSize = math.min(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH, length).toInt
val out = new ChunkedByteBufferOutputStream(chunkSize, ByteBuffer.allocate _)
Utils.tryWithSafeFinally {
- IOUtils.copy(in, out)
+ in.transferTo(out)
} {
in.close()
out.close()
diff --git a/core/src/main/scala/org/apache/spark/util/logging/DriverLogger.scala b/core/src/main/scala/org/apache/spark/util/logging/DriverLogger.scala
index 840fb59d410a6..9854c99707b1c 100644
--- a/core/src/main/scala/org/apache/spark/util/logging/DriverLogger.scala
+++ b/core/src/main/scala/org/apache/spark/util/logging/DriverLogger.scala
@@ -21,7 +21,6 @@ import java.io._
import java.util.EnumSet
import java.util.concurrent.{ScheduledExecutorService, TimeUnit}
-import org.apache.commons.io.FileUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, FSDataOutputStream, Path}
import org.apache.hadoop.fs.permission.FsPermission
@@ -33,7 +32,7 @@ import org.apache.logging.log4j.core.layout.PatternLayout
import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.network.util.JavaUtils
@@ -47,8 +46,8 @@ private[spark] class DriverLogger(conf: SparkConf) extends Logging {
private val LOG_FILE_PERMISSIONS = new FsPermission(Integer.parseInt("770", 8).toShort)
private val localLogFile: String = conf.get(DRIVER_LOG_LOCAL_DIR).map {
- FileUtils.getFile(_, DriverLogger.DRIVER_LOG_FILE).getAbsolutePath()
- }.getOrElse(FileUtils.getFile(
+ Utils.getFile(_, DriverLogger.DRIVER_LOG_FILE).getAbsolutePath()
+ }.getOrElse(Utils.getFile(
Utils.getLocalDir(conf),
DriverLogger.DRIVER_LOG_DIR,
DriverLogger.DRIVER_LOG_FILE).getAbsolutePath())
@@ -106,7 +105,7 @@ private[spark] class DriverLogger(conf: SparkConf) extends Logging {
logError(s"Error in persisting driver logs", e)
} finally {
Utils.tryLogNonFatalError {
- JavaUtils.deleteRecursively(FileUtils.getFile(localLogFile).getParentFile())
+ JavaUtils.deleteRecursively(Utils.getFile(localLogFile).getParentFile())
}
}
}
diff --git a/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala b/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
index 2fec1ff832c0a..39fde0cef534b 100644
--- a/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
+++ b/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
@@ -20,7 +20,7 @@ package org.apache.spark.util.logging
import java.io.{File, FileOutputStream, InputStream, IOException}
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.{IntParam, Utils}
diff --git a/core/src/main/scala/org/apache/spark/util/logging/RollingFileAppender.scala b/core/src/main/scala/org/apache/spark/util/logging/RollingFileAppender.scala
index 6927c119a91c5..544ab1711fa37 100644
--- a/core/src/main/scala/org/apache/spark/util/logging/RollingFileAppender.scala
+++ b/core/src/main/scala/org/apache/spark/util/logging/RollingFileAppender.scala
@@ -20,13 +20,11 @@ package org.apache.spark.util.logging
import java.io._
import java.util.zip.GZIPOutputStream
-import com.google.common.io.Files
-import org.apache.commons.io.IOUtils
-
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{config, MDC}
import org.apache.spark.internal.LogKeys._
+import org.apache.spark.internal.config
import org.apache.spark.util.ArrayImplicits._
+import org.apache.spark.util.Utils
/**
* Continuously appends data from input stream into the given file, and rolls
@@ -91,16 +89,16 @@ private[spark] class RollingFileAppender(
try {
inputStream = new FileInputStream(activeFile)
gzOutputStream = new GZIPOutputStream(new FileOutputStream(gzFile))
- IOUtils.copy(inputStream, gzOutputStream)
+ inputStream.transferTo(gzOutputStream)
inputStream.close()
gzOutputStream.close()
activeFile.delete()
} finally {
- IOUtils.closeQuietly(inputStream)
- IOUtils.closeQuietly(gzOutputStream)
+ Utils.closeQuietly(inputStream)
+ Utils.closeQuietly(gzOutputStream)
}
} else {
- Files.move(activeFile, rolloverFile)
+ Utils.moveFile(activeFile, rolloverFile)
}
}
diff --git a/core/src/main/scala/org/apache/spark/util/logging/RollingPolicy.scala b/core/src/main/scala/org/apache/spark/util/logging/RollingPolicy.scala
index 310e895930943..7f4fc65fd92bf 100644
--- a/core/src/main/scala/org/apache/spark/util/logging/RollingPolicy.scala
+++ b/core/src/main/scala/org/apache/spark/util/logging/RollingPolicy.scala
@@ -20,7 +20,7 @@ package org.apache.spark.util.logging
import java.text.SimpleDateFormat
import java.util.{Calendar, Locale}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
/**
diff --git a/core/src/test/java/org/apache/spark/io/GenericFileInputStreamSuite.java b/core/src/test/java/org/apache/spark/io/GenericFileInputStreamSuite.java
index 7d5237eecbbf8..bbfa25e60d74b 100644
--- a/core/src/test/java/org/apache/spark/io/GenericFileInputStreamSuite.java
+++ b/core/src/test/java/org/apache/spark/io/GenericFileInputStreamSuite.java
@@ -16,7 +16,6 @@
*/
package org.apache.spark.io;
-import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -24,6 +23,7 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
import java.util.concurrent.ThreadLocalRandom;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -44,7 +44,7 @@ public abstract class GenericFileInputStreamSuite {
public void setUp() throws IOException {
ThreadLocalRandom.current().nextBytes(randomBytes);
inputFile = File.createTempFile("temp-file", ".tmp");
- FileUtils.writeByteArrayToFile(inputFile, randomBytes);
+ Files.write(inputFile.toPath(), randomBytes);
}
@AfterEach
diff --git a/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java b/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java
index c55254e04f401..26f0a86354478 100644
--- a/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java
+++ b/core/src/test/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriterSuite.java
@@ -21,6 +21,7 @@
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.spark.*;
import org.apache.spark.network.shuffle.checksum.ShuffleChecksumHelper;
@@ -49,6 +50,7 @@
import org.apache.spark.scheduler.MapStatus;
import org.apache.spark.security.CryptoStreamUtils;
import org.apache.spark.serializer.*;
+import org.apache.spark.shuffle.checksum.RowBasedChecksum;
import org.apache.spark.shuffle.IndexShuffleBlockResolver;
import org.apache.spark.shuffle.sort.io.LocalDiskShuffleExecutorComponents;
import org.apache.spark.storage.*;
@@ -174,11 +176,18 @@ public void setUp() throws Exception {
File file = (File) invocationOnMock.getArguments()[0];
return Utils.tempFileWith(file);
});
-
+ resetDependency(false);
when(taskContext.taskMetrics()).thenReturn(taskMetrics);
+ when(taskContext.taskMemoryManager()).thenReturn(taskMemoryManager);
+ }
+
+ private void resetDependency(boolean rowBasedChecksumEnabled) {
when(shuffleDep.serializer()).thenReturn(serializer);
when(shuffleDep.partitioner()).thenReturn(hashPartitioner);
- when(taskContext.taskMemoryManager()).thenReturn(taskMemoryManager);
+ final int checksumSize = rowBasedChecksumEnabled ? NUM_PARTITIONS : 0;
+ final RowBasedChecksum[] rowBasedChecksums =
+ createPartitionRowBasedChecksums(checksumSize);
+ when(shuffleDep.rowBasedChecksums()).thenReturn(rowBasedChecksums);
}
private UnsafeShuffleWriter createWriter(boolean transferToEnabled)
@@ -317,7 +326,7 @@ public void writeWithoutSpilling() throws Exception {
@Test
public void writeChecksumFileWithoutSpill() throws Exception {
IndexShuffleBlockResolver blockResolver =
- new IndexShuffleBlockResolver(conf, blockManager, Collections.emptyMap());
+ new IndexShuffleBlockResolver(conf, blockManager, new ConcurrentHashMap<>());
ShuffleChecksumBlockId checksumBlockId =
new ShuffleChecksumBlockId(0, 0, IndexShuffleBlockResolver.NOOP_REDUCE_ID());
String checksumAlgorithm = conf.get(package$.MODULE$.SHUFFLE_CHECKSUM_ALGORITHM());
@@ -348,7 +357,7 @@ public void writeChecksumFileWithoutSpill() throws Exception {
@Test
public void writeChecksumFileWithSpill() throws Exception {
IndexShuffleBlockResolver blockResolver =
- new IndexShuffleBlockResolver(conf, blockManager, Collections.emptyMap());
+ new IndexShuffleBlockResolver(conf, blockManager, new ConcurrentHashMap<>());
ShuffleChecksumBlockId checksumBlockId =
new ShuffleChecksumBlockId(0, 0, IndexShuffleBlockResolver.NOOP_REDUCE_ID());
String checksumAlgorithm = conf.get(package$.MODULE$.SHUFFLE_CHECKSUM_ALGORITHM());
@@ -613,6 +622,43 @@ public void spillFilesAreDeletedWhenStoppingAfterError() throws IOException, Spa
assertSpillFilesWereCleanedUp();
}
+ @Test
+ public void testRowBasedChecksum() throws IOException, SparkException {
+ final ArrayList> dataToWrite = new ArrayList<>();
+ for (int i = 0; i < NUM_PARTITIONS; i++) {
+ for (int j = 0; j < 5; j++) {
+ dataToWrite.add(new Tuple2<>(i, i + j));
+ }
+ }
+
+ long[] checksumValues = new long[0];
+ long aggregatedChecksumValue = 0;
+ try {
+ for (int i = 0; i < 100; i++) {
+ resetDependency(true);
+ final UnsafeShuffleWriter writer = createWriter(false);
+ Collections.shuffle(dataToWrite);
+ writer.write(dataToWrite.iterator());
+ writer.stop(true);
+
+ if (i == 0) {
+ checksumValues = getRowBasedChecksumValues(writer.getRowBasedChecksums());
+ assertEquals(checksumValues.length, NUM_PARTITIONS);
+ Arrays.stream(checksumValues).allMatch(v -> v > 0);
+
+ aggregatedChecksumValue = writer.getAggregatedChecksumValue();
+ assert(aggregatedChecksumValue != 0);
+ } else {
+ assertArrayEquals(checksumValues,
+ getRowBasedChecksumValues(writer.getRowBasedChecksums()));
+ assertEquals(aggregatedChecksumValue, writer.getAggregatedChecksumValue());
+ }
+ }
+ } finally {
+ resetDependency(false);
+ }
+ }
+
@Test
public void testPeakMemoryUsed() throws Exception {
final long recordLengthBytes = 8;
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
index 9e83717f52087..532566741206f 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
@@ -87,9 +87,13 @@ public int compare(
private final long pageSizeBytes = conf.getSizeAsBytes(
package$.MODULE$.BUFFER_PAGESIZE().key(), "4m");
- private final int spillThreshold =
+ private final int spillElementsThreshold =
(int) conf.get(package$.MODULE$.SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD());
+ private final long spillSizeThreshold =
+ (long) conf.get(package$.MODULE$.SHUFFLE_SPILL_MAX_SIZE_FORCE_SPILL_THRESHOLD());
+
+
@BeforeEach
public void setUp() throws Exception {
MockitoAnnotations.openMocks(this).close();
@@ -163,7 +167,8 @@ private UnsafeExternalSorter newSorter() throws IOException {
prefixComparator,
/* initialSize */ 1024,
pageSizeBytes,
- spillThreshold,
+ spillElementsThreshold,
+ spillSizeThreshold,
shouldUseRadixSort());
}
@@ -453,7 +458,8 @@ public void forcedSpillingWithoutComparator() throws Exception {
null,
/* initialSize */ 1024,
pageSizeBytes,
- spillThreshold,
+ spillElementsThreshold,
+ spillSizeThreshold,
shouldUseRadixSort());
long[] record = new long[100];
int recordSize = record.length * 8;
@@ -515,7 +521,8 @@ public void testPeakMemoryUsed() throws Exception {
prefixComparator,
1024,
pageSizeBytes,
- spillThreshold,
+ spillElementsThreshold,
+ spillSizeThreshold,
shouldUseRadixSort());
// Peak memory should be monotonically increasing. More specifically, every time
@@ -584,7 +591,7 @@ public void testNoOOMDuringSpill() throws Exception {
}
// Check that spilling still succeeds when the task is starved for memory.
- memoryManager.markconsequentOOM(Integer.MAX_VALUE);
+ memoryManager.markConsequentOOM(Integer.MAX_VALUE);
sorter.spill();
memoryManager.resetConsequentOOM();
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
index 91f90d7d145aa..a612824fed498 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
@@ -172,7 +172,7 @@ public int compare(
recordComparator, prefixComparator, 100, shouldUseRadixSort());
// Ensure that the sorter does not OOM while freeing its memory.
- testMemoryManager.markconsequentOOM(Integer.MAX_VALUE);
+ testMemoryManager.markConsequentOOM(Integer.MAX_VALUE);
sorter.freeMemory();
testMemoryManager.resetConsequentOOM();
Assertions.assertFalse(sorter.hasSpaceForAnotherRecord());
@@ -182,7 +182,7 @@ public int compare(
sorter.expandPointerArray(array);
// Ensure that it is safe to call freeMemory() multiple times.
- testMemoryManager.markconsequentOOM(Integer.MAX_VALUE);
+ testMemoryManager.markConsequentOOM(Integer.MAX_VALUE);
sorter.freeMemory();
sorter.freeMemory();
testMemoryManager.resetConsequentOOM();
diff --git a/core/src/test/java/test/org/apache/spark/JavaAPISuite.java b/core/src/test/java/test/org/apache/spark/JavaAPISuite.java
index 802cb2667cc88..f2f4101877e51 100644
--- a/core/src/test/java/test/org/apache/spark/JavaAPISuite.java
+++ b/core/src/test/java/test/org/apache/spark/JavaAPISuite.java
@@ -21,6 +21,7 @@
import java.nio.channels.FileChannel;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -44,12 +45,8 @@
import scala.Tuple4;
import scala.jdk.javaapi.CollectionConverters;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import com.google.common.base.Throwables;
-import com.google.common.io.Files;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
@@ -70,6 +67,7 @@
import org.apache.spark.api.java.Optional;
import org.apache.spark.api.java.function.*;
import org.apache.spark.input.PortableDataStream;
+import org.apache.spark.network.util.JavaUtils;
import org.apache.spark.partial.BoundedDouble;
import org.apache.spark.partial.PartialResult;
import org.apache.spark.rdd.RDD;
@@ -334,7 +332,8 @@ public void foreachPartition() {
public void toLocalIterator() {
List