or apply the shuffle sort merge join hint as described in the Spark documentation: https://spark.apache.org/docs/latest/sql-ref-syntax-qry-select-hints.html#join-hints."
]
},
"_LEGACY_ERROR_TEMP_2251" : {
diff --git a/common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala b/common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
index 85d460f618a79..2ff8112989018 100644
--- a/common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
+++ b/common/utils/src/main/scala/org/apache/spark/ErrorClassesJSONReader.scala
@@ -19,13 +19,10 @@ package org.apache.spark
import java.net.URL
-import scala.jdk.CollectionConverters._
-
import com.fasterxml.jackson.annotation.JsonIgnore
import com.fasterxml.jackson.core.`type`.TypeReference
import com.fasterxml.jackson.databind.json.JsonMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
-import org.apache.commons.text.StringSubstitutor
import org.apache.spark.annotation.DeveloperApi
@@ -48,9 +45,7 @@ class ErrorClassesJsonReader(jsonFileURLs: Seq[URL]) {
case (key, null) => key -> "null"
case (key, value) => key -> value
}
- val sub = new StringSubstitutor(sanitizedParameters.asJava)
- sub.setEnableUndefinedVariableException(true)
- sub.setDisableSubstitutionInValues(true)
+ val sub = new StringSubstitutor(sanitizedParameters)
val errorMessage = try {
sub.replace(ErrorClassesJsonReader.TEMPLATE_REGEX.replaceAllIn(
messageTemplate, "\\$\\{$1\\}"))
diff --git a/common/utils/src/main/scala/org/apache/spark/SparkException.scala b/common/utils/src/main/scala/org/apache/spark/SparkException.scala
index 00989fd29095c..f438c62253475 100644
--- a/common/utils/src/main/scala/org/apache/spark/SparkException.scala
+++ b/common/utils/src/main/scala/org/apache/spark/SparkException.scala
@@ -131,6 +131,14 @@ object SparkException {
messageParameters: java.util.Map[String, String]): Map[String, String] = {
messageParameters.asScala.toMap
}
+
+ def mustOverrideOneMethodError(methodName: String): RuntimeException = {
+ val msg = s"You must override one `$methodName`. It's preferred to not override the " +
+ "deprecated one."
+ new SparkRuntimeException(
+ "INTERNAL_ERROR",
+ Map("message" -> msg))
+ }
}
/**
@@ -145,8 +153,8 @@ private[spark] class SparkDriverExecutionException(cause: Throwable)
* Exception thrown when the main user code is run as a child process (e.g. pyspark) and we want
* the parent SparkSubmit process to exit with the same exit code.
*/
-private[spark] case class SparkUserAppException(exitCode: Int)
- extends SparkException(s"User application exited with $exitCode")
+private[spark] case class SparkUserAppException(exitCode: Int, cause: Throwable = null)
+ extends SparkException(s"User application exited with $exitCode", cause)
/**
* Exception thrown when the relative executor to access is dead.
diff --git a/common/utils/src/main/scala/org/apache/spark/StringSubstitutor.scala b/common/utils/src/main/scala/org/apache/spark/StringSubstitutor.scala
new file mode 100644
index 0000000000000..6513c7dc7e8ec
--- /dev/null
+++ b/common/utils/src/main/scala/org/apache/spark/StringSubstitutor.scala
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark
+
+import scala.collection.mutable.{ListBuffer, StringBuilder}
+
+class StringSubstitutor(
+ resolver: Map[String, Any],
+ prefix: String = "${",
+ suffix: String = "}",
+ escape: Char = '$',
+ valueDelimiter: String = ":-",
+ preserveEscapes: Boolean = true,
+ enableSubstitutionInVariables: Boolean = false,
+ enableUndefinedVariableException: Boolean = true) {
+
+ private val prefixLen = prefix.length
+ private val suffixLen = suffix.length
+
+ def replace(source: String): String = {
+ if (source == null) return null
+ val buf = new StringBuilder(source)
+ if (!substitute(buf, 0, buf.length, ListBuffer.empty[String])) {
+ source
+ } else {
+ buf.toString
+ }
+ }
+
+ private def substitute(
+ buf: StringBuilder,
+ offset: Int,
+ length: Int,
+ priorVariables: ListBuffer[String]): Boolean = {
+ var altered = false
+ var currentPos = offset
+ val endPos = offset + length
+
+ while (currentPos < endPos) {
+ // Find prefix
+ val prefixPos = buf.indexOf(prefix, currentPos)
+ if (prefixPos < 0 || prefixPos >= endPos) {
+ return altered
+ }
+
+ if (prefixPos > 0 && buf.charAt(prefixPos - 1) == escape) {
+ if (preserveEscapes) {
+ currentPos = prefixPos + prefixLen
+ } else {
+ buf.deleteCharAt(prefixPos - 1)
+ altered = true
+ currentPos = prefixPos - 1 + prefixLen // adjust position
+ }
+ } else {
+ // Find suffix
+ val suffixPos = buf.indexOf(suffix, prefixPos + prefixLen)
+ if (suffixPos < 0) {
+ currentPos = prefixPos + prefixLen
+ } else {
+ // Extract variable name and default
+ val varFull = buf.substring(prefixPos + prefixLen, suffixPos)
+ val index = varFull.indexOf(valueDelimiter)
+ val (varName, defaultValue) = if (index >= 0) {
+ (varFull.substring(0, index), Some(varFull.substring(index + valueDelimiter.length)))
+ } else {
+ (varFull, None)
+ }
+
+ // Check for cycle
+ if (priorVariables.contains(varName)) {
+ throw new IllegalStateException(
+ s"Infinite loop in property interpolation of $varName")
+ }
+
+ resolver.get(varName) match {
+ case Some(value) =>
+ var replacement = value.toString
+ if (enableSubstitutionInVariables) {
+ val newPrior = priorVariables.clone() += varName
+ val tempBuf = new StringBuilder(replacement)
+ if (substitute(tempBuf, 0, tempBuf.length, newPrior)) {
+ replacement = tempBuf.toString
+ }
+ }
+ buf.replace(prefixPos, suffixPos + suffixLen, replacement)
+ // Adjust positions and recurse on the changed part
+ val changeInLength = replacement.length - (suffixPos + suffixLen - prefixPos)
+ val newLength = length + changeInLength
+ substitute(buf, prefixPos + replacement.length, newLength, priorVariables)
+ return true // restart scan after change
+
+ case None =>
+ defaultValue match {
+ case Some(value) =>
+ buf.replace(prefixPos, suffixPos + suffixLen, value)
+ val changeInLength = value.length - (suffixPos + suffixLen - prefixPos)
+ val newLength = length + changeInLength
+ substitute(buf, prefixPos + value.length, length, priorVariables)
+ return true
+ case None =>
+ if (enableUndefinedVariableException) {
+ throw new IllegalArgumentException(s"Undefined variable: $varName")
+ }
+ }
+ }
+ currentPos = suffixPos + suffixLen
+ }
+ }
+ }
+ altered
+ }
+}
diff --git a/common/utils/src/main/java/org/apache/spark/api/java/function/package.scala b/common/utils/src/main/scala/org/apache/spark/api/java/function/package.scala
similarity index 100%
rename from common/utils/src/main/java/org/apache/spark/api/java/function/package.scala
rename to common/utils/src/main/scala/org/apache/spark/api/java/function/package.scala
diff --git a/common/utils/src/main/scala/org/apache/spark/internal/LogKey.scala b/common/utils/src/main/scala/org/apache/spark/internal/LogKey.scala
deleted file mode 100644
index b0fae3fd9443b..0000000000000
--- a/common/utils/src/main/scala/org/apache/spark/internal/LogKey.scala
+++ /dev/null
@@ -1,915 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.internal
-
-import java.util.Locale
-
-/**
- * All structured logging `keys` used in `MDC` must be extends `LogKey`
- *
- *
- * `LogKey`s serve as identifiers for mapped diagnostic contexts (MDC) within logs.
- * Follow these guidelines when adding a new LogKey:
- *
- * -
- * Define all structured logging keys in `LogKey.scala`, and sort them alphabetically for
- * ease of search.
- *
- * -
- * Use `UPPER_SNAKE_CASE` for key names.
- *
- * -
- * Key names should be both simple and broad, yet include specific identifiers like `STAGE_ID`,
- * `TASK_ID`, and `JOB_ID` when needed for clarity. For instance, use `MAX_ATTEMPTS` as a
- * general key instead of creating separate keys for each scenario such as
- * `EXECUTOR_STATE_SYNC_MAX_ATTEMPTS` and `MAX_TASK_FAILURES`.
- * This balances simplicity with the detail needed for effective logging.
- *
- * -
- * Use abbreviations in names if they are widely understood,
- * such as `APP_ID` for APPLICATION_ID, and `K8S` for KUBERNETES.
- *
- * -
- * For time-related keys, use milliseconds as the unit of time.
- *
- *
- */
-trait LogKey {
- private lazy val _name: String = getClass.getSimpleName.stripSuffix("$").toLowerCase(Locale.ROOT)
- def name: String = _name
-}
-
-/**
- * Various keys used for mapped diagnostic contexts(MDC) in logging. All structured logging keys
- * should be defined here for standardization.
- */
-private[spark] object LogKeys {
- case object ACCUMULATOR_ID extends LogKey
- case object ACL_ENABLED extends LogKey
- case object ACTUAL_NUM_FILES extends LogKey
- case object ACTUAL_PARTITION_COLUMN extends LogKey
- case object ADDED_JARS extends LogKey
- case object ADMIN_ACLS extends LogKey
- case object ADMIN_ACL_GROUPS extends LogKey
- case object ADVISORY_TARGET_SIZE extends LogKey
- case object AGGREGATE_FUNCTIONS extends LogKey
- case object ALIGNED_FROM_TIME extends LogKey
- case object ALIGNED_TO_TIME extends LogKey
- case object ALPHA extends LogKey
- case object ANALYSIS_ERROR extends LogKey
- case object APP_ATTEMPT_ID extends LogKey
- case object APP_ATTEMPT_SHUFFLE_MERGE_ID extends LogKey
- case object APP_DESC extends LogKey
- case object APP_EXECUTOR_ID extends LogKey
- case object APP_ID extends LogKey
- case object APP_NAME extends LogKey
- case object APP_STATE extends LogKey
- case object ARCHIVE_NAME extends LogKey
- case object ARGS extends LogKey
- case object ARTIFACTS extends LogKey
- case object ARTIFACT_ID extends LogKey
- case object ATTRIBUTE_MAP extends LogKey
- case object AUTH_ENABLED extends LogKey
- case object AVG_BATCH_PROC_TIME extends LogKey
- case object BACKUP_FILE extends LogKey
- case object BARRIER_EPOCH extends LogKey
- case object BARRIER_ID extends LogKey
- case object BATCH_ID extends LogKey
- case object BATCH_NAME extends LogKey
- case object BATCH_TIMES extends LogKey
- case object BATCH_TIMESTAMP extends LogKey
- case object BATCH_WRITE extends LogKey
- case object BIND_ADDRESS extends LogKey
- case object BLOCK_GENERATOR_STATUS extends LogKey
- case object BLOCK_ID extends LogKey
- case object BLOCK_IDS extends LogKey
- case object BLOCK_MANAGER_ID extends LogKey
- case object BLOCK_MANAGER_IDS extends LogKey
- case object BLOCK_TYPE extends LogKey
- case object BOOT extends LogKey
- case object BOOTSTRAP_TIME extends LogKey
- case object BOOT_TIME extends LogKey
- case object BROADCAST extends LogKey
- case object BROADCAST_ID extends LogKey
- case object BROADCAST_OUTPUT_STATUS_SIZE extends LogKey
- case object BUCKET extends LogKey
- case object BYTECODE_SIZE extends LogKey
- case object BYTE_BUFFER extends LogKey
- case object BYTE_SIZE extends LogKey
- case object CACHED_TABLE_PARTITION_METADATA_SIZE extends LogKey
- case object CACHE_AUTO_REMOVED_SIZE extends LogKey
- case object CACHE_SIZE extends LogKey
- case object CACHE_UNTIL_HIGHEST_CONSUMED_SIZE extends LogKey
- case object CACHE_UNTIL_LAST_PRODUCED_SIZE extends LogKey
- case object CALL_SITE_LONG_FORM extends LogKey
- case object CALL_SITE_SHORT_FORM extends LogKey
- case object CANCEL_FUTURE_JOBS extends LogKey
- case object CATALOG_NAME extends LogKey
- case object CATEGORICAL_FEATURES extends LogKey
- case object CHECKPOINT_FILE extends LogKey
- case object CHECKPOINT_INTERVAL extends LogKey
- case object CHECKPOINT_LOCATION extends LogKey
- case object CHECKPOINT_PATH extends LogKey
- case object CHECKPOINT_ROOT extends LogKey
- case object CHECKPOINT_TIME extends LogKey
- case object CHOSEN_WATERMARK extends LogKey
- case object CLASSIFIER extends LogKey
- case object CLASS_LOADER extends LogKey
- case object CLASS_NAME extends LogKey
- case object CLASS_PATH extends LogKey
- case object CLASS_PATHS extends LogKey
- case object CLAUSES extends LogKey
- case object CLEANUP_LOCAL_DIRS extends LogKey
- case object CLUSTER_CENTROIDS extends LogKey
- case object CLUSTER_ID extends LogKey
- case object CLUSTER_LABEL extends LogKey
- case object CLUSTER_LEVEL extends LogKey
- case object CLUSTER_WEIGHT extends LogKey
- case object CODE extends LogKey
- case object CODEC_LEVEL extends LogKey
- case object CODEC_NAME extends LogKey
- case object CODEGEN_STAGE_ID extends LogKey
- case object COLUMN_DATA_TYPE_SOURCE extends LogKey
- case object COLUMN_DATA_TYPE_TARGET extends LogKey
- case object COLUMN_DEFAULT_VALUE extends LogKey
- case object COLUMN_NAME extends LogKey
- case object COMMAND extends LogKey
- case object COMMAND_OUTPUT extends LogKey
- case object COMMITTED_VERSION extends LogKey
- case object COMPACT_INTERVAL extends LogKey
- case object COMPONENT extends LogKey
- case object COMPUTE extends LogKey
- case object CONFIG extends LogKey
- case object CONFIG2 extends LogKey
- case object CONFIG3 extends LogKey
- case object CONFIG4 extends LogKey
- case object CONFIG5 extends LogKey
- case object CONFIG_DEPRECATION_MESSAGE extends LogKey
- case object CONFIG_KEY_UPDATED extends LogKey
- case object CONFIG_VERSION extends LogKey
- case object CONSUMER extends LogKey
- case object CONTAINER extends LogKey
- case object CONTAINER_ID extends LogKey
- case object CONTAINER_STATE extends LogKey
- case object CONTEXT extends LogKey
- case object COST extends LogKey
- case object COUNT extends LogKey
- case object CREATED_POOL_NAME extends LogKey
- case object CREATION_SITE extends LogKey
- case object CREDENTIALS_RENEWAL_INTERVAL_RATIO extends LogKey
- case object CROSS_VALIDATION_METRIC extends LogKey
- case object CROSS_VALIDATION_METRICS extends LogKey
- case object CSV_HEADER_COLUMN_NAME extends LogKey
- case object CSV_HEADER_COLUMN_NAMES extends LogKey
- case object CSV_HEADER_LENGTH extends LogKey
- case object CSV_SCHEMA_FIELD_NAME extends LogKey
- case object CSV_SCHEMA_FIELD_NAMES extends LogKey
- case object CSV_SOURCE extends LogKey
- case object CURRENT_BATCH_ID extends LogKey
- case object CURRENT_DISK_SIZE extends LogKey
- case object CURRENT_FILE extends LogKey
- case object CURRENT_MEMORY_SIZE extends LogKey
- case object CURRENT_PATH extends LogKey
- case object CURRENT_TIME extends LogKey
- case object DATA extends LogKey
- case object DATABASE_NAME extends LogKey
- case object DATAFRAME_CACHE_ENTRY extends LogKey
- case object DATAFRAME_ID extends LogKey
- case object DATA_FILE extends LogKey
- case object DATA_SOURCE extends LogKey
- case object DATA_SOURCES extends LogKey
- case object DEFAULT_COMPACT_INTERVAL extends LogKey
- case object DEFAULT_ISOLATION_LEVEL extends LogKey
- case object DEFAULT_NAME extends LogKey
- case object DEFAULT_VALUE extends LogKey
- case object DELAY extends LogKey
- case object DELEGATE extends LogKey
- case object DELTA extends LogKey
- case object DEPRECATED_KEY extends LogKey
- case object DERIVATIVE extends LogKey
- case object DESCRIPTION extends LogKey
- case object DESIRED_NUM_PARTITIONS extends LogKey
- case object DESIRED_TREE_DEPTH extends LogKey
- case object DESTINATION_PATH extends LogKey
- case object DFS_FILE extends LogKey
- case object DIFF_DELTA extends LogKey
- case object DIVISIBLE_CLUSTER_INDICES_SIZE extends LogKey
- case object DRIVER_ID extends LogKey
- case object DRIVER_MEMORY_SIZE extends LogKey
- case object DRIVER_STATE extends LogKey
- case object DROPPED_PARTITIONS extends LogKey
- case object DSTREAM extends LogKey
- case object DURATION extends LogKey
- case object EARLIEST_LOADED_VERSION extends LogKey
- case object EFFECTIVE_STORAGE_LEVEL extends LogKey
- case object ELAPSED_TIME extends LogKey
- case object ENCODING extends LogKey
- case object ENDPOINT_NAME extends LogKey
- case object END_INDEX extends LogKey
- case object END_POINT extends LogKey
- case object END_VERSION extends LogKey
- case object ENGINE extends LogKey
- case object EPOCH extends LogKey
- case object ERROR extends LogKey
- case object ESTIMATOR_PARAM_MAP extends LogKey
- case object EVALUATED_FILTERS extends LogKey
- case object EVENT extends LogKey
- case object EVENT_LOG_DESTINATION extends LogKey
- case object EVENT_LOOP extends LogKey
- case object EVENT_NAME extends LogKey
- case object EVENT_QUEUE extends LogKey
- case object EXCEPTION extends LogKey
- case object EXECUTE_INFO extends LogKey
- case object EXECUTE_KEY extends LogKey
- case object EXECUTION_MEMORY_SIZE extends LogKey
- case object EXECUTION_PLAN_LEAVES extends LogKey
- case object EXECUTOR_BACKEND extends LogKey
- case object EXECUTOR_ENVS extends LogKey
- case object EXECUTOR_ENV_REGEX extends LogKey
- case object EXECUTOR_ID extends LogKey
- case object EXECUTOR_IDS extends LogKey
- case object EXECUTOR_LAUNCH_COMMANDS extends LogKey
- case object EXECUTOR_MEMORY_OFFHEAP extends LogKey
- case object EXECUTOR_MEMORY_OVERHEAD_SIZE extends LogKey
- case object EXECUTOR_MEMORY_SIZE extends LogKey
- case object EXECUTOR_RESOURCES extends LogKey
- case object EXECUTOR_SHUFFLE_INFO extends LogKey
- case object EXECUTOR_STATE extends LogKey
- case object EXECUTOR_TIMEOUT extends LogKey
- case object EXECUTOR_USER_CLASS_PATH_FIRST extends LogKey
- case object EXEC_AMOUNT extends LogKey
- case object EXISTING_FILE extends LogKey
- case object EXISTING_PATH extends LogKey
- case object EXIT_CODE extends LogKey
- case object EXPECTED_NUM_FILES extends LogKey
- case object EXPECTED_PARTITION_COLUMN extends LogKey
- case object EXPIRY_TIMESTAMP extends LogKey
- case object EXPR extends LogKey
- case object EXPR_TERMS extends LogKey
- case object EXTENDED_EXPLAIN_GENERATOR extends LogKey
- case object FAILED_STAGE extends LogKey
- case object FAILED_STAGE_NAME extends LogKey
- case object FAILURES extends LogKey
- case object FALLBACK_VERSION extends LogKey
- case object FEATURE_COLUMN extends LogKey
- case object FEATURE_DIMENSION extends LogKey
- case object FEATURE_NAME extends LogKey
- case object FETCH_SIZE extends LogKey
- case object FIELD_NAME extends LogKey
- case object FIELD_TYPE extends LogKey
- case object FILES extends LogKey
- case object FILE_ABSOLUTE_PATH extends LogKey
- case object FILE_END_OFFSET extends LogKey
- case object FILE_FORMAT extends LogKey
- case object FILE_FORMAT2 extends LogKey
- case object FILE_LENGTH_XATTR extends LogKey
- case object FILE_MODIFICATION_TIME extends LogKey
- case object FILE_NAME extends LogKey
- case object FILE_NAME2 extends LogKey
- case object FILE_NAME3 extends LogKey
- case object FILE_NAMES extends LogKey
- case object FILE_START_OFFSET extends LogKey
- case object FILE_SYSTEM extends LogKey
- case object FILE_VERSION extends LogKey
- case object FILTER extends LogKey
- case object FINAL_CONTEXT extends LogKey
- case object FINAL_OUTPUT_PATH extends LogKey
- case object FINAL_PATH extends LogKey
- case object FINISH_TIME extends LogKey
- case object FINISH_TRIGGER_DURATION extends LogKey
- case object FLOW_NAME extends LogKey
- case object FREE_MEMORY_SIZE extends LogKey
- case object FROM_OFFSET extends LogKey
- case object FROM_TIME extends LogKey
- case object FS_DATA_OUTPUT_STREAM extends LogKey
- case object FUNCTION_NAME extends LogKey
- case object FUNCTION_PARAM extends LogKey
- case object GLOBAL_INIT_FILE extends LogKey
- case object GLOBAL_WATERMARK extends LogKey
- case object GROUP_BY_EXPRS extends LogKey
- case object GROUP_ID extends LogKey
- case object HADOOP_VERSION extends LogKey
- case object HASH_JOIN_KEYS extends LogKey
- case object HASH_MAP_SIZE extends LogKey
- case object HEARTBEAT extends LogKey
- case object HEARTBEAT_INTERVAL extends LogKey
- case object HISTORY_DIR extends LogKey
- case object HIVE_CLIENT_VERSION extends LogKey
- case object HIVE_METASTORE_VERSION extends LogKey
- case object HIVE_OPERATION_STATE extends LogKey
- case object HIVE_OPERATION_TYPE extends LogKey
- case object HOST extends LogKey
- case object HOSTS extends LogKey
- case object HOST_LOCAL_BLOCKS_SIZE extends LogKey
- case object HOST_PORT extends LogKey
- case object HOST_PORT2 extends LogKey
- case object HUGE_METHOD_LIMIT extends LogKey
- case object HYBRID_STORE_DISK_BACKEND extends LogKey
- case object IDENTIFIER extends LogKey
- case object INCOMPATIBLE_TYPES extends LogKey
- case object INDEX extends LogKey
- case object INDEX_FILE extends LogKey
- case object INDEX_NAME extends LogKey
- case object INFERENCE_MODE extends LogKey
- case object INIT extends LogKey
- case object INITIAL_CAPACITY extends LogKey
- case object INITIAL_HEARTBEAT_INTERVAL extends LogKey
- case object INIT_MODE extends LogKey
- case object INIT_TIME extends LogKey
- case object INPUT extends LogKey
- case object INPUT_SPLIT extends LogKey
- case object INTEGRAL extends LogKey
- case object INTERRUPT_THREAD extends LogKey
- case object INTERVAL extends LogKey
- case object INVALID_PARAMS extends LogKey
- case object ISOLATION_LEVEL extends LogKey
- case object ISSUE_DATE extends LogKey
- case object IS_NETWORK_REQUEST_DONE extends LogKey
- case object JAR_ENTRY extends LogKey
- case object JAR_MESSAGE extends LogKey
- case object JAR_URL extends LogKey
- case object JAVA_VERSION extends LogKey
- case object JAVA_VM_NAME extends LogKey
- case object JOB_ID extends LogKey
- case object JOIN_CONDITION extends LogKey
- case object JOIN_CONDITION_SUB_EXPR extends LogKey
- case object JOIN_TYPE extends LogKey
- case object K8S_CONTEXT extends LogKey
- case object KEY extends LogKey
- case object KEY2 extends LogKey
- case object KEYTAB extends LogKey
- case object KEYTAB_FILE extends LogKey
- case object KILL_EXECUTORS extends LogKey
- case object KINESIS_REASON extends LogKey
- case object LABEL_COLUMN extends LogKey
- case object LARGEST_CLUSTER_INDEX extends LogKey
- case object LAST_ACCESS_TIME extends LogKey
- case object LAST_COMMITTED_CHECKPOINT_ID extends LogKey
- case object LAST_COMMIT_BASED_CHECKPOINT_ID extends LogKey
- case object LAST_VALID_TIME extends LogKey
- case object LATEST_BATCH_ID extends LogKey
- case object LATEST_COMMITTED_BATCH_ID extends LogKey
- case object LATEST_SHUFFLE_MERGE_ID extends LogKey
- case object LEARNING_RATE extends LogKey
- case object LEFT_EXPR extends LogKey
- case object LEFT_LOGICAL_PLAN_STATS_SIZE_IN_BYTES extends LogKey
- case object LINE extends LogKey
- case object LINEAGE extends LogKey
- case object LINE_NUM extends LogKey
- case object LISTENER extends LogKey
- case object LOADED_CHECKPOINT_ID extends LogKey
- case object LOADED_VERSION extends LogKey
- case object LOAD_FACTOR extends LogKey
- case object LOAD_TIME extends LogKey
- case object LOCALE extends LogKey
- case object LOCAL_BLOCKS_SIZE extends LogKey
- case object LOCAL_SCRATCH_DIR extends LogKey
- case object LOCATION extends LogKey
- case object LOGICAL_PLAN extends LogKey
- case object LOGICAL_PLAN_COLUMNS extends LogKey
- case object LOGICAL_PLAN_LEAVES extends LogKey
- case object LOG_ID extends LogKey
- case object LOG_LEVEL extends LogKey
- case object LOG_OFFSET extends LogKey
- case object LOG_TYPE extends LogKey
- case object LOSSES extends LogKey
- case object LOWER_BOUND extends LogKey
- case object MALFORMATTED_STRING extends LogKey
- case object MAP_ID extends LogKey
- case object MASTER_URL extends LogKey
- case object MAX_ATTEMPTS extends LogKey
- case object MAX_CACHE_UNTIL_HIGHEST_CONSUMED_SIZE extends LogKey
- case object MAX_CACHE_UNTIL_LAST_PRODUCED_SIZE extends LogKey
- case object MAX_CAPACITY extends LogKey
- case object MAX_CATEGORIES extends LogKey
- case object MAX_EXECUTOR_FAILURES extends LogKey
- case object MAX_FILE_VERSION extends LogKey
- case object MAX_JVM_METHOD_PARAMS_LENGTH extends LogKey
- case object MAX_MEMORY_SIZE extends LogKey
- case object MAX_METHOD_CODE_SIZE extends LogKey
- case object MAX_NUM_BINS extends LogKey
- case object MAX_NUM_CHUNKS extends LogKey
- case object MAX_NUM_FILES extends LogKey
- case object MAX_NUM_LOG_POLICY extends LogKey
- case object MAX_NUM_PARTITIONS extends LogKey
- case object MAX_NUM_POSSIBLE_BINS extends LogKey
- case object MAX_NUM_ROWS_IN_MEMORY_BUFFER extends LogKey
- case object MAX_SEEN_VERSION extends LogKey
- case object MAX_SERVICE_NAME_LENGTH extends LogKey
- case object MAX_SIZE extends LogKey
- case object MAX_SLOTS extends LogKey
- case object MAX_SPLIT_BYTES extends LogKey
- case object MAX_TABLE_PARTITION_METADATA_SIZE extends LogKey
- case object MEMORY_CONSUMER extends LogKey
- case object MEMORY_POOL_NAME extends LogKey
- case object MEMORY_SIZE extends LogKey
- case object MEMORY_THRESHOLD_SIZE extends LogKey
- case object MERGE_DIR_NAME extends LogKey
- case object MESSAGE extends LogKey
- case object METADATA extends LogKey
- case object METADATA_DIRECTORY extends LogKey
- case object METADATA_JSON extends LogKey
- case object META_FILE extends LogKey
- case object METHOD_NAME extends LogKey
- case object METHOD_PARAM_TYPES extends LogKey
- case object METRICS_JSON extends LogKey
- case object METRIC_NAME extends LogKey
- case object MINI_BATCH_FRACTION extends LogKey
- case object MIN_COMPACTION_BATCH_ID extends LogKey
- case object MIN_NUM_FREQUENT_PATTERN extends LogKey
- case object MIN_POINT_PER_CLUSTER extends LogKey
- case object MIN_RATE extends LogKey
- case object MIN_SEEN_VERSION extends LogKey
- case object MIN_SHARE extends LogKey
- case object MIN_SIZE extends LogKey
- case object MIN_TIME extends LogKey
- case object MIN_VERSIONS_TO_DELETE extends LogKey
- case object MIN_VERSION_NUM extends LogKey
- case object MISSING_PARENT_STAGES extends LogKey
- case object MODEL_WEIGHTS extends LogKey
- case object MODIFY_ACLS extends LogKey
- case object MODIFY_ACLS_GROUPS extends LogKey
- case object MODULE_NAME extends LogKey
- case object NAME extends LogKey
- case object NAMESPACE extends LogKey
- case object NETWORK_IF extends LogKey
- case object NEW_FEATURE_COLUMN_NAME extends LogKey
- case object NEW_LABEL_COLUMN_NAME extends LogKey
- case object NEW_PATH extends LogKey
- case object NEW_RDD_ID extends LogKey
- case object NEW_STATE extends LogKey
- case object NEW_VALUE extends LogKey
- case object NEXT_RENEWAL_TIME extends LogKey
- case object NODES extends LogKey
- case object NODE_LOCATION extends LogKey
- case object NON_BUILT_IN_CONNECTORS extends LogKey
- case object NORM extends LogKey
- case object NUM_ADDED_PARTITIONS extends LogKey
- case object NUM_APPS extends LogKey
- case object NUM_ATTEMPT extends LogKey
- case object NUM_BATCHES extends LogKey
- case object NUM_BIN extends LogKey
- case object NUM_BLOCKS extends LogKey
- case object NUM_BLOCK_IDS extends LogKey
- case object NUM_BROADCAST_BLOCK extends LogKey
- case object NUM_BYTES extends LogKey
- case object NUM_BYTES_CURRENT extends LogKey
- case object NUM_BYTES_EVICTED extends LogKey
- case object NUM_BYTES_MAX extends LogKey
- case object NUM_BYTES_TO_FREE extends LogKey
- case object NUM_BYTES_TO_WARN extends LogKey
- case object NUM_BYTES_USED extends LogKey
- case object NUM_CATEGORIES extends LogKey
- case object NUM_CHECKSUM_FILE extends LogKey
- case object NUM_CHUNKS extends LogKey
- case object NUM_CLASSES extends LogKey
- case object NUM_COEFFICIENTS extends LogKey
- case object NUM_COLUMNS extends LogKey
- case object NUM_CONCURRENT_WRITER extends LogKey
- case object NUM_CORES extends LogKey
- case object NUM_DATA_FILE extends LogKey
- case object NUM_DATA_FILES extends LogKey
- case object NUM_DECOMMISSIONED extends LogKey
- case object NUM_DRIVERS extends LogKey
- case object NUM_DROPPED_PARTITIONS extends LogKey
- case object NUM_EFFECTIVE_RULE_OF_RUNS extends LogKey
- case object NUM_ELEMENTS_SPILL_THRESHOLD extends LogKey
- case object NUM_EVENTS extends LogKey
- case object NUM_EXAMPLES extends LogKey
- case object NUM_EXECUTORS extends LogKey
- case object NUM_EXECUTORS_EXITED extends LogKey
- case object NUM_EXECUTORS_KILLED extends LogKey
- case object NUM_EXECUTOR_CORES extends LogKey
- case object NUM_EXECUTOR_CORES_REMAINING extends LogKey
- case object NUM_EXECUTOR_CORES_TOTAL extends LogKey
- case object NUM_EXECUTOR_DESIRED extends LogKey
- case object NUM_EXECUTOR_LAUNCH extends LogKey
- case object NUM_EXECUTOR_TARGET extends LogKey
- case object NUM_FAILURES extends LogKey
- case object NUM_FEATURES extends LogKey
- case object NUM_FILES extends LogKey
- case object NUM_FILES_COPIED extends LogKey
- case object NUM_FILES_FAILED_TO_DELETE extends LogKey
- case object NUM_FILES_REUSED extends LogKey
- case object NUM_FREQUENT_ITEMS extends LogKey
- case object NUM_HOST_LOCAL_BLOCKS extends LogKey
- case object NUM_INDEX_FILE extends LogKey
- case object NUM_INDEX_FILES extends LogKey
- case object NUM_ITERATIONS extends LogKey
- case object NUM_KAFKA_PULLS extends LogKey
- case object NUM_KAFKA_RECORDS_PULLED extends LogKey
- case object NUM_LAGGING_STORES extends LogKey
- case object NUM_LEADING_SINGULAR_VALUES extends LogKey
- case object NUM_LEFT_PARTITION_VALUES extends LogKey
- case object NUM_LOADED_ENTRIES extends LogKey
- case object NUM_LOCAL_BLOCKS extends LogKey
- case object NUM_LOCAL_DIRS extends LogKey
- case object NUM_LOCAL_FREQUENT_PATTERN extends LogKey
- case object NUM_MERGERS extends LogKey
- case object NUM_MERGER_LOCATIONS extends LogKey
- case object NUM_META_FILES extends LogKey
- case object NUM_NODES extends LogKey
- case object NUM_PARTITIONS extends LogKey
- case object NUM_PARTITIONS2 extends LogKey
- case object NUM_PATHS extends LogKey
- case object NUM_PEERS extends LogKey
- case object NUM_PEERS_REPLICATED_TO extends LogKey
- case object NUM_PEERS_TO_REPLICATE_TO extends LogKey
- case object NUM_PENDING_LAUNCH_TASKS extends LogKey
- case object NUM_POD extends LogKey
- case object NUM_POD_SHARED_SLOT extends LogKey
- case object NUM_POD_TARGET extends LogKey
- case object NUM_POINT extends LogKey
- case object NUM_PREFIXES extends LogKey
- case object NUM_PRUNED extends LogKey
- case object NUM_PUSH_MERGED_LOCAL_BLOCKS extends LogKey
- case object NUM_RECEIVERS extends LogKey
- case object NUM_RECORDS_READ extends LogKey
- case object NUM_RELEASED_LOCKS extends LogKey
- case object NUM_REMAINED extends LogKey
- case object NUM_REMOTE_BLOCKS extends LogKey
- case object NUM_REMOVED_WORKERS extends LogKey
- case object NUM_REPLICAS extends LogKey
- case object NUM_REQUESTS extends LogKey
- case object NUM_REQUEST_SYNC_TASK extends LogKey
- case object NUM_RESOURCE_SLOTS extends LogKey
- case object NUM_RETRIES extends LogKey
- case object NUM_RETRY extends LogKey
- case object NUM_RIGHT_PARTITION_VALUES extends LogKey
- case object NUM_ROWS extends LogKey
- case object NUM_RULE_OF_RUNS extends LogKey
- case object NUM_SEQUENCES extends LogKey
- case object NUM_SKIPPED extends LogKey
- case object NUM_SLOTS extends LogKey
- case object NUM_SPILLS extends LogKey
- case object NUM_SPILL_WRITERS extends LogKey
- case object NUM_SUB_DIRS extends LogKey
- case object NUM_SUCCESSFUL_TASKS extends LogKey
- case object NUM_TASKS extends LogKey
- case object NUM_TASK_CPUS extends LogKey
- case object NUM_TRAIN_WORD extends LogKey
- case object NUM_UNFINISHED_DECOMMISSIONED extends LogKey
- case object NUM_VERSIONS_RETAIN extends LogKey
- case object NUM_WEIGHTED_EXAMPLES extends LogKey
- case object NUM_WORKERS extends LogKey
- case object OBJECT_AGG_SORT_BASED_FALLBACK_THRESHOLD extends LogKey
- case object OBJECT_ID extends LogKey
- case object OFFSET extends LogKey
- case object OFFSETS extends LogKey
- case object OFFSET_SEQUENCE_METADATA extends LogKey
- case object OLD_BLOCK_MANAGER_ID extends LogKey
- case object OLD_GENERATION_GC extends LogKey
- case object OLD_VALUE extends LogKey
- case object OPEN_COST_IN_BYTES extends LogKey
- case object OPERATION_HANDLE extends LogKey
- case object OPERATION_HANDLE_ID extends LogKey
- case object OPERATION_ID extends LogKey
- case object OPTIMIZED_PLAN_COLUMNS extends LogKey
- case object OPTIMIZER_CLASS_NAME extends LogKey
- case object OPTIONS extends LogKey
- case object OP_ID extends LogKey
- case object OP_TYPE extends LogKey
- case object ORIGINAL_DISK_SIZE extends LogKey
- case object ORIGINAL_MEMORY_SIZE extends LogKey
- case object OS_ARCH extends LogKey
- case object OS_NAME extends LogKey
- case object OS_VERSION extends LogKey
- case object OUTPUT extends LogKey
- case object OUTPUT_BUFFER extends LogKey
- case object OVERHEAD_MEMORY_SIZE extends LogKey
- case object PAGE_SIZE extends LogKey
- case object PARENT_STAGES extends LogKey
- case object PARSE_MODE extends LogKey
- case object PARTITIONED_FILE_READER extends LogKey
- case object PARTITIONER extends LogKey
- case object PARTITION_ID extends LogKey
- case object PARTITION_IDS extends LogKey
- case object PARTITION_SIZE extends LogKey
- case object PARTITION_SPECIFICATION extends LogKey
- case object PARTITION_SPECS extends LogKey
- case object PATH extends LogKey
- case object PATHS extends LogKey
- case object PEER extends LogKey
- case object PENDING_TIMES extends LogKey
- case object PERCENT extends LogKey
- case object PIPELINE_STAGE_UID extends LogKey
- case object PLUGIN_NAME extends LogKey
- case object POD_ID extends LogKey
- case object POD_NAME extends LogKey
- case object POD_NAMESPACE extends LogKey
- case object POD_PHASE extends LogKey
- case object POD_STATE extends LogKey
- case object POINT_OF_CENTER extends LogKey
- case object POLICY extends LogKey
- case object POOL_NAME extends LogKey
- case object PORT extends LogKey
- case object PORT2 extends LogKey
- case object POST_SCAN_FILTERS extends LogKey
- case object PREDICATE extends LogKey
- case object PREDICATES extends LogKey
- case object PREFERRED_SERVICE_NAME extends LogKey
- case object PREFIX extends LogKey
- case object PRETTY_ID_STRING extends LogKey
- case object PRINCIPAL extends LogKey
- case object PROCESS extends LogKey
- case object PROCESSING_TIME extends LogKey
- case object PRODUCER_ID extends LogKey
- case object PROPERTY_NAME extends LogKey
- case object PROPORTIONAL extends LogKey
- case object PROTOCOL_VERSION extends LogKey
- case object PROVIDER extends LogKey
- case object PUSHED_FILTERS extends LogKey
- case object PUSH_MERGED_LOCAL_BLOCKS_SIZE extends LogKey
- case object PVC_METADATA_NAME extends LogKey
- case object PYTHON_EXEC extends LogKey
- case object PYTHON_PACKAGES extends LogKey
- case object PYTHON_VERSION extends LogKey
- case object PYTHON_WORKER_CHANNEL_IS_BLOCKING_MODE extends LogKey
- case object PYTHON_WORKER_CHANNEL_IS_CONNECTED extends LogKey
- case object PYTHON_WORKER_HAS_INPUTS extends LogKey
- case object PYTHON_WORKER_IDLE_TIMEOUT extends LogKey
- case object PYTHON_WORKER_IS_ALIVE extends LogKey
- case object PYTHON_WORKER_MODULE extends LogKey
- case object PYTHON_WORKER_RESPONSE extends LogKey
- case object PYTHON_WORKER_SELECTION_KEY_INTERESTS extends LogKey
- case object PYTHON_WORKER_SELECTION_KEY_IS_VALID extends LogKey
- case object PYTHON_WORKER_SELECTOR_IS_OPEN extends LogKey
- case object QUANTILES extends LogKey
- case object QUERY_CACHE_VALUE extends LogKey
- case object QUERY_HINT extends LogKey
- case object QUERY_ID extends LogKey
- case object QUERY_PLAN extends LogKey
- case object QUERY_PLAN_COMPARISON extends LogKey
- case object QUERY_PLAN_LENGTH_ACTUAL extends LogKey
- case object QUERY_PLAN_LENGTH_MAX extends LogKey
- case object QUERY_RUN_ID extends LogKey
- case object RANGE extends LogKey
- case object RATE_LIMIT extends LogKey
- case object RATIO extends LogKey
- case object RDD extends LogKey
- case object RDD_CHECKPOINT_DIR extends LogKey
- case object RDD_DEBUG_STRING extends LogKey
- case object RDD_DESCRIPTION extends LogKey
- case object RDD_ID extends LogKey
- case object READ_LIMIT extends LogKey
- case object REASON extends LogKey
- case object REATTACHABLE extends LogKey
- case object RECEIVED_BLOCK_INFO extends LogKey
- case object RECEIVED_BLOCK_TRACKER_LOG_EVENT extends LogKey
- case object RECEIVER_ID extends LogKey
- case object RECEIVER_IDS extends LogKey
- case object RECORDS extends LogKey
- case object RECOVERY_STATE extends LogKey
- case object RECURSIVE_DEPTH extends LogKey
- case object REDACTED_STATEMENT extends LogKey
- case object REDUCE_ID extends LogKey
- case object REGEX extends LogKey
- case object REGISTERED_EXECUTOR_FILE extends LogKey
- case object REGISTER_MERGE_RESULTS extends LogKey
- case object RELATION_NAME extends LogKey
- case object RELATION_OUTPUT extends LogKey
- case object RELATIVE_TOLERANCE extends LogKey
- case object RELEASED_LOCKS extends LogKey
- case object REMAINING_PARTITIONS extends LogKey
- case object REMOTE_ADDRESS extends LogKey
- case object REMOTE_BLOCKS_SIZE extends LogKey
- case object REMOVE_FROM_MASTER extends LogKey
- case object REPORT_DETAILS extends LogKey
- case object REQUESTER_SIZE extends LogKey
- case object REQUEST_EXECUTORS extends LogKey
- case object REQUEST_ID extends LogKey
- case object RESOURCE extends LogKey
- case object RESOURCE_NAME extends LogKey
- case object RESOURCE_PROFILE_ID extends LogKey
- case object RESOURCE_PROFILE_IDS extends LogKey
- case object RESOURCE_PROFILE_TO_TOTAL_EXECS extends LogKey
- case object RESPONSE_BODY_SIZE extends LogKey
- case object RESTART_TIME extends LogKey
- case object RESULT extends LogKey
- case object RESULT_SIZE_BYTES extends LogKey
- case object RESULT_SIZE_BYTES_MAX extends LogKey
- case object RETRY_INTERVAL extends LogKey
- case object RETRY_WAIT_TIME extends LogKey
- case object RIGHT_EXPR extends LogKey
- case object RIGHT_LOGICAL_PLAN_STATS_SIZE_IN_BYTES extends LogKey
- case object RMSE extends LogKey
- case object ROCKS_DB_FILE_MAPPING extends LogKey
- case object ROCKS_DB_LOG_LEVEL extends LogKey
- case object ROCKS_DB_LOG_MESSAGE extends LogKey
- case object RPC_ADDRESS extends LogKey
- case object RPC_ENDPOINT_REF extends LogKey
- case object RPC_MESSAGE_CAPACITY extends LogKey
- case object RPC_SSL_ENABLED extends LogKey
- case object RULE_EXECUTOR_NAME extends LogKey
- case object RULE_NAME extends LogKey
- case object RUN_ID extends LogKey
- case object RUN_ID_STRING extends LogKey
- case object SCALA_VERSION extends LogKey
- case object SCALING_DOWN_RATIO extends LogKey
- case object SCALING_UP_RATIO extends LogKey
- case object SCHEDULER_POOL_NAME extends LogKey
- case object SCHEDULING_MODE extends LogKey
- case object SCHEMA extends LogKey
- case object SCHEMA2 extends LogKey
- case object SERVER_NAME extends LogKey
- case object SERVICE_NAME extends LogKey
- case object SERVLET_CONTEXT_HANDLER_PATH extends LogKey
- case object SESSION_HANDLE extends LogKey
- case object SESSION_HOLD_INFO extends LogKey
- case object SESSION_ID extends LogKey
- case object SESSION_KEY extends LogKey
- case object SET_CLIENT_INFO_REQUEST extends LogKey
- case object SHARD_ID extends LogKey
- case object SHORTER_SERVICE_NAME extends LogKey
- case object SHORT_USER_NAME extends LogKey
- case object SHUFFLE_BLOCK_INFO extends LogKey
- case object SHUFFLE_DB_BACKEND_KEY extends LogKey
- case object SHUFFLE_DB_BACKEND_NAME extends LogKey
- case object SHUFFLE_ID extends LogKey
- case object SHUFFLE_IDS extends LogKey
- case object SHUFFLE_MERGE_ID extends LogKey
- case object SHUFFLE_MERGE_RECOVERY_FILE extends LogKey
- case object SHUFFLE_SERVICE_CONF_OVERLAY_URL extends LogKey
- case object SHUFFLE_SERVICE_METRICS_NAMESPACE extends LogKey
- case object SHUFFLE_SERVICE_NAME extends LogKey
- case object SIGMAS_LENGTH extends LogKey
- case object SIGNAL extends LogKey
- case object SINK extends LogKey
- case object SIZE extends LogKey
- case object SLEEP_TIME extends LogKey
- case object SLIDE_DURATION extends LogKey
- case object SMALLEST_CLUSTER_INDEX extends LogKey
- case object SNAPSHOT_EVENT extends LogKey
- case object SNAPSHOT_EVENT_TIME_DELTA extends LogKey
- case object SNAPSHOT_EVENT_VERSION_DELTA extends LogKey
- case object SNAPSHOT_VERSION extends LogKey
- case object SOCKET_ADDRESS extends LogKey
- case object SOURCE extends LogKey
- case object SOURCE_PATH extends LogKey
- case object SPARK_BRANCH extends LogKey
- case object SPARK_BUILD_DATE extends LogKey
- case object SPARK_BUILD_USER extends LogKey
- case object SPARK_DATA_STREAM extends LogKey
- case object SPARK_PLAN_ID extends LogKey
- case object SPARK_REPO_URL extends LogKey
- case object SPARK_REVISION extends LogKey
- case object SPARK_VERSION extends LogKey
- case object SPILL_TIMES extends LogKey
- case object SQL_TEXT extends LogKey
- case object SRC_PATH extends LogKey
- case object STAGE extends LogKey
- case object STAGES extends LogKey
- case object STAGE_ATTEMPT extends LogKey
- case object STAGE_ATTEMPT_ID extends LogKey
- case object STAGE_ID extends LogKey
- case object STAGE_NAME extends LogKey
- case object START_INDEX extends LogKey
- case object START_TIME extends LogKey
- case object STATEMENT_ID extends LogKey
- case object STATE_NAME extends LogKey
- case object STATE_STORE_COORDINATOR extends LogKey
- case object STATE_STORE_ID extends LogKey
- case object STATE_STORE_PROVIDER extends LogKey
- case object STATE_STORE_PROVIDER_ID extends LogKey
- case object STATE_STORE_PROVIDER_IDS extends LogKey
- case object STATE_STORE_VERSION extends LogKey
- case object STATS extends LogKey
- case object STATUS extends LogKey
- case object STDERR extends LogKey
- case object STOP_SITE_SHORT_FORM extends LogKey
- case object STORAGE_LEVEL extends LogKey
- case object STORAGE_LEVEL_DESERIALIZED extends LogKey
- case object STORAGE_LEVEL_REPLICATION extends LogKey
- case object STORAGE_MEMORY_SIZE extends LogKey
- case object STORE_ID extends LogKey
- case object STRATEGY extends LogKey
- case object STREAMING_CONTEXT extends LogKey
- case object STREAMING_DATA_SOURCE_DESCRIPTION extends LogKey
- case object STREAMING_DATA_SOURCE_NAME extends LogKey
- case object STREAMING_OFFSETS_END extends LogKey
- case object STREAMING_OFFSETS_START extends LogKey
- case object STREAMING_QUERY_PROGRESS extends LogKey
- case object STREAMING_SOURCE extends LogKey
- case object STREAMING_TABLE extends LogKey
- case object STREAMING_WRITE extends LogKey
- case object STREAM_CHUNK_ID extends LogKey
- case object STREAM_ID extends LogKey
- case object STREAM_NAME extends LogKey
- case object SUBMISSION_ID extends LogKey
- case object SUBSAMPLING_RATE extends LogKey
- case object SUB_QUERY extends LogKey
- case object TABLE_NAME extends LogKey
- case object TABLE_TYPE extends LogKey
- case object TABLE_TYPES extends LogKey
- case object TAG extends LogKey
- case object TARGET_NUM_EXECUTOR extends LogKey
- case object TARGET_NUM_EXECUTOR_DELTA extends LogKey
- case object TARGET_PATH extends LogKey
- case object TARGET_SIZE extends LogKey
- case object TASK_ATTEMPT_ID extends LogKey
- case object TASK_ID extends LogKey
- case object TASK_INDEX extends LogKey
- case object TASK_LOCALITY extends LogKey
- case object TASK_NAME extends LogKey
- case object TASK_REQUIREMENTS extends LogKey
- case object TASK_RESOURCES extends LogKey
- case object TASK_RESOURCE_ASSIGNMENTS extends LogKey
- case object TASK_SET_MANAGER extends LogKey
- case object TASK_SET_NAME extends LogKey
- case object TASK_STATE extends LogKey
- case object TEMP_FILE extends LogKey
- case object TEMP_OUTPUT_PATH extends LogKey
- case object TEMP_PATH extends LogKey
- case object TEST_SIZE extends LogKey
- case object THREAD extends LogKey
- case object THREAD_ID extends LogKey
- case object THREAD_NAME extends LogKey
- case object THREAD_POOL_KEEPALIVE_TIME extends LogKey
- case object THREAD_POOL_SIZE extends LogKey
- case object THREAD_POOL_WAIT_QUEUE_SIZE extends LogKey
- case object THRESHOLD extends LogKey
- case object THRESH_TIME extends LogKey
- case object TIME extends LogKey
- case object TIMEOUT extends LogKey
- case object TIMER extends LogKey
- case object TIMESTAMP extends LogKey
- case object TIME_UNITS extends LogKey
- case object TIP extends LogKey
- case object TOKEN extends LogKey
- case object TOKEN_KIND extends LogKey
- case object TOKEN_REGEX extends LogKey
- case object TOKEN_RENEWER extends LogKey
- case object TOPIC extends LogKey
- case object TOPIC_PARTITION extends LogKey
- case object TOPIC_PARTITIONS extends LogKey
- case object TOPIC_PARTITION_OFFSET extends LogKey
- case object TOPIC_PARTITION_OFFSET_RANGE extends LogKey
- case object TOTAL extends LogKey
- case object TOTAL_EFFECTIVE_TIME extends LogKey
- case object TOTAL_SIZE extends LogKey
- case object TOTAL_TIME extends LogKey
- case object TOTAL_TIME_READ extends LogKey
- case object TO_TIME extends LogKey
- case object TRAINING_SIZE extends LogKey
- case object TRAIN_VALIDATION_SPLIT_METRIC extends LogKey
- case object TRAIN_VALIDATION_SPLIT_METRICS extends LogKey
- case object TRANSFER_TYPE extends LogKey
- case object TREE_NODE extends LogKey
- case object TRIGGER_INTERVAL extends LogKey
- case object UI_ACLS extends LogKey
- case object UI_FILTER extends LogKey
- case object UI_FILTER_PARAMS extends LogKey
- case object UI_PROXY_BASE extends LogKey
- case object UNKNOWN_PARAM extends LogKey
- case object UNSUPPORTED_EXPR extends LogKey
- case object UNSUPPORTED_HINT_REASON extends LogKey
- case object UNTIL_OFFSET extends LogKey
- case object UPPER_BOUND extends LogKey
- case object URI extends LogKey
- case object URIS extends LogKey
- case object URL extends LogKey
- case object URL2 extends LogKey
- case object URLS extends LogKey
- case object USER_ID extends LogKey
- case object USER_NAME extends LogKey
- case object UUID extends LogKey
- case object VALUE extends LogKey
- case object VERSIONS_TO_DELETE extends LogKey
- case object VERSION_NUM extends LogKey
- case object VIEW_ACLS extends LogKey
- case object VIEW_ACLS_GROUPS extends LogKey
- case object VIRTUAL_CORES extends LogKey
- case object VOCAB_SIZE extends LogKey
- case object WAIT_RESULT_TIME extends LogKey
- case object WAIT_SEND_TIME extends LogKey
- case object WATERMARK_CONSTRAINT extends LogKey
- case object WEB_URL extends LogKey
- case object WEIGHT extends LogKey
- case object WORKER extends LogKey
- case object WORKER_HOST extends LogKey
- case object WORKER_ID extends LogKey
- case object WORKER_PORT extends LogKey
- case object WORKER_URL extends LogKey
- case object WRITE_AHEAD_LOG_INFO extends LogKey
- case object WRITE_AHEAD_LOG_RECORD_HANDLE extends LogKey
- case object WRITE_JOB_UUID extends LogKey
- case object XML_SCHEDULING_MODE extends LogKey
- case object XSD_PATH extends LogKey
- case object YARN_RESOURCE extends LogKey
- case object YOUNG_GENERATION_GC extends LogKey
- case object ZERO_TIME extends LogKey
-}
diff --git a/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala b/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala
index cc5d0281829d0..810bdabebb38a 100644
--- a/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala
+++ b/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala
@@ -17,6 +17,8 @@
package org.apache.spark.internal
+import java.util.Locale
+
import scala.concurrent.duration._
import scala.jdk.CollectionConverters._
@@ -29,7 +31,6 @@ import org.slf4j.{Logger, LoggerFactory}
import org.slf4j.event.{Level => Slf4jLevel}
import org.apache.spark.internal.Logging.SparkShellLoggingFilter
-import org.apache.spark.internal.LogKeys
import org.apache.spark.util.SparkClassUtils
/**
@@ -61,29 +62,16 @@ import org.apache.spark.util.SparkClassUtils
*
*
* If you want to output logs in `scala code` through the structured log framework,
- * you can define `custom LogKey` and use it in `scala` code as follows:
+ * you can define `custom LogKey` in `java` and use it in `scala` code as follows:
*
*
- * // To add a `custom LogKey`, implement `LogKey`
- * case object CUSTOM_LOG_KEY extends LogKey
- * import org.apache.spark.internal.MDC;
+ * // Add a `CustomLogKeys`, implement `LogKey`
+ * public enum CustomLogKeys implements LogKey {
+ * CUSTOM_LOG_KEY
+ * }
* logInfo(log"${MDC(CUSTOM_LOG_KEY, "key")}")
*/
-/**
- * Mapped Diagnostic Context (MDC) that will be used in log messages.
- * The values of the MDC will be inline in the log message, while the key-value pairs will be
- * part of the ThreadContext.
- */
-case class MDC(key: LogKey, value: Any) {
- require(!value.isInstanceOf[MessageWithContext],
- "the class of value cannot be MessageWithContext")
-}
-
-object MDC {
- def of(key: LogKey, value: Any): MDC = MDC(key, value)
-}
-
/**
* Wrapper class for log messages that include a logging context.
* This is used as the return type of the string interpolator `LogStringContext`.
@@ -155,7 +143,7 @@ trait Logging {
val value = if (mdc.value != null) mdc.value.toString else null
sb.append(value)
if (Logging.isStructuredLoggingEnabled) {
- context.put(mdc.key.name, value)
+ context.put(mdc.key.name.toLowerCase(Locale.ROOT), value)
}
if (processedParts.hasNext) {
@@ -182,6 +170,12 @@ trait Logging {
}
}
+ protected def MDC(key: LogKey, value: Any): MDC = {
+ require(!value.isInstanceOf[MessageWithContext],
+ "the class of value cannot be MessageWithContext")
+ new MDC(key, value)
+ }
+
// Log methods that take only a String
protected def logInfo(msg: => String): Unit = {
if (log.isInfoEnabled) log.info(msg)
@@ -411,7 +405,6 @@ private[spark] object Logging {
@volatile private var initialized = false
@volatile private var defaultRootLevel: Level = null
@volatile private var defaultSparkLog4jConfig = false
- @volatile private var structuredLoggingEnabled = false
@volatile private[spark] var sparkShellThresholdLevel: Level = null
@volatile private[spark] var setLogLevelPrinted: Boolean = false
@@ -484,21 +477,21 @@ private[spark] object Logging {
* Enable Structured logging framework.
*/
private[spark] def enableStructuredLogging(): Unit = {
- structuredLoggingEnabled = true
+ SparkLoggerFactory.enableStructuredLogging()
}
/**
* Disable Structured logging framework.
*/
private[spark] def disableStructuredLogging(): Unit = {
- structuredLoggingEnabled = false
+ SparkLoggerFactory.disableStructuredLogging()
}
/**
* Return true if Structured logging framework is enabled.
*/
private[spark] def isStructuredLoggingEnabled: Boolean = {
- structuredLoggingEnabled
+ SparkLoggerFactory.isStructuredLoggingEnabled
}
private[spark] class SparkShellLoggingFilter extends AbstractFilter {
diff --git a/common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala b/common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
index 5ea3c9afa9c10..db6953031fd2c 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
@@ -24,7 +24,6 @@ import java.lang.reflect.{Field, Modifier}
import scala.collection.mutable.{Map, Queue, Set, Stack}
import scala.jdk.CollectionConverters._
-import org.apache.commons.lang3.ClassUtils
import org.apache.xbean.asm9.{ClassReader, ClassVisitor, Handle, MethodVisitor, Type}
import org.apache.xbean.asm9.Opcodes._
import org.apache.xbean.asm9.tree.{ClassNode, MethodNode}
@@ -619,7 +618,7 @@ private[spark] object IndylambdaScalaClosures extends Logging {
def getSerializationProxy(maybeClosure: AnyRef): Option[SerializedLambda] = {
def isClosureCandidate(cls: Class[_]): Boolean = {
// TODO: maybe lift this restriction to support other functional interfaces in the future
- val implementedInterfaces = ClassUtils.getAllInterfaces(cls).asScala
+ val implementedInterfaces = SparkClassUtils.getAllInterfaces(cls)
implementedInterfaces.exists(_.getName.startsWith("scala.Function"))
}
diff --git a/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala b/common/utils/src/main/scala/org/apache/spark/util/CompletionIterator.scala
similarity index 100%
rename from core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
rename to common/utils/src/main/scala/org/apache/spark/util/CompletionIterator.scala
diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
index d54a2f2ed9cea..5e923ad35a3ab 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala
@@ -22,7 +22,6 @@ import java.net.URI
import java.text.ParseException
import java.util.UUID
-import org.apache.commons.lang3.StringUtils
import org.apache.ivy.Ivy
import org.apache.ivy.core.LogOptions
import org.apache.ivy.core.module.descriptor.{Artifact, DefaultDependencyDescriptor, DefaultExcludeRule, DefaultModuleDescriptor, ExcludeRule}
@@ -36,7 +35,7 @@ import org.apache.ivy.plugins.repository.file.FileRepository
import org.apache.ivy.plugins.resolver.{ChainResolver, FileSystemResolver, IBiblioResolver}
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.util.ArrayImplicits._
/** Provides utility functions to be used inside SparkSubmit. */
@@ -567,7 +566,7 @@ private[spark] object MavenUtils extends Logging {
}
private def isInvalidQueryString(tokens: Array[String]): Boolean = {
- tokens.length != 2 || StringUtils.isBlank(tokens(0)) || StringUtils.isBlank(tokens(1))
+ tokens.length != 2 || SparkStringUtils.isBlank(tokens(0)) || SparkStringUtils.isBlank(tokens(1))
}
/**
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkClassUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkClassUtils.scala
index 307006315a3c4..3f22719240c06 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkClassUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkClassUtils.scala
@@ -18,6 +18,7 @@ package org.apache.spark.util
import java.util.Random
+import scala.collection.mutable.LinkedHashSet
import scala.util.Try
private[spark] trait SparkClassUtils {
@@ -136,6 +137,33 @@ private[spark] trait SparkClassUtils {
}
}
}
+
+ /**
+ * Gets a list of all interfaces implemented by the given class and its superclasses.
+ */
+ def getAllInterfaces(cls: Class[_]): List[Class[_]] = {
+ if (cls == null) {
+ return null
+ }
+ val interfacesFound = LinkedHashSet[Class[_]]()
+ getAllInterfacesHelper(cls, interfacesFound)
+ interfacesFound.toList
+ }
+
+ private def getAllInterfacesHelper(
+ clazz: Class[_],
+ interfacesFound: LinkedHashSet[Class[_]]): Unit = {
+ var currentClass = clazz
+ while (currentClass != null) {
+ val interfaces = currentClass.getInterfaces
+ for (i <- interfaces) {
+ if (interfacesFound.add(i)) {
+ getAllInterfacesHelper(i, interfacesFound)
+ }
+ }
+ currentClass = currentClass.getSuperclass
+ }
+ }
}
private[spark] object SparkClassUtils extends SparkClassUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkCollectionUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkCollectionUtils.scala
index eaab50117abdc..9c255daa0522d 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkCollectionUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkCollectionUtils.scala
@@ -16,7 +16,10 @@
*/
package org.apache.spark.util
+import java.util.Arrays
+
import scala.collection.immutable
+import scala.reflect.ClassTag
private[spark] trait SparkCollectionUtils {
/**
@@ -32,6 +35,36 @@ private[spark] trait SparkCollectionUtils {
}
builder.result()
}
+
+ def isEmpty[K, V](map: java.util.Map[K, V]): Boolean = {
+ map == null || map.isEmpty()
+ }
+
+ def isNotEmpty[K, V](map: java.util.Map[K, V]): Boolean = !isEmpty(map)
+
+ def createArray[K: ClassTag](size: Int, defaultValue: K): Array[K] = {
+ val arr = Array.ofDim[K](size)
+ val classTag = implicitly[ClassTag[K]]
+ classTag.runtimeClass match {
+ case c if c == classOf[Boolean] =>
+ Arrays.fill(arr.asInstanceOf[Array[Boolean]], defaultValue.asInstanceOf[Boolean])
+ case c if c == classOf[Byte] =>
+ Arrays.fill(arr.asInstanceOf[Array[Byte]], defaultValue.asInstanceOf[Byte])
+ case c if c == classOf[Short] =>
+ Arrays.fill(arr.asInstanceOf[Array[Short]], defaultValue.asInstanceOf[Short])
+ case c if c == classOf[Int] =>
+ Arrays.fill(arr.asInstanceOf[Array[Int]], defaultValue.asInstanceOf[Int])
+ case c if c == classOf[Long] =>
+ Arrays.fill(arr.asInstanceOf[Array[Long]], defaultValue.asInstanceOf[Long])
+ case c if c == classOf[Float] =>
+ Arrays.fill(arr.asInstanceOf[Array[Float]], defaultValue.asInstanceOf[Float])
+ case c if c == classOf[Double] =>
+ Arrays.fill(arr.asInstanceOf[Array[Double]], defaultValue.asInstanceOf[Double])
+ case _ =>
+ Arrays.fill(arr.asInstanceOf[Array[AnyRef]], defaultValue.asInstanceOf[AnyRef])
+ }
+ arr
+ }
}
private[spark] object SparkCollectionUtils extends SparkCollectionUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkEnvUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkEnvUtils.scala
index b54e6ee5d7309..01e3f52de41f3 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkEnvUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkEnvUtils.scala
@@ -16,16 +16,14 @@
*/
package org.apache.spark.util
+import org.apache.spark.network.util.JavaUtils
+
private[spark] trait SparkEnvUtils {
/**
* Indicates whether Spark is currently running unit tests.
*/
- def isTesting: Boolean = {
- // Scala's `sys.env` creates a ton of garbage by constructing Scala immutable maps, so
- // we directly use the Java APIs instead.
- System.getenv("SPARK_TESTING") != null || System.getProperty("spark.testing") != null
- }
+ def isTesting: Boolean = JavaUtils.isTesting
}
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala
index 872c89e5a29a2..2f16c90ad7149 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkErrorUtils.scala
@@ -16,12 +16,14 @@
*/
package org.apache.spark.util
-import java.io.{Closeable, IOException, PrintWriter}
-import java.nio.charset.StandardCharsets.UTF_8
+import java.io.{Closeable, IOException}
+import scala.annotation.tailrec
+import scala.collection.mutable
import scala.util.control.NonFatal
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
+import org.apache.spark.network.util.JavaUtils
private[spark] trait SparkErrorUtils extends Logging {
/**
@@ -45,7 +47,11 @@ private[spark] trait SparkErrorUtils extends Logging {
def tryWithResource[R <: Closeable, T](createResource: => R)(f: R => T): T = {
val resource = createResource
- try f.apply(resource) finally resource.close()
+ try {
+ f.apply(resource)
+ } finally {
+ closeQuietly(resource)
+ }
}
/**
@@ -59,7 +65,7 @@ private[spark] trait SparkErrorUtils extends Logging {
initialize(resource)
} catch {
case e: Throwable =>
- resource.close()
+ closeQuietly(resource)
throw e
}
}
@@ -97,13 +103,54 @@ private[spark] trait SparkErrorUtils extends Logging {
}
}
- def stackTraceToString(t: Throwable): String = {
- val out = new java.io.ByteArrayOutputStream
- SparkErrorUtils.tryWithResource(new PrintWriter(out)) { writer =>
- t.printStackTrace(writer)
- writer.flush()
+ def stackTraceToString(t: Throwable): String = JavaUtils.stackTraceToString(t)
+
+ /**
+ * Walks the [[Throwable]] to obtain its root cause.
+ *
+ * This method walks through the exception chain until the last element,
+ * the root cause of the chain, using `getCause()`, and
+ * returns that exception.
+ *
+ * This method handles recursive cause chains that might
+ * otherwise cause infinite loops. The cause chain is processed until
+ * the end, or until the next item in the chain is already
+ * processed. If we detect a loop, then return the element before the loop.
+ *
+ * @param throwable the throwable to get the root cause for, may be null
+ * @return the root cause of the [[Throwable]], `null` if null throwable input
+ */
+ def getRootCause(throwable: Throwable): Throwable = {
+ @tailrec
+ def findRoot(
+ current: Throwable,
+ visited: mutable.Set[Throwable] = mutable.Set.empty): Throwable = {
+ if (current == null) null
+ else {
+ visited += current
+ val cause = current.getCause
+ if (cause == null) {
+ current
+ } else if (visited.contains(cause)) {
+ current
+ } else {
+ findRoot(cause, visited)
+ }
+ }
+ }
+
+ findRoot(throwable)
+ }
+
+ /** Try to close by ignoring all exceptions. This is different from JavaUtils.closeQuietly. */
+ def closeQuietly(closeable: Closeable): Unit = {
+ if (closeable != null) {
+ try {
+ closeable.close()
+ } catch {
+ case _: Exception =>
+ }
}
- new String(out.toByteArray, UTF_8)
}
}
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala
index 22f03df1b2697..3f1f9c1f9df7f 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkFileUtils.scala
@@ -17,10 +17,11 @@
package org.apache.spark.util
import java.io.File
-import java.net.{URI, URISyntaxException}
-import java.nio.file.Files
+import java.net.{URI, URISyntaxException, URL}
+import java.nio.file.{Files, Path, StandardCopyOption}
+import java.nio.file.attribute.FileTime
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.network.util.JavaUtils
private[spark] trait SparkFileUtils extends Logging {
@@ -49,6 +50,13 @@ private[spark] trait SparkFileUtils extends Logging {
new File(path).getCanonicalFile().toURI()
}
+ /**
+ * Size of files recursively.
+ */
+ def sizeOf(f: File): Long = {
+ JavaUtils.sizeOf(f)
+ }
+
/**
* Lists files recursively.
*/
@@ -65,6 +73,20 @@ private[spark] trait SparkFileUtils extends Logging {
result.toArray
}
+ /**
+ * Lists regular files recursively.
+ */
+ def listFiles(f: File): java.util.Set[File] = {
+ JavaUtils.listFiles(f)
+ }
+
+ /**
+ * Lists regular paths recursively.
+ */
+ def listPaths(f: File): java.util.Set[Path] = {
+ JavaUtils.listPaths(f)
+ }
+
/**
* Create a directory given the abstract pathname
* @return true, if the directory is successfully created; otherwise, return false.
@@ -112,6 +134,21 @@ private[spark] trait SparkFileUtils extends Logging {
createDirectory(root, namePrefix)
}
+ def createParentDirs(file: File): Unit = {
+ if (file == null) {
+ throw new IllegalArgumentException("Input should not be null.")
+ }
+ val parent = file.getParentFile()
+ if (parent != null) {
+ Files.createDirectories(parent.toPath())
+ }
+ }
+
+ /** Delete recursively while keeping the given directory itself. */
+ def cleanDirectory(dir: File): Unit = {
+ JavaUtils.cleanDirectory(dir)
+ }
+
/**
* Delete a file or directory and its contents recursively.
* Don't follow directories if they are symlinks.
@@ -120,6 +157,87 @@ private[spark] trait SparkFileUtils extends Logging {
def deleteRecursively(file: File): Unit = {
JavaUtils.deleteRecursively(file)
}
+
+ /** Delete a file or directory and its contents recursively without throwing exceptions. */
+ def deleteQuietly(file: File): Unit = {
+ JavaUtils.deleteQuietly(file)
+ }
+
+ /** Registers the file or directory for deletion when the JVM exists. */
+ def forceDeleteOnExit(file: File): Unit = {
+ JavaUtils.forceDeleteOnExit(file)
+ }
+
+ def getFile(names: String*): File = {
+ require(names != null && names.forall(_ != null))
+ names.tail.foldLeft(Path.of(names.head)) { (path, part) =>
+ path.resolve(part)
+ }.toFile
+ }
+
+ def getFile(parent: File, names: String*): File = {
+ require(parent != null && names != null && names.forall(_ != null))
+ names.foldLeft(parent.toPath) { (path, part) =>
+ path.resolve(part)
+ }.toFile
+ }
+
+ /** Move src to dst simply. File attribute times are not copied. */
+ def moveFile(src: File, dst: File): Unit = {
+ JavaUtils.moveFile(src, dst)
+ }
+
+ /** Move src to dst simply. File attribute times are not copied. */
+ def moveDirectory(src: File, dst: File): Unit = {
+ JavaUtils.moveDirectory(src, dst)
+ }
+
+ /** Copy src to the target directory simply. File attribute times are not copied. */
+ def copyDirectory(src: File, dir: File): Unit = {
+ JavaUtils.copyDirectory(src, dir)
+ }
+
+ /** Copy file to the target directory simply. File attribute times are not copied. */
+ def copyFileToDirectory(file: File, dir: File): Unit = {
+ if (file == null || dir == null || !file.exists() || (dir.exists() && !dir.isDirectory())) {
+ throw new IllegalArgumentException(s"Invalid input file $file or directory $dir")
+ }
+ Files.createDirectories(dir.toPath())
+ val newFile = new File(dir, file.getName())
+ Files.copy(file.toPath(), newFile.toPath(), StandardCopyOption.REPLACE_EXISTING)
+ }
+
+ def copyFile(src: File, dst: File): Unit = {
+ if (src == null || dst == null || !src.exists() || (dst.exists() && dst.isDirectory())) {
+ throw new IllegalArgumentException(s"Invalid input file $src or directory $dst")
+ }
+ Files.copy(src.toPath(), dst.toPath(), StandardCopyOption.REPLACE_EXISTING)
+ }
+
+ def copyURLToFile(url: URL, file: File): Unit = {
+ JavaUtils.copyURLToFile(url, file)
+ }
+
+ /** Return true if the content of the files are equal or they both don't exist */
+ def contentEquals(file1: File, file2: File): Boolean = {
+ JavaUtils.contentEquals(file1, file2)
+ }
+
+ def touch(file: File): Unit = {
+ if (file == null) {
+ throw new IllegalArgumentException("Invalid input file: null")
+ }
+ val path = file.toPath
+ if (Files.exists(path)) {
+ Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis()))
+ } else {
+ val parent = path.getParent()
+ if (parent != null && !Files.exists(parent)) {
+ Files.createDirectories(parent)
+ }
+ Files.createFile(path)
+ }
+ }
}
private[spark] object SparkFileUtils extends SparkFileUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkStreamUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkStreamUtils.scala
index b9148901f1ac6..a2556770ed357 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkStreamUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkStreamUtils.scala
@@ -19,6 +19,7 @@ package org.apache.spark.util
import java.io.{FileInputStream, FileOutputStream, InputStream, OutputStream}
import java.nio.channels.{FileChannel, WritableByteChannel}
+import org.apache.spark.network.util.JavaUtils
import org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally
private[spark] trait SparkStreamUtils {
@@ -104,6 +105,14 @@ private[spark] trait SparkStreamUtils {
""".stripMargin)
}
}
+
+ def toString(in: InputStream): String = {
+ JavaUtils.toString(in)
+ }
+
+ def readFully(in: InputStream, arr: Array[Byte], off: Int, len: Int): Unit = {
+ JavaUtils.readFully(in, arr, off, len)
+ }
}
private [spark] object SparkStreamUtils extends SparkStreamUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkStringUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkStringUtils.scala
index 9f58288ee266c..53f46a5d9c8ed 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkStringUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkStringUtils.scala
@@ -16,11 +16,131 @@
*/
package org.apache.spark.util
+import java.util.HexFormat
+import java.util.concurrent.atomic.AtomicBoolean
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.util.ArrayImplicits._
+
private[spark] trait SparkStringUtils {
+ private final lazy val SPACE_DELIMITED_UPPERCASE_HEX =
+ HexFormat.of().withDelimiter(" ").withUpperCase()
+
+ /**
+ * Returns a pretty string of the byte array which prints each byte as a hex digit and add
+ * spaces between them. For example, [1A C0].
+ */
+ def getHexString(bytes: Array[Byte]): String = {
+ s"[${SPACE_DELIMITED_UPPERCASE_HEX.formatHex(bytes)}]"
+ }
+
+ def fromHexString(hex: String): Array[Byte] = {
+ SPACE_DELIMITED_UPPERCASE_HEX.parseHex(hex.stripPrefix("[").stripSuffix("]"))
+ }
+
+ def isEmpty(str: String): Boolean = str == null || str.length() == 0
+
+ def isNotEmpty(str: String): Boolean = !isEmpty(str)
+
+ def isBlank(str: String): Boolean = str == null || str.isBlank
+
+ def isNotBlank(str: String): Boolean = !isBlank(str)
+
+ def abbreviate(str: String, abbrevMarker: String, len: Int): String = {
+ if (str == null || abbrevMarker == null) {
+ null
+ } else if (str.length() <= len || str.length() <= abbrevMarker.length()) {
+ str
+ } else {
+ str.substring(0, len - abbrevMarker.length()) + abbrevMarker
+ }
+ }
+
+ def abbreviate(str: String, len: Int): String = abbreviate(str, "...", len)
+
+ def sideBySide(left: String, right: String): Seq[String] = {
+ sideBySide(left.split("\n").toImmutableArraySeq, right.split("\n").toImmutableArraySeq)
+ }
+
+ def sideBySide(left: Seq[String], right: Seq[String]): Seq[String] = {
+ val maxLeftSize = left.map(_.length).max
+ val leftPadded = left ++ Seq.fill(math.max(right.size - left.size, 0))("")
+ val rightPadded = right ++ Seq.fill(math.max(left.size - right.size, 0))("")
+
+ leftPadded.zip(rightPadded).map { case (l, r) =>
+ (if (l == r) " " else "!") + l + " ".repeat((maxLeftSize - l.length) + 3) + r
+ }
+ }
+
def stringToSeq(str: String): Seq[String] = {
import org.apache.spark.util.ArrayImplicits._
str.split(",").map(_.trim()).filter(_.nonEmpty).toImmutableArraySeq
}
+
+ /** Try to strip prefix and suffix with the given string 's' */
+ def strip(str: String, s: String): String =
+ if (str == null || s == null) str else str.stripPrefix(s).stripSuffix(s)
+
+ def leftPad(str: String, width: Int): String =
+ if (str == null || str.length >= width) str else String.format(s"%${width}s", str)
+
+ def rightPad(str: String, width: Int): String =
+ if (str == null || str.length >= width) str else String.format(s"%-${width}s", str)
+
+ def rightPad(str: String, width: Int, s: String): String =
+ if (str == null || str.length >= width) {
+ str
+ } else {
+ val tmp = str + s.repeat((width - str.length)/s.length)
+ tmp + s.substring(0, width - tmp.length)
+ }
}
-private[spark] object SparkStringUtils extends SparkStringUtils
+private[spark] object SparkStringUtils extends SparkStringUtils with Logging {
+
+ /** Whether we have warned about plan string truncation yet. */
+ private val truncationWarningPrinted = new AtomicBoolean(false)
+
+ /**
+ * Format a sequence with semantics similar to calling .mkString(). Any elements beyond
+ * `maxFields` will be dropped and replaced by a "... N more fields" placeholder.
+ *
+ * @return
+ * the trimmed and formatted string.
+ */
+ def truncatedString[T](
+ seq: Seq[T],
+ start: String,
+ sep: String,
+ end: String,
+ maxFields: Int,
+ customToString: Option[T => String] = None): String = {
+ if (seq.length > maxFields) {
+ if (truncationWarningPrinted.compareAndSet(false, true)) {
+ logWarning(
+ "Truncated the string representation of a plan since it was too large. This " +
+ s"behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.")
+ }
+ val numFields = math.max(0, maxFields)
+ val restNum = seq.length - numFields
+ val ending = (if (numFields == 0) "" else sep) +
+ (if (restNum == 0) "" else s"... $restNum more fields") + end
+ if (customToString.isDefined) {
+ seq.take(numFields).map(customToString.get).mkString(start, sep, ending)
+ } else {
+ seq.take(numFields).mkString(start, sep, ending)
+ }
+ } else {
+ if (customToString.isDefined) {
+ seq.map(customToString.get).mkString(start, sep, end)
+ } else {
+ seq.mkString(start, sep, end)
+ }
+ }
+ }
+
+ /** Shorthand for calling truncatedString() without start or end strings. */
+ def truncatedString[T](seq: Seq[T], sep: String, maxFields: Int): String = {
+ truncatedString(seq, "", sep, "", maxFields)
+ }
+}
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkSystemUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkSystemUtils.scala
new file mode 100644
index 0000000000000..eccf556c424eb
--- /dev/null
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkSystemUtils.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.util
+
+import org.apache.spark.network.util.JavaUtils
+
+private[spark] trait SparkSystemUtils {
+ /**
+ * The `os.name` system property.
+ */
+ val osName = JavaUtils.osName
+
+ /**
+ * The `os.version` system property.
+ */
+ val osVersion = JavaUtils.osVersion
+
+ /**
+ * The `os.arch` system property.
+ */
+ val osArch = JavaUtils.osArch
+
+ /**
+ * The `java.version` system property.
+ */
+ val javaVersion = JavaUtils.javaVersion
+
+ /**
+ * Whether the underlying operating system is Windows.
+ */
+ val isWindows = JavaUtils.isWindows
+
+ /**
+ * Whether the underlying operating system is Mac OS X.
+ */
+ val isMac = JavaUtils.isMac
+
+ /**
+ * Whether the underlying operating system is Mac OS X and processor is Apple Silicon.
+ */
+ val isMacOnAppleSilicon = JavaUtils.isMacOnAppleSilicon
+
+ /**
+ * Whether the underlying operating system is Linux.
+ */
+ val isLinux = JavaUtils.isLinux
+
+ /**
+ * Whether the underlying operating system is UNIX.
+ */
+ val isUnix = JavaUtils.isUnix
+}
+
+object SparkSystemUtils extends SparkSystemUtils
diff --git a/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
index 34b40a5a8890c..83028ad06a8f3 100644
--- a/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
+++ b/common/utils/src/main/scala/org/apache/spark/util/SparkTestUtils.scala
@@ -66,8 +66,6 @@ private[spark] trait SparkTestUtils {
assert(result.exists(), "Compiled file not found: " + result.getAbsolutePath())
val out = new File(destDir, fileName)
- // renameTo cannot handle in and out files in different filesystems
- // use google's Files.move instead
Files.move(result.toPath, out.toPath)
assert(out.exists(), "Destination file not moved: " + out.getAbsolutePath())
diff --git a/common/utils/src/test/scala/org/apache/spark/util/IvyTestUtils.scala b/common/utils/src/test/scala/org/apache/spark/util/IvyTestUtils.scala
index 140de836622f4..e240603ee2527 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/IvyTestUtils.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/IvyTestUtils.scala
@@ -23,10 +23,10 @@ import java.util.jar.Attributes.Name
import scala.collection.mutable.ArrayBuffer
-import org.apache.commons.io.FileUtils
import org.apache.ivy.core.settings.IvySettings
import org.apache.spark.util.MavenUtils.MavenCoordinate
+import org.apache.spark.util.SparkFileUtils.deleteRecursively
private[spark] object IvyTestUtils {
@@ -95,7 +95,7 @@ private[spark] object IvyTestUtils {
className: String,
packageName: String): Seq[(String, File)] = {
val rFilesDir = new File(dir, "R" + File.separator + "pkg")
- new File(rFilesDir, "R").mkdirs()
+ SparkFileUtils.createDirectory(new File(rFilesDir, "R"))
val contents =
s"""myfunc <- function(x) {
| SparkR:::callJStatic("$packageName.$className", "myFunc", x)
@@ -150,20 +150,20 @@ private[spark] object IvyTestUtils {
useIvyLayout: Boolean): File = {
if (useIvyLayout) {
val ivyXmlPath = pathFromCoordinate(artifact, tempPath, "ivy", true)
- ivyXmlPath.mkdirs()
+ SparkFileUtils.createDirectory(ivyXmlPath)
createIvyDescriptor(ivyXmlPath, artifact, dependencies)
} else {
val pomPath = pathFromCoordinate(artifact, tempPath, "pom", useIvyLayout)
- pomPath.mkdirs()
+ SparkFileUtils.createDirectory(pomPath)
createPom(pomPath, artifact, dependencies)
}
}
/** Helper method to write artifact information in the pom. */
private def pomArtifactWriter(artifact: MavenCoordinate, tabCount: Int = 1): String = {
- var result = "\n" + " " * tabCount + s"${artifact.groupId}"
- result += "\n" + " " * tabCount + s"${artifact.artifactId}"
- result += "\n" + " " * tabCount + s"${artifact.version}"
+ var result = "\n" + " ".repeat(tabCount) + s"${artifact.groupId}"
+ result += "\n" + " ".repeat(tabCount) + s"${artifact.artifactId}"
+ result += "\n" + " ".repeat(tabCount) + s"${artifact.version}"
result
}
@@ -293,13 +293,13 @@ private[spark] object IvyTestUtils {
// Where the root of the repository exists, and what Ivy will search in
val tempPath = tempDir.getOrElse(SparkFileUtils.createTempDir())
// Create directory if it doesn't exist
- tempPath.mkdirs()
+ SparkFileUtils.createDirectory(tempPath)
// Where to create temporary class files and such
val root = new File(tempPath, tempPath.hashCode().toString)
- root.mkdirs()
+ SparkFileUtils.createDirectory(root)
try {
val jarPath = pathFromCoordinate(artifact, tempPath, "jar", useIvyLayout)
- jarPath.mkdirs()
+ SparkFileUtils.createDirectory(jarPath)
val className = "MyLib"
val javaClass = createJavaClass(root, className, artifact.groupId)
@@ -319,7 +319,7 @@ private[spark] object IvyTestUtils {
val descriptor = createDescriptor(tempPath, artifact, dependencies, useIvyLayout)
assert(descriptor.exists(), "Problem creating Pom file")
} finally {
- FileUtils.deleteDirectory(root)
+ deleteRecursively(root)
}
tempPath
}
@@ -377,13 +377,13 @@ private[spark] object IvyTestUtils {
if (repo.toString.contains(".m2") || repo.toString.contains(".ivy2") ||
repo.toString.contains(".ivy2.5.2")) {
val groupDir = getBaseGroupDirectory(artifact, useIvyLayout)
- FileUtils.deleteDirectory(new File(repo, groupDir + File.separator + artifact.artifactId))
+ deleteRecursively(new File(repo, groupDir + File.separator + artifact.artifactId))
deps.foreach { _.foreach { dep =>
- FileUtils.deleteDirectory(new File(repo, getBaseGroupDirectory(dep, useIvyLayout)))
+ deleteRecursively(new File(repo, getBaseGroupDirectory(dep, useIvyLayout)))
}
}
} else {
- FileUtils.deleteDirectory(repo)
+ deleteRecursively(repo)
}
purgeLocalIvyCache(artifact, deps, ivySettings)
}
@@ -395,9 +395,9 @@ private[spark] object IvyTestUtils {
dependencies: Option[Seq[MavenCoordinate]],
ivySettings: IvySettings): Unit = {
// delete the artifact from the cache as well if it already exists
- FileUtils.deleteDirectory(new File(ivySettings.getDefaultCache, artifact.groupId))
+ deleteRecursively(new File(ivySettings.getDefaultCache, artifact.groupId))
dependencies.foreach { _.foreach { dep =>
- FileUtils.deleteDirectory(new File(ivySettings.getDefaultCache, dep.groupId))
+ deleteRecursively(new File(ivySettings.getDefaultCache, dep.groupId))
}
}
}
diff --git a/common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala b/common/utils/src/test/scala/org/apache/spark/util/LogKeysSuite.scala
similarity index 65%
rename from common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala
rename to common/utils/src/test/scala/org/apache/spark/util/LogKeysSuite.scala
index 17e360f510a24..25a908a8b96e2 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/LogKeysSuite.scala
@@ -19,25 +19,23 @@ package org.apache.spark.util
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path}
-import java.util.{ArrayList => JList}
import scala.jdk.CollectionConverters._
-import scala.reflect.runtime.universe._
-import org.apache.commons.io.FileUtils
import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
import org.apache.spark.internal.{Logging, LogKeys}
+import org.apache.spark.util.ArrayImplicits._
// scalastyle:off line.size.limit
/**
- * To re-generate the file `LogKey.scala`, run:
+ * To re-generate the file `LogKeys.java`, run:
* {{{
- * SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "common-utils/testOnly org.apache.spark.util.LogKeySuite"
+ * SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "common-utils/testOnly org.apache.spark.util.LogKeysSuite"
* }}}
*/
// scalastyle:on line.size.limit
-class LogKeySuite
+class LogKeysSuite
extends AnyFunSuite // scalastyle:ignore funsuite
with Logging {
@@ -54,37 +52,33 @@ class LogKeySuite
private val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
- private val logKeyFilePath = getWorkspaceFilePath("common", "utils", "src", "main", "scala",
- "org", "apache", "spark", "internal", "LogKey.scala")
+ private val logKeyFilePath = getWorkspaceFilePath("common", "utils", "src", "main", "java",
+ "org", "apache", "spark", "internal", "LogKeys.java")
- // regenerate the file `LogKey.scala` with its members sorted alphabetically
+ // regenerate the file `LogKeys.java` with its members sorted alphabetically
private def regenerateLogKeyFile(
originalKeys: Seq[String], sortedKeys: Seq[String]): Unit = {
if (originalKeys != sortedKeys) {
val logKeyFile = logKeyFilePath.toFile
logInfo(s"Regenerating the file $logKeyFile")
- val originalContents = FileUtils.readLines(logKeyFile, StandardCharsets.UTF_8)
- val sortedContents = new JList[String]()
- var firstMatch = false
- originalContents.asScala.foreach { line =>
- if (line.trim.startsWith("case object ") && line.trim.endsWith(" extends LogKey")) {
- if (!firstMatch) {
- sortedKeys.foreach { key =>
- sortedContents.add(s" case object $key extends LogKey")
- }
- firstMatch = true
- }
- } else {
- sortedContents.add(line)
- }
- }
+ val originalContents = Files.readAllLines(logKeyFile.toPath, StandardCharsets.UTF_8).asScala
+
+ val beforeFirstIndex = originalContents
+ .indexWhere(_.contains("public enum LogKeys implements LogKey"))
+
+ val content =
+ s"""${originalContents.take(beforeFirstIndex + 1).mkString("\n")}
+ |${sortedKeys.map { key => s" $key" }.mkString(",\n")}
+ |}
+ |""".stripMargin
+
Files.delete(logKeyFile.toPath)
- FileUtils.writeLines(logKeyFile, StandardCharsets.UTF_8.name(), sortedContents)
+ Files.writeString(logKeyFile.toPath, content, StandardCharsets.UTF_8)
}
}
test("The members of LogKeys are correctly sorted") {
- val originalKeys = getAllLogKeys.reverse
+ val originalKeys = LogKeys.values.map(_.name).toImmutableArraySeq
val sortedKeys = originalKeys.sorted
if (regenerateGoldenFiles) {
regenerateLogKeyFile(originalKeys, sortedKeys)
@@ -93,11 +87,4 @@ class LogKeySuite
"The members of LogKeys must be sorted alphabetically")
}
}
-
- private def getAllLogKeys: Seq[String] = {
- val logKeysType = typeOf[LogKeys.type]
- val classSymbol = logKeysType.typeSymbol.asClass
- val members = classSymbol.typeSignature.members
- members.filter(m => m.isTerm && !m.isMethod).map(_.name.toString).toSeq
- }
}
diff --git a/common/utils/src/test/scala/org/apache/spark/util/MDCSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/MDCSuite.scala
index 9615eb2263636..f086b6d18dafc 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/MDCSuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/MDCSuite.scala
@@ -22,7 +22,7 @@ import scala.jdk.CollectionConverters._
import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{EXIT_CODE, OFFSET, RANGE}
class MDCSuite
diff --git a/common/utils/src/test/scala/org/apache/spark/util/SparkErrorUtilsSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/SparkErrorUtilsSuite.scala
new file mode 100644
index 0000000000000..791003b9cddce
--- /dev/null
+++ b/common/utils/src/test/scala/org/apache/spark/util/SparkErrorUtilsSuite.scala
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import java.io.{Closeable, IOException}
+
+import scala.annotation.nowarn
+
+import org.scalatest.BeforeAndAfterEach
+import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
+
+class SparkErrorUtilsSuite
+ extends AnyFunSuite // scalastyle:ignore funsuite
+ with BeforeAndAfterEach {
+
+ private var withoutCause: Throwable = _
+ private var withCause: Throwable = _
+ private var jdkNoCause: Throwable = _
+ private var nested: NestableException = _
+ private var cyclicCause: ExceptionWithCause = _
+
+ override def beforeEach(): Unit = {
+ withoutCause = createExceptionWithoutCause
+ nested = new NestableException(withoutCause)
+ withCause = new ExceptionWithCause(nested)
+ jdkNoCause = new NullPointerException
+ val exceptionA = new ExceptionWithCause(null.asInstanceOf[Throwable])
+ val exceptionB = new ExceptionWithCause(exceptionA)
+ exceptionA.setCauseValue(exceptionB)
+ cyclicCause = new ExceptionWithCause(exceptionA)
+ }
+
+ override def afterEach(): Unit = {
+ withoutCause = null
+ nested = null
+ withCause = null
+ jdkNoCause = null
+ cyclicCause = null
+ }
+
+ test("getRootCause") {
+ assert(SparkErrorUtils.getRootCause(null) == null)
+ assert(SparkErrorUtils.getRootCause(withoutCause) == withoutCause)
+ assert(SparkErrorUtils.getRootCause(nested) == withoutCause)
+ assert(SparkErrorUtils.getRootCause(withCause) == withoutCause)
+ assert(SparkErrorUtils.getRootCause(jdkNoCause) == jdkNoCause)
+ assert(SparkErrorUtils.getRootCause(cyclicCause) == cyclicCause.getCause.getCause)
+ }
+
+ test("tryWithResource / tryInitializeResource") {
+ val closeable = new Closeable {
+ override def close(): Unit = {
+ throw new IOException("Catch me if you can")
+ }
+ }
+ val e1 = intercept[IOException] {
+ SparkErrorUtils.tryWithResource(closeable)(_ => throw new IOException("You got me!"))
+ }
+ assert(e1.getMessage === "You got me!")
+ val e2 = intercept[IOException] {
+ SparkErrorUtils.tryInitializeResource(closeable)(_ => throw new IOException("You got me!"))
+ }
+ assert(e2.getMessage === "You got me!")
+ }
+
+ private def createExceptionWithoutCause: Throwable =
+ try throw new ExceptionWithoutCause
+ catch {
+ case t: Throwable => t
+ }
+
+ private final class ExceptionWithoutCause extends Exception {
+ @nowarn def getTargetException(): Unit = {}
+ }
+
+ private final class NestableException extends Exception {
+ def this(t: Throwable) = {
+ this()
+ initCause(t)
+ }
+ }
+
+ private final class ExceptionWithCause(message: String = null) extends Exception(message) {
+
+ private var _cause: Throwable = _
+
+ def this(message: String, cause: Throwable) = {
+ this(message)
+ this._cause = cause
+ }
+
+ def this(cause: Throwable) = {
+ this(null.asInstanceOf[String])
+ this._cause = cause
+ }
+
+ override def getCause: Throwable = synchronized {
+ _cause
+ }
+
+ def setCauseValue(cause: Throwable): Unit = {
+ this._cause = cause
+ }
+ }
+}
diff --git a/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala
index 0026b696f0695..20f6d2347eabe 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/StructuredLoggingSuite.scala
@@ -26,7 +26,7 @@ import org.apache.logging.log4j.Level
import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
-import org.apache.spark.internal.{LogEntry, Logging, LogKey, LogKeys, MDC, MessageWithContext}
+import org.apache.spark.internal.{LogEntry, Logging, LogKeys, MessageWithContext}
trait LoggingSuiteBase
extends AnyFunSuite // scalastyle:ignore funsuite
@@ -491,8 +491,3 @@ class StructuredLoggingSuite extends LoggingSuiteBase {
assert(mdcPattern.r.findFirstIn(logOutputWithoutMDCSet).isEmpty)
}
}
-
-object CustomLogKeys {
- // Custom `LogKey` must be `extends LogKey`
- case object CUSTOM_LOG_KEY extends LogKey
-}
diff --git a/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java b/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
index 9c4487dbd3f75..1bd008a5c9149 100644
--- a/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
+++ b/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
@@ -229,7 +229,7 @@ public void appendTimestampNtz(long microsSinceEpoch) {
public void appendFloat(float f) {
checkCapacity(1 + 4);
writeBuffer[writePos++] = primitiveHeader(FLOAT);
- writeLong(writeBuffer, writePos, Float.floatToIntBits(f), 8);
+ writeLong(writeBuffer, writePos, Float.floatToIntBits(f), 4);
writePos += 4;
}
diff --git a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
index 40ea064cbf115..795d46ec2062b 100644
--- a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
+++ b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
@@ -19,6 +19,7 @@
import org.apache.spark.QueryContext;
import org.apache.spark.SparkRuntimeException;
+import org.apache.spark.network.util.JavaUtils;
import scala.collection.immutable.Map$;
import java.math.BigDecimal;
@@ -140,7 +141,7 @@ public class VariantUtil {
// Both variant value and variant metadata need to be no longer than 128MiB.
// Note: to make tests more reliable, we set the max size to 16Mib to avoid OOM in tests.
public static final int SIZE_LIMIT =
- System.getenv("SPARK_TESTING") != null ? U24_MAX + 1 : 128 * 1024 * 1024;
+ JavaUtils.isTesting() ? U24_MAX + 1 : 128 * 1024 * 1024;
public static final int MAX_DECIMAL4_PRECISION = 9;
public static final int MAX_DECIMAL8_PRECISION = 18;
diff --git a/connect-examples/server-library-example/README.md b/connect-examples/server-library-example/README.md
deleted file mode 100644
index adf4830d58ff5..0000000000000
--- a/connect-examples/server-library-example/README.md
+++ /dev/null
@@ -1,133 +0,0 @@
-# Spark Server Library Example - Custom Datasource Handler
-
-This example demonstrates a modular maven-based project architecture with separate client, server
-and common components. It leverages the extensibility of Spark Connect to create a server library
-that may be attached to the server to extend the functionality of the Spark Connect server as a whole. Below is a detailed overview of the setup and functionality.
-
-## Project Structure
-
-```
-├── common/ # Shared protobuf/utilities/classes
-├── client/ # Sample client implementation
-│ ├── src/ # Source code for client functionality
-│ ├── pom.xml # Maven configuration for the client
-├── server/ # Server-side plugin extension
-│ ├── src/ # Source code for server functionality
-│ ├── pom.xml # Maven configuration for the server
-├── resources/ # Static resources
-├── pom.xml # Parent Maven configuration
-```
-
-## Functionality Overview
-
-To demonstrate the extensibility of Spark Connect, a custom datasource handler, `CustomTable` is
-implemented in the server module. The class handles reading, writing and processing data stored in
-a custom format, here we simply use the `.custom` extension (which itself is a wrapper over `.csv`
-files).
-
-First and foremost, the client and the server must be able to communicate with each other through
-custom messages that 'understand' our custom data format. This is achieved by defining custom
-protobuf messages in the `common` module. The client and server modules both depend on the `common`
-module to access these messages.
-- `common/src/main/protobuf/base.proto`: Defines the base `CustomTable` which is simply represented
-by a path and a name.
-```protobuf
-message CustomTable {
- string path = 1;
- string name = 2;
-}
-```
-- `common/src/main/protobuf/commands.proto`: Defines the custom commands that the client can send
-to the server. These commands are typically operations that the server can perform, such as cloning
-an existing custom table.
-```protobuf
-message CustomCommand {
- oneof command_type {
- CreateTable create_table = 1;
- CloneTable clone_table = 2;
- }
-}
-```
-- `common/src/main/protobuf/relations.proto`: Defines custom `relations`, which are a mechanism through which an optional input dataset is transformed into an
- output dataset such as a Scan.
-```protobuf
-message Scan {
- CustomTable table = 1;
-}
-```
-
-On the client side, the `CustomTable` class mimics the style of Spark's `Dataset` API, allowing the
-user to perform and chain operations on a `CustomTable` object.
-
-On the server side, a similar `CustomTable` class is implemented to handle the core functionality of
-reading, writing and processing data in the custom format. The plugins (`CustomCommandPlugin` and
-`CustomRelationPlugin`) are responsible for processing the custom protobuf messages sent from the client
-(those defined in the `common` module) and delegating the appropriate actions to the `CustomTable`.
-
-
-
-## Build and Run Instructions
-
-1. **Navigate to the sample project from `SPARK_HOME`**:
- ```bash
- cd connect-examples/server-library-example
- ```
-
-2. **Build and package the modules**:
- ```bash
- mvn clean package
- ```
-
-3. **Download the `4.0.0-preview2` release to use as the Spark Connect Server**:
- - Choose a distribution from https://archive.apache.org/dist/spark/spark-4.0.0-preview2/.
- - Example: `curl -L https://archive.apache.org/dist/spark/spark-4.0.0-preview2/spark-4.0.0-preview2-bin-hadoop3.tgz | tar xz`
-
-4. **Copy relevant JARs to the root of the unpacked Spark distribution**:
- ```bash
- cp \
- /connect-examples/server-library-example/common/target/spark-daria_2.13-1.2.3.jar \
- /connect-examples/server-library-example/common/target/spark-server-library-example-common-1.0.0.jar \
- /connect-examples/server-library-example/server/target/spark-server-library-example-server-extension-1.0.0.jar \
- .
- ```
-5. **Start the Spark Connect Server with the relevant JARs**:
- ```bash
- bin/spark-connect-shell \
- --jars spark-server-library-example-server-extension-1.0.0.jar,spark-server-library-example-common-1.0.0.jar,spark-daria_2.13-1.2.3.jar \
- --conf spark.connect.extensions.relation.classes=org.apache.connect.examples.serverlibrary.CustomRelationPlugin \
- --conf spark.connect.extensions.command.classes=org.apache.connect.examples.serverlibrary.CustomCommandPlugin
- ```
-6. **In a different terminal, navigate back to the root of the sample project and start the client**:
- ```bash
- java -cp client/target/spark-server-library-client-package-scala-1.0.0.jar org.apache.connect.examples.serverlibrary.CustomTableExample
- ```
-7. **Notice the printed output in the client terminal as well as the creation of the cloned table**:
-```protobuf
-Explaining plan for custom table: sample_table with path: /spark/connect-examples/server-library-example/client/../resources/dummy_data.custom
-== Parsed Logical Plan ==
-Relation [id#2,name#3] csv
-
-== Analyzed Logical Plan ==
-id: int, name: string
-Relation [id#2,name#3] csv
-
-== Optimized Logical Plan ==
-Relation [id#2,name#3] csv
-
-== Physical Plan ==
-FileScan csv [id#2,name#3] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/Users/venkata.gudesa/spark/connect-examples/server-library-example/resou..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct
-
-Explaining plan for custom table: cloned_table with path: /connect-examples/server-library-example/client/../resources/cloned_data.data
-== Parsed Logical Plan ==
-Relation [id#2,name#3] csv
-
-== Analyzed Logical Plan ==
-id: int, name: string
-Relation [id#2,name#3] csv
-
-== Optimized Logical Plan ==
-Relation [id#2,name#3] csv
-
-== Physical Plan ==
-FileScan csv [id#2,name#3] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/Users/venkata.gudesa/spark/connect-examples/server-library-example/resou..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct
-```
\ No newline at end of file
diff --git a/connect-examples/server-library-example/client/pom.xml b/connect-examples/server-library-example/client/pom.xml
deleted file mode 100644
index 364920a2ec22b..0000000000000
--- a/connect-examples/server-library-example/client/pom.xml
+++ /dev/null
@@ -1,112 +0,0 @@
-
-
-
-
- 4.0.0
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example
- 1.0.0
- ../pom.xml
-
-
- spark-server-library-client-package-scala
- jar
-
-
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example-common
- 1.0.0
-
-
- com.google.protobuf
- protobuf-java
-
-
-
-
-
- org.apache.spark
- spark-connect-common_${scala.binary}
- ${spark.version}
-
-
-
- org.apache.spark
- spark-connect-client-jvm_${scala.binary}
- ${spark.version}
-
-
-
- org.scala-lang
- scala-library
- ${scala.version}
-
-
-
-
-
- src/main/scala
-
-
-
-
- net.alchim31.maven
- scala-maven-plugin
- 4.9.2
-
-
-
- compile
- testCompile
-
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-shade-plugin
- 3.5.1
-
-
- package
-
- shade
-
-
- false
- true
- false
-
-
-
-
-
-
-
-
-
-
-
diff --git a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala b/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala
deleted file mode 100644
index 782a246d92984..0000000000000
--- a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import com.google.protobuf.Any
-import org.apache.spark.connect.proto.Command
-import org.apache.spark.sql.{functions, Column, DataFrame, Dataset, Row, SparkSession}
-
-import org.apache.connect.examples.serverlibrary.proto
-import org.apache.connect.examples.serverlibrary.proto.CreateTable.Column.{DataType => ProtoDataType}
-
-/**
- * Represents a custom table with associated DataFrame and metadata.
- *
- * @param df The underlying DataFrame.
- * @param table The metadata of the custom table.
- */
-class CustomTable private (private val df: Dataset[Row], private val table: proto.CustomTable) {
-
- /**
- * Returns the Spark session associated with the DataFrame.
- */
- private def spark = df.sparkSession
-
- /**
- * Converts the custom table to a DataFrame.
- *
- * @return The underlying DataFrame.
- */
- def toDF: Dataset[Row] = df
-
- /**
- * Prints the execution plan of the custom table.
- */
- def explain(): Unit = {
- println(s"Explaining plan for custom table: ${table.getName} with path: ${table.getPath}")
- df.explain("extended")
- }
-
- /**
- * Clones the custom table to a new location with a new name.
- *
- * @param target The target path for the cloned table.
- * @param newName The new name for the cloned table.
- * @param replace Whether to replace the target location if it exists.
- * @return A new `CustomTable` instance representing the cloned table.
- */
- def clone(target: String, newName: String, replace: Boolean): CustomTable = {
- val cloneTableProto = proto.CloneTable
- .newBuilder()
- .setTable(
- proto.CustomTable
- .newBuilder()
- .setName(table.getName)
- .setPath(table.getPath))
- .setClone(
- proto.CustomTable
- .newBuilder()
- .setName(newName)
- .setPath(target))
- .setReplace(replace)
- .build()
- val customCommand = proto.CustomCommand
- .newBuilder()
- .setCloneTable(cloneTableProto)
- .build()
- // Pack the CustomCommand into Any
- val customCommandAny = Any.pack(customCommand)
- // Set the Any as the extension of a Command
- val commandProto = Command
- .newBuilder()
- .setExtension(customCommandAny)
- .build()
- // Execute the command
- spark.execute(commandProto)
- CustomTable.from(spark, newName, target)
- }
-}
-
-object CustomTable {
- /**
- * Creates a `CustomTable` from the given Spark session, name, and path.
- *
- * @param spark The Spark session.
- * @param name The name of the table.
- * @param path The path of the table.
- * @return A new `CustomTable` instance.
- */
- def from(spark: SparkSession, name: String, path: String): CustomTable = {
- val table = proto.CustomTable
- .newBuilder()
- .setName(name)
- .setPath(path)
- .build()
- val relation = proto.CustomRelation
- .newBuilder()
- .setScan(proto.Scan.newBuilder().setTable(table))
- .build()
- val customRelation = Any.pack(relation)
- val df = spark.newDataFrame(f => f.setExtension(customRelation))
- new CustomTable(df, table)
- }
-
- /**
- * Creates a new `CustomTableBuilder` instance.
- *
- * @param spark The Spark session.
- * @return A new `CustomTableBuilder` instance.
- */
- def create(spark: SparkSession): CustomTableBuilder = new CustomTableBuilder(spark)
-
- /**
- * Enumeration for data types.
- */
- object DataType extends Enumeration {
- type DataType = Value
- val Int, String, Float, Boolean = Value
-
- /**
- * Converts a `DataType` to its corresponding `ProtoDataType`.
- *
- * @param dataType The data type to convert.
- * @return The corresponding `ProtoDataType`.
- */
- def toProto(dataType: DataType): ProtoDataType = {
- dataType match {
- case Int => ProtoDataType.INT
- case String => ProtoDataType.STRING
- case Float => ProtoDataType.FLOAT
- case Boolean => ProtoDataType.BOOLEAN
- }
- }
- }
-}
diff --git a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableBuilder.scala b/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableBuilder.scala
deleted file mode 100644
index a1b8ffdb8dd72..0000000000000
--- a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableBuilder.scala
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import com.google.protobuf.Any
-import org.apache.spark.connect.proto.Command
-import org.apache.spark.sql.SparkSession
-
-import org.apache.connect.examples.serverlibrary.CustomTable
-
-/**
- * Builder class for constructing a `CustomTable` instance.
- *
- * @param spark The Spark session.
- */
-class CustomTableBuilder private[serverlibrary] (spark: SparkSession) {
- import CustomTableBuilder._
-
- private var name: String = _
- private var path: String = _
- private var columns: Seq[Column] = Seq.empty
-
- /**
- * Sets the name of the custom table.
- *
- * @param name The name of the table.
- * @return The current `CustomTableBuilder` instance.
- */
- def name(name: String): CustomTableBuilder = {
- this.name = name
- this
- }
-
- /**
- * Sets the path of the custom table.
- *
- * @param path The path of the table.
- * @return The current `CustomTableBuilder` instance.
- */
- def path(path: String): CustomTableBuilder = {
- this.path = path
- this
- }
-
- /**
- * Adds a column to the custom table.
- *
- * @param name The name of the column.
- * @param dataType The data type of the column.
- * @return The current `CustomTableBuilder` instance.
- */
- def addColumn(name: String, dataType: CustomTable.DataType.Value): CustomTableBuilder = {
- columns = columns :+ Column(name, dataType)
- this
- }
-
- /**
- * Builds the `CustomTable` instance.
- *
- * @return A new `CustomTable` instance.
- * @throws IllegalArgumentException if name, path, or columns are not set.
- */
- def build(): CustomTable = {
- require(name != null, "Name must be set")
- require(path != null, "Path must be set")
- require(columns.nonEmpty, "At least one column must be added")
-
- // Define the table creation proto
- val createTableProtoBuilder = proto.CreateTable
- .newBuilder()
- .setTable(
- proto.CustomTable
- .newBuilder()
- .setPath(path)
- .setName(name)
- .build())
-
- // Add columns to the table creation proto
- columns.foreach { column =>
- createTableProtoBuilder.addColumns(
- proto.CreateTable.Column
- .newBuilder()
- .setName(column.name)
- .setDataType(CustomTable.DataType.toProto(column.dataType))
- .build())
- }
- val createTableProto = createTableProtoBuilder.build() // Build the CreateTable object
-
- // Wrap the CreateTable proto in CustomCommand
- val customCommand = proto.CustomCommand
- .newBuilder()
- .setCreateTable(createTableProto)
- .build()
-
- // Pack the CustomCommand into Any
- val customCommandAny = Any.pack(customCommand)
- // Set the Any as the extension of a Command
- val commandProto = Command
- .newBuilder()
- .setExtension(customCommandAny)
- .build()
-
- // Execute the command
- spark.execute(commandProto)
-
- // After the command is executed, create a client-side representation of the table with the
- // leaf node of the client-side dataset being the Scan node for the custom table.
- CustomTable.from(spark, name, path)
- }
-}
-
-object CustomTableBuilder {
- /**
- * Case class representing a column in the custom table.
- *
- * @param name The name of the column.
- * @param dataType The data type of the column.
- */
- private case class Column(name: String, dataType: CustomTable.DataType.Value)
-}
diff --git a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableExample.scala b/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableExample.scala
deleted file mode 100644
index 8470465cd7a0b..0000000000000
--- a/connect-examples/server-library-example/client/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTableExample.scala
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import java.nio.file.{Path, Paths}
-
-import com.google.protobuf.Any
-import org.apache.spark.connect.proto.Command
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.types.{StructType, StructField, StringType, IntegerType}
-
-import org.apache.connect.examples.serverlibrary.proto
-import org.apache.connect.examples.serverlibrary.proto.CreateTable.Column.{DataType => ProtoDataType}
-
-object CustomTableExample {
- def main(args: Array[String]): Unit = {
- val spark = SparkSession.builder().remote("sc://localhost").build()
-
- // Step 1: Create a custom table from existing data
- val tableName = "sample_table"
- val fileName = "dummy_data.data"
-
- val workingDirectory = System.getProperty("user.dir")
- val fullPath = Paths.get(workingDirectory, s"resources/$fileName")
- val customTable = CustomTable
- .create(spark)
- .name(tableName)
- .path(fullPath.toString)
- .addColumn("id", CustomTable.DataType.Int)
- .addColumn("name", CustomTable.DataType.String)
- .build()
-
- // Step 2: Verify
- customTable.explain()
-
- // Step 3: Clone the custom table
- val clonedPath = fullPath.getParent.resolve("cloned_data.data")
- val clonedName = "cloned_table"
- val clonedTable =
- customTable.clone(target = clonedPath.toString, newName = clonedName, replace = true)
-
- // Step 4: Verify
- clonedTable.explain()
- }
-}
diff --git a/connect-examples/server-library-example/common/pom.xml b/connect-examples/server-library-example/common/pom.xml
deleted file mode 100644
index 592c43f26770b..0000000000000
--- a/connect-examples/server-library-example/common/pom.xml
+++ /dev/null
@@ -1,76 +0,0 @@
-
-
-
-
- 4.0.0
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example
- 1.0.0
- ../pom.xml
-
-
- spark-server-library-example-common
- jar
-
-
-
-
- com.google.protobuf
- protobuf-java
- ${protobuf.version}
-
-
- org.scala-lang
- scala-library
- ${scala.version}
-
-
-
-
-
-
-
- kr.motd.maven
- os-maven-plugin
- 1.7.0
- true
-
-
- org.xolstice.maven.plugins
- protobuf-maven-plugin
- 0.6.1
-
- com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
- src/main/protobuf
-
-
-
- generate-sources
-
- compile
-
-
-
-
-
-
-
diff --git a/connect-examples/server-library-example/common/src/main/protobuf/commands.proto b/connect-examples/server-library-example/common/src/main/protobuf/commands.proto
deleted file mode 100644
index 13d9945cfe61d..0000000000000
--- a/connect-examples/server-library-example/common/src/main/protobuf/commands.proto
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = 'proto3';
-
-option java_multiple_files = true;
-option java_package = "org.apache.connect.examples.serverlibrary.proto";
-
-import "base.proto";
-
-message CustomCommand {
- oneof command_type {
- CreateTable create_table = 1;
- CloneTable clone_table = 2;
- }
-}
-
-message CreateTable {
-
- // Column in the schema of the table.
- message Column {
- // (Required) Name of the column.
- string name = 1;
- // (Required) Data type of the column.
- enum DataType {
- DATA_TYPE_UNSPECIFIED = 0; // Default value
- INT = 1; // Integer data type
- STRING = 2; // String data type
- FLOAT = 3; // Float data type
- BOOLEAN = 4; // Boolean data type
- }
- DataType data_type = 2;
- }
- // (Required) Table properties.
- CustomTable table = 1;
- // (Required) List of columns in the schema of the table.
- repeated Column columns = 2;
-}
-
-message CloneTable {
- // (Required) The source table to clone.
- CustomTable table = 1;
- // (Required) Path to the location where the data of the cloned table should be stored.
- CustomTable clone = 2;
- // (Required) Overwrites the target location when true.
- bool replace = 3;
-}
diff --git a/connect-examples/server-library-example/pom.xml b/connect-examples/server-library-example/pom.xml
deleted file mode 100644
index 1723f3b0154fa..0000000000000
--- a/connect-examples/server-library-example/pom.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-
-
-
-
- 4.0.0
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example
- 1.0.0
- pom
-
- common
- server
- client
-
-
- 17
- 17
- UTF-8
- 2.13
- 2.13.15
- 3.25.4
- 4.0.0-preview2
-
-
diff --git a/connect-examples/server-library-example/resources/dummy_data.data b/connect-examples/server-library-example/resources/dummy_data.data
deleted file mode 100644
index 0a6645b757222..0000000000000
--- a/connect-examples/server-library-example/resources/dummy_data.data
+++ /dev/null
@@ -1,6 +0,0 @@
-id,name
-1,John Doe
-2,Jane Smith
-3,Bob Johnson
-4,Alice Williams
-5,Charlie Brown
\ No newline at end of file
diff --git a/connect-examples/server-library-example/server/pom.xml b/connect-examples/server-library-example/server/pom.xml
deleted file mode 100644
index b13a7537f9c13..0000000000000
--- a/connect-examples/server-library-example/server/pom.xml
+++ /dev/null
@@ -1,90 +0,0 @@
-
-
-
-
- 4.0.0
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example
- 1.0.0
- ../pom.xml
-
-
- spark-server-library-example-server-extension
- jar
-
-
-
-
- org.apache.connect.examples.serverlibrary
- spark-server-library-example-common
- 1.0.0
-
-
-
- org.scala-lang
- scala-library
- ${scala.version}
-
-
-
- org.apache.spark
- spark-sql_${scala.binary}
- ${spark.version}
- provided
-
-
-
- org.apache.spark
- spark-connect_${scala.binary}
- ${spark.version}
- provided
-
-
-
-
- com.github.mrpowers
- spark-daria_${scala.binary}
- 1.2.3
-
-
-
-
- src/main/scala
-
-
-
-
- net.alchim31.maven
- scala-maven-plugin
- 4.9.2
-
-
-
- compile
- testCompile
-
-
-
-
-
-
-
diff --git a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomCommandPlugin.scala b/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomCommandPlugin.scala
deleted file mode 100644
index 2253c4b238be4..0000000000000
--- a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomCommandPlugin.scala
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import scala.collection.JavaConverters._
-
-import com.google.protobuf.Any
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.connect.planner.SparkConnectPlanner
-import org.apache.spark.sql.connect.plugin.CommandPlugin
-import org.apache.spark.sql.types.{StringType, IntegerType, FloatType, DoubleType, BooleanType, LongType, StructType, StructField, DataType}
-
-import org.apache.connect.examples.serverlibrary.CustomTable
-import org.apache.connect.examples.serverlibrary.proto
-import org.apache.connect.examples.serverlibrary.proto.CreateTable.Column.{DataType => ProtoDataType}
-
-/**
- * Commands are distinct actions that can be executed. Unlike relations, which focus on the
- * transformation and nesting of output data, commands represent singular operations that perform
- * specific tasks on the data.
- * In this example, the `CustomCommandPlugin` handles operations related to creating and duplicating
- * custom tables.
- */
-class CustomCommandPlugin extends CommandPlugin with CustomPluginBase {
-
- /**
- * Processes the raw byte array containing the command.
- *
- * @param raw The raw byte array of the command.
- * @param planner The SparkConnectPlanner instance.
- * @return True if the command was processed, false otherwise.
- */
- override def process(raw: Array[Byte], planner: SparkConnectPlanner): Boolean = {
- val command = Any.parseFrom(raw)
- if (command.is(classOf[proto.CustomCommand])) {
- processInternal(command.unpack(classOf[proto.CustomCommand]), planner)
- true
- } else {
- false
- }
- }
-
- /**
- * Processes the unpacked CustomCommand.
- *
- * @param command The unpacked CustomCommand.
- * @param planner The SparkConnectPlanner instance.
- */
- private def processInternal(
- command: proto.CustomCommand,
- planner: SparkConnectPlanner): Unit = {
- command.getCommandTypeCase match {
- case proto.CustomCommand.CommandTypeCase.CREATE_TABLE =>
- processCreateTable(planner, command.getCreateTable)
- case proto.CustomCommand.CommandTypeCase.CLONE_TABLE =>
- processCloneTable(planner, command.getCloneTable)
- case _ =>
- throw new IllegalArgumentException(
- s"Unsupported command type: ${command.getCommandTypeCase}")
- }
- }
-
- /**
- * Processes the CreateTable command.
- *
- * @param planner The SparkConnectPlanner instance.
- * @param createTable The CreateTable message.
- */
- private def processCreateTable(
- planner: SparkConnectPlanner,
- createTable: proto.CreateTable): Unit = {
- val tableName = createTable.getTable.getName
- val tablePath = createTable.getTable.getPath
-
- // Convert the list of columns from the protobuf message to a Spark schema
- val schema = StructType(createTable.getColumnsList.asScala.toSeq.map { column =>
- StructField(
- column.getName,
- protoDataTypeToSparkType(column.getDataType),
- nullable = true // Assuming all columns are nullable for simplicity
- )
- })
-
- // Create the table using the CustomTable utility
- CustomTable.createTable(tableName, tablePath, planner.session, schema)
- }
-
- /**
- * Converts a protobuf DataType to a Spark DataType.
- *
- * @param protoType The protobuf DataType.
- * @return The corresponding Spark DataType.
- */
- private def protoDataTypeToSparkType(protoType: ProtoDataType): DataType = {
- protoType match {
- case ProtoDataType.INT => IntegerType
- case ProtoDataType.STRING => StringType
- case ProtoDataType.FLOAT => FloatType
- case ProtoDataType.BOOLEAN => BooleanType
- case _ =>
- throw new IllegalArgumentException(s"Unsupported or unknown data type: ${protoType}")
- }
- }
-
- /**
- * Processes the CloneTable command.
- *
- * @param planner The SparkConnectPlanner instance.
- * @param msg The CloneTable message.
- */
- private def processCloneTable(planner: SparkConnectPlanner, msg: proto.CloneTable): Unit = {
- val sourceTable = getCustomTable(msg.getTable)
- CustomTable.cloneTable(
- sourceTable,
- msg.getClone.getName,
- msg.getClone.getPath,
- msg.getReplace)
- }
-}
diff --git a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomRelationPlugin.scala b/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomRelationPlugin.scala
deleted file mode 100644
index 7b444803a065f..0000000000000
--- a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomRelationPlugin.scala
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import java.util.Optional
-
-import com.google.protobuf.Any
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.connect.planner.SparkConnectPlanner
-import org.apache.spark.sql.connect.plugin.RelationPlugin
-
-import org.apache.connect.examples.serverlibrary.{CustomPluginBase, CustomTable}
-import org.apache.connect.examples.serverlibrary.proto
-
-/**
- * Relations are fundamental to dataset transformations, acting as the mechanism through which an
- * input dataset is transformed into an output dataset. Conceptually, relations can be likened to
- * tables within a database, manipulated to achieve desired outcomes.
- * In this example, the `CustomRelationPlugin` handles the transformation related to scanning
- * custom tables. The scan relation would appear as leaf nodes in a dataset's associated logical
- * plan node when it involves reads from the custom tables.
- */
-class CustomRelationPlugin extends RelationPlugin with CustomPluginBase {
-
- /**
- * Transforms the raw byte array containing the relation into a Spark logical plan.
- *
- * @param raw The raw byte array of the relation.
- * @param planner The SparkConnectPlanner instance.
- * @return An Optional containing the LogicalPlan if the relation was processed, empty otherwise.
- */
- override def transform(
- raw: Array[Byte],
- planner: SparkConnectPlanner): Optional[LogicalPlan] = {
- val rel = Any.parseFrom(raw)
- if (rel.is(classOf[proto.CustomRelation])) {
- val customRelation = rel.unpack(classOf[proto.CustomRelation])
- // Transform the custom relation
- Optional.of(transformInner(customRelation, planner))
- } else {
- Optional.empty()
- }
- }
-
- /**
- * Transforms the unpacked CustomRelation into a Spark logical plan.
- *
- * @param relation The unpacked CustomRelation.
- * @param planner The SparkConnectPlanner instance.
- * @return The corresponding Spark LogicalPlan.
- */
- private def transformInner(
- relation: proto.CustomRelation,
- planner: SparkConnectPlanner): LogicalPlan = {
- relation.getRelationTypeCase match {
- case proto.CustomRelation.RelationTypeCase.SCAN =>
- transformScan(relation.getScan, planner)
- case _ =>
- throw new IllegalArgumentException(
- s"Unsupported relation type: ${relation.getRelationTypeCase}")
- }
- }
-
- /**
- * Transforms the Scan relation into a Spark logical plan.
- *
- * @param scan The Scan message.
- * @param planner The SparkConnectPlanner instance.
- * @return The corresponding Spark LogicalPlan.
- */
- private def transformScan(scan: proto.Scan, planner: SparkConnectPlanner): LogicalPlan = {
- val customTable = getCustomTable(scan.getTable)
- // Convert the custom table to a DataFrame and get its logical plan
- customTable.toDF().queryExecution.analyzed
- }
-}
diff --git a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala b/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala
deleted file mode 100644
index e1630cc25edd4..0000000000000
--- a/connect-examples/server-library-example/server/src/main/scala/org/apache/connect/examples/serverlibrary/CustomTable.scala
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.connect.examples.serverlibrary
-
-import java.util.UUID
-
-import com.github.mrpowers.spark.daria.sql.DariaWriters
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.sql.{Dataset, Row, SparkSession}
-
-/**
- * Represents a custom table with an identifier and a DataFrame.
- *
- * @param identifier The unique identifier for the table.
- * @param df The DataFrame associated with the table.
- */
-class CustomTable private (identifier: CustomTable.Identifier, df: Dataset[Row]) {
-
- /**
- * Returns the DataFrame associated with the table.
- *
- * @return The DataFrame.
- */
- def toDF(): Dataset[Row] = df
-
- /**
- * Writes the DataFrame to disk as a CSV file.
- */
- def flush(): Unit = {
- // Write dataset to disk as a CSV file
- DariaWriters.writeSingleFile(
- df = df,
- format = "csv",
- sc = df.sparkSession.sparkContext,
- tmpFolder = s"./${UUID.randomUUID().toString}",
- filename = identifier.path,
- saveMode = "overwrite")
- }
-}
-
-object CustomTable {
-
- /**
- * Represents the unique identifier for a custom table.
- *
- * @param name The name of the table.
- * @param path The path where the table is stored.
- */
- private case class Identifier(name: String, path: String)
-
- // Collection holding all the CustomTable instances and searchable by the identifier
- private val tablesByIdentifier = scala.collection.mutable.Map[Identifier, CustomTable]()
-
- /**
- * Creates a new custom table.
- *
- * @param name The name of the table.
- * @param path The path where the table is stored.
- * @param spark The SparkSession instance.
- * @param schema The schema of the table.
- * @return The created CustomTable instance.
- */
- private[serverlibrary] def createTable(
- name: String,
- path: String,
- spark: SparkSession,
- schema: StructType): CustomTable = {
- val identifier = Identifier(name, path)
- val df = spark.read
- .option("header", "true")
- .schema(schema)
- .csv(path)
- val table = new CustomTable(identifier, df)
- tablesByIdentifier(identifier) = table
- table
- }
-
- /**
- * Clones an existing custom table.
- *
- * @param sourceTable The source table to clone.
- * @param newName The name of the new table.
- * @param newPath The path where the new table will be stored.
- * @param replace Whether to replace the existing table if it exists.
- * @return The cloned CustomTable instance.
- */
- private[serverlibrary] def cloneTable(
- sourceTable: CustomTable,
- newName: String,
- newPath: String,
- replace: Boolean): CustomTable = {
- val newIdentifier = Identifier(newName, newPath)
- val clonedDf = sourceTable.toDF()
- val clonedTable = new CustomTable(newIdentifier, clonedDf)
- clonedTable.flush()
- tablesByIdentifier(newIdentifier) = clonedTable
- clonedTable
- }
-
- /**
- * Retrieves a custom table based on its identifier.
- *
- * @param name The name of the table.
- * @param path The path where the table is stored.
- * @return The CustomTable instance.
- * @throws IllegalArgumentException if the table is not found.
- */
- def getTable(name: String, path: String): CustomTable = {
- val identifier = Identifier(name, path)
- tablesByIdentifier.get(identifier) match {
- case Some(table) => table
- case None =>
- throw new IllegalArgumentException(s"Table with identifier $identifier not found")
- }
- }
-}
diff --git a/connector/avro/benchmarks/AvroReadBenchmark-jdk21-results.txt b/connector/avro/benchmarks/AvroReadBenchmark-jdk21-results.txt
index e01810e1e970c..181508f416ad4 100644
--- a/connector/avro/benchmarks/AvroReadBenchmark-jdk21-results.txt
+++ b/connector/avro/benchmarks/AvroReadBenchmark-jdk21-results.txt
@@ -2,140 +2,140 @@
SQL Single Numeric Column Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single TINYINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 2045 2062 24 7.7 130.0 1.0X
+Sum 2024 2082 83 7.8 128.7 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single SMALLINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 2033 2040 11 7.7 129.2 1.0X
+Sum 2032 2047 21 7.7 129.2 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single INT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1959 1965 8 8.0 124.6 1.0X
+Sum 1990 2019 40 7.9 126.5 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single BIGINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1983 2019 51 7.9 126.1 1.0X
+Sum 1988 1989 2 7.9 126.4 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single FLOAT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1935 1941 7 8.1 123.1 1.0X
+Sum 1992 2015 32 7.9 126.6 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single DOUBLE Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1960 1976 23 8.0 124.6 1.0X
+Sum 1976 2008 45 8.0 125.6 1.0X
================================================================================================
Int and String Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Int and String Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of columns 3561 3573 16 2.9 339.6 1.0X
+Sum of columns 3538 3559 29 3.0 337.4 1.0X
================================================================================================
Partitioned Table Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Partitioned Table: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Data column 2138 2158 28 7.4 135.9 1.0X
-Partition column 1817 1828 15 8.7 115.5 1.2X
-Both columns 2066 2092 36 7.6 131.4 1.0X
+Data column 2317 2368 73 6.8 147.3 1.0X
+Partition column 1988 1990 3 7.9 126.4 1.2X
+Both columns 2280 2360 114 6.9 144.9 1.0X
================================================================================================
Repeated String Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Repeated String: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 2183 2212 41 4.8 208.2 1.0X
+Sum of string length 2125 2148 33 4.9 202.7 1.0X
================================================================================================
String with Nulls Scan
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (0.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 3278 3296 26 3.2 312.6 1.0X
+Sum of string length 3365 3371 8 3.1 321.0 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (50.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 2148 2152 5 4.9 204.9 1.0X
+Sum of string length 2153 2159 8 4.9 205.4 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (95.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 1128 1129 1 9.3 107.6 1.0X
+Sum of string length 1128 1131 5 9.3 107.6 1.0X
================================================================================================
Select All From Wide Columns
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Wide Column Scan from 1000 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Select of all columns 20327 20405 111 0.0 40653.8 1.0X
+Select of all columns 19734 20045 439 0.0 39467.9 1.0X
================================================================================================
Single Column Scan From Wide Columns
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 100 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 3365 3372 9 0.3 3209.4 1.0X
+Sum of single column 3445 3445 1 0.3 3285.3 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 200 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 6611 6633 32 0.2 6304.7 1.0X
+Sum of single column 6726 6730 7 0.2 6414.0 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 300 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 10030 10036 9 0.1 9565.3 1.0X
+Sum of single column 10099 10107 12 0.1 9631.1 1.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Filters pushdown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-w/o filters 5455 5470 21 0.2 5455.0 1.0X
-pushdown disabled 5362 5367 5 0.2 5362.0 1.0X
-w/ filters 2367 2384 15 0.4 2367.4 2.3X
+w/o filters 5598 5610 18 0.2 5598.4 1.0X
+pushdown disabled 5501 5520 18 0.2 5500.6 1.0X
+w/ filters 2332 2350 23 0.4 2332.2 2.4X
diff --git a/connector/avro/benchmarks/AvroReadBenchmark-results.txt b/connector/avro/benchmarks/AvroReadBenchmark-results.txt
index 85f3a34340cd7..3a41c98083171 100644
--- a/connector/avro/benchmarks/AvroReadBenchmark-results.txt
+++ b/connector/avro/benchmarks/AvroReadBenchmark-results.txt
@@ -2,140 +2,140 @@
SQL Single Numeric Column Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single TINYINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1925 1972 66 8.2 122.4 1.0X
+Sum 2001 2017 23 7.9 127.2 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single SMALLINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1958 1971 18 8.0 124.5 1.0X
+Sum 1980 1981 2 7.9 125.9 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single INT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1918 1920 4 8.2 121.9 1.0X
+Sum 1972 1973 1 8.0 125.4 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single BIGINT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1877 1888 16 8.4 119.3 1.0X
+Sum 1937 1964 39 8.1 123.1 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single FLOAT Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1886 1891 7 8.3 119.9 1.0X
+Sum 1905 1934 41 8.3 121.1 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
SQL Single DOUBLE Column Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum 1870 1871 1 8.4 118.9 1.0X
+Sum 1884 1888 5 8.3 119.8 1.0X
================================================================================================
Int and String Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Int and String Scan: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of columns 3539 3544 7 3.0 337.5 1.0X
+Sum of columns 3423 3462 54 3.1 326.5 1.0X
================================================================================================
Partitioned Table Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Partitioned Table: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Data column 1949 1950 1 8.1 123.9 1.0X
-Partition column 1780 1791 16 8.8 113.2 1.1X
-Both columns 1985 2001 23 7.9 126.2 1.0X
+Data column 2067 2068 1 7.6 131.4 1.0X
+Partition column 1924 1954 43 8.2 122.3 1.1X
+Both columns 2222 2269 67 7.1 141.2 0.9X
================================================================================================
Repeated String Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Repeated String: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 2081 2096 21 5.0 198.4 1.0X
+Sum of string length 1991 2029 53 5.3 189.9 1.0X
================================================================================================
String with Nulls Scan
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (0.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 3094 3096 2 3.4 295.1 1.0X
+Sum of string length 3148 3150 2 3.3 300.2 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (50.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 2161 2185 34 4.9 206.1 1.0X
+Sum of string length 2092 2092 0 5.0 199.5 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
String with Nulls Scan (95.0%): Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of string length 1106 1110 6 9.5 105.4 1.0X
+Sum of string length 1126 1131 8 9.3 107.3 1.0X
================================================================================================
Select All From Wide Columns
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Wide Column Scan from 1000 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Select of all columns 18000 18000 0 0.0 35999.1 1.0X
+Select of all columns 19944 20078 190 0.0 39887.7 1.0X
================================================================================================
Single Column Scan From Wide Columns
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 100 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 3094 3103 13 0.3 2950.6 1.0X
+Sum of single column 3555 3630 106 0.3 3390.3 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 200 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 6074 6083 13 0.2 5792.7 1.0X
+Sum of single column 6793 6796 5 0.2 6478.4 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Single Column Scan from 300 columns: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Sum of single column 9105 9122 23 0.1 8683.6 1.0X
+Sum of single column 10218 10229 16 0.1 9744.2 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Filters pushdown: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-w/o filters 5123 5132 12 0.2 5122.5 1.0X
-pushdown disabled 5071 5085 20 0.2 5070.8 1.0X
-w/ filters 1967 1984 26 0.5 1967.2 2.6X
+w/o filters 5835 5841 7 0.2 5835.4 1.0X
+pushdown disabled 5703 5716 13 0.2 5703.1 1.0X
+w/ filters 2217 2227 10 0.5 2217.5 2.6X
diff --git a/connector/avro/benchmarks/AvroWriteBenchmark-jdk21-results.txt b/connector/avro/benchmarks/AvroWriteBenchmark-jdk21-results.txt
index 2014407498850..b18b2661be071 100644
--- a/connector/avro/benchmarks/AvroWriteBenchmark-jdk21-results.txt
+++ b/connector/avro/benchmarks/AvroWriteBenchmark-jdk21-results.txt
@@ -1,56 +1,56 @@
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro writer benchmark: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column 1672 1680 12 9.4 106.3 1.0X
-Output Single Double Column 1572 1572 0 10.0 100.0 1.1X
-Output Int and String Column 3337 3351 20 4.7 212.1 0.5X
-Output Partitions 2984 2998 20 5.3 189.7 0.6X
-Output Buckets 3939 3948 14 4.0 250.4 0.4X
+Output Single Int Column 1651 1721 98 9.5 105.0 1.0X
+Output Single Double Column 1856 1864 12 8.5 118.0 0.9X
+Output Int and String Column 3670 3671 1 4.3 233.3 0.5X
+Output Partitions 3288 3290 2 4.8 209.0 0.5X
+Output Buckets 3909 3935 37 4.0 248.5 0.4X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro compression with different codec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-BZIP2: 117102 117155 74 0.0 1171024.1 1.0X
-DEFLATE: 6412 6438 36 0.0 64121.2 18.3X
-UNCOMPRESSED: 5015 5034 26 0.0 50153.9 23.3X
-SNAPPY: 4540 4542 2 0.0 45403.0 25.8X
-XZ: 41913 42574 934 0.0 419134.4 2.8X
-ZSTANDARD: 4807 4848 58 0.0 48065.1 24.4X
+BZIP2: 122117 122139 32 0.0 1221169.3 1.0X
+DEFLATE: 6544 6576 46 0.0 65435.4 18.7X
+UNCOMPRESSED: 5013 5028 22 0.0 50129.3 24.4X
+SNAPPY: 4664 4676 17 0.0 46641.2 26.2X
+XZ: 43536 43997 652 0.0 435362.0 2.8X
+ZSTANDARD: 4882 4901 27 0.0 48819.9 25.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro deflate with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-DEFLATE: deflate.level=1 4691 4719 40 0.0 46909.5 1.0X
-DEFLATE: deflate.level=3 4773 4773 0 0.0 47731.7 1.0X
-DEFLATE: deflate.level=5 6225 6233 11 0.0 62254.0 0.8X
-DEFLATE: deflate.level=7 6308 6323 21 0.0 63075.8 0.7X
-DEFLATE: deflate.level=9 6632 6639 9 0.0 66320.6 0.7X
+DEFLATE: deflate.level=1 4863 4874 15 0.0 48629.6 1.0X
+DEFLATE: deflate.level=3 4915 4919 6 0.0 49148.2 1.0X
+DEFLATE: deflate.level=5 6512 6521 14 0.0 65117.5 0.7X
+DEFLATE: deflate.level=7 6580 6595 22 0.0 65798.8 0.7X
+DEFLATE: deflate.level=9 6829 6868 56 0.0 68288.6 0.7X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro xz with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-XZ: xz.level=1 11561 11582 30 0.0 115608.9 1.0X
-XZ: xz.level=3 21560 21618 82 0.0 215604.1 0.5X
-XZ: xz.level=5 40673 40707 48 0.0 406733.7 0.3X
-XZ: xz.level=7 58646 59202 787 0.0 586455.4 0.2X
-XZ: xz.level=9 130798 131012 302 0.0 1307981.2 0.1X
+XZ: xz.level=1 11635 11646 16 0.0 116350.7 1.0X
+XZ: xz.level=3 21619 21735 165 0.0 216185.2 0.5X
+XZ: xz.level=5 41636 41887 355 0.0 416358.8 0.3X
+XZ: xz.level=7 61117 61329 301 0.0 611167.0 0.2X
+XZ: xz.level=9 140074 142026 2761 0.0 1400739.5 0.1X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro zstandard with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------------------
-ZSTANDARD: zstandard.level=1 4842 4883 58 0.0 48423.9 1.0X
-ZSTANDARD: zstandard.level=1, zstandard.bufferPool.enabled=true 4659 4685 37 0.0 46593.3 1.0X
-ZSTANDARD: zstandard.level=3 4808 4874 93 0.0 48075.0 1.0X
-ZSTANDARD: zstandard.level=3, zstandard.bufferPool.enabled=true 4554 4586 44 0.0 45541.4 1.1X
-ZSTANDARD: zstandard.level=5 5024 5026 2 0.0 50244.9 1.0X
-ZSTANDARD: zstandard.level=5, zstandard.bufferPool.enabled=true 4810 4812 3 0.0 48099.6 1.0X
-ZSTANDARD: zstandard.level=7 5223 5237 20 0.0 52232.5 0.9X
-ZSTANDARD: zstandard.level=7, zstandard.bufferPool.enabled=true 5184 5192 12 0.0 51837.0 0.9X
-ZSTANDARD: zstandard.level=9 6023 6059 52 0.0 60226.6 0.8X
-ZSTANDARD: zstandard.level=9, zstandard.bufferPool.enabled=true 5886 5896 15 0.0 58855.8 0.8X
+ZSTANDARD: zstandard.level=1 4785 4831 65 0.0 47853.1 1.0X
+ZSTANDARD: zstandard.level=1, zstandard.bufferPool.enabled=true 4843 4852 12 0.0 48430.1 1.0X
+ZSTANDARD: zstandard.level=3 4934 4954 28 0.0 49341.5 1.0X
+ZSTANDARD: zstandard.level=3, zstandard.bufferPool.enabled=true 4831 4832 2 0.0 48307.1 1.0X
+ZSTANDARD: zstandard.level=5 5157 5311 218 0.0 51573.5 0.9X
+ZSTANDARD: zstandard.level=5, zstandard.bufferPool.enabled=true 4956 4959 4 0.0 49560.3 1.0X
+ZSTANDARD: zstandard.level=7 5394 5435 59 0.0 53939.4 0.9X
+ZSTANDARD: zstandard.level=7, zstandard.bufferPool.enabled=true 5273 5333 84 0.0 52734.7 0.9X
+ZSTANDARD: zstandard.level=9 5954 6054 142 0.0 59536.1 0.8X
+ZSTANDARD: zstandard.level=9, zstandard.bufferPool.enabled=true 5930 6060 184 0.0 59295.7 0.8X
diff --git a/connector/avro/benchmarks/AvroWriteBenchmark-results.txt b/connector/avro/benchmarks/AvroWriteBenchmark-results.txt
index 4582347ceb11b..cfe3eb7a08300 100644
--- a/connector/avro/benchmarks/AvroWriteBenchmark-results.txt
+++ b/connector/avro/benchmarks/AvroWriteBenchmark-results.txt
@@ -1,56 +1,56 @@
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro writer benchmark: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Output Single Int Column 1556 1579 33 10.1 98.9 1.0X
-Output Single Double Column 1591 1601 14 9.9 101.2 1.0X
-Output Int and String Column 3363 3379 22 4.7 213.8 0.5X
-Output Partitions 2937 2952 21 5.4 186.8 0.5X
-Output Buckets 3823 3825 3 4.1 243.0 0.4X
+Output Single Int Column 1559 1630 99 10.1 99.1 1.0X
+Output Single Double Column 1762 1804 59 8.9 112.0 0.9X
+Output Int and String Column 3831 3832 1 4.1 243.6 0.4X
+Output Partitions 2914 2944 41 5.4 185.3 0.5X
+Output Buckets 3718 3725 11 4.2 236.4 0.4X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro compression with different codec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-BZIP2: 132833 133636 1136 0.0 1328332.0 1.0X
-DEFLATE: 6522 6554 45 0.0 65216.8 20.4X
-UNCOMPRESSED: 5131 5150 27 0.0 51311.0 25.9X
-SNAPPY: 4709 4766 80 0.0 47091.8 28.2X
-XZ: 42816 43168 498 0.0 428160.6 3.1X
-ZSTANDARD: 4835 4850 22 0.0 48345.6 27.5X
+BZIP2: 132034 132397 514 0.0 1320336.8 1.0X
+DEFLATE: 6582 6587 8 0.0 65820.4 20.1X
+UNCOMPRESSED: 5144 5147 5 0.0 51436.8 25.7X
+SNAPPY: 4756 4757 1 0.0 47563.8 27.8X
+XZ: 50584 51077 697 0.0 505841.3 2.6X
+ZSTANDARD: 4930 4938 11 0.0 49302.3 26.8X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro deflate with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-DEFLATE: deflate.level=1 4832 4836 6 0.0 48320.1 1.0X
-DEFLATE: deflate.level=3 4798 4818 28 0.0 47979.1 1.0X
-DEFLATE: deflate.level=5 6472 6473 2 0.0 64717.5 0.7X
-DEFLATE: deflate.level=7 6516 6519 5 0.0 65161.0 0.7X
-DEFLATE: deflate.level=9 6877 6879 3 0.0 68768.4 0.7X
+DEFLATE: deflate.level=1 4819 4851 45 0.0 48186.1 1.0X
+DEFLATE: deflate.level=3 4943 5010 94 0.0 49427.9 1.0X
+DEFLATE: deflate.level=5 6366 6381 22 0.0 63658.6 0.8X
+DEFLATE: deflate.level=7 6465 6476 16 0.0 64647.8 0.7X
+DEFLATE: deflate.level=9 6768 6798 44 0.0 67675.3 0.7X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro xz with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-XZ: xz.level=1 11901 11909 11 0.0 119013.0 1.0X
-XZ: xz.level=3 22159 22204 64 0.0 221592.3 0.5X
-XZ: xz.level=5 41324 41591 378 0.0 413243.8 0.3X
-XZ: xz.level=7 59391 59735 487 0.0 593906.4 0.2X
-XZ: xz.level=9 132884 135160 3219 0.0 1328837.1 0.1X
+XZ: xz.level=1 12255 12266 15 0.0 122554.1 1.0X
+XZ: xz.level=3 23906 23913 10 0.0 239062.9 0.5X
+XZ: xz.level=5 43890 44695 1139 0.0 438895.6 0.3X
+XZ: xz.level=7 71200 72059 1215 0.0 712003.3 0.2X
+XZ: xz.level=9 157018 160566 5018 0.0 1570183.3 0.1X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Avro zstandard with different levels: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------------------
-ZSTANDARD: zstandard.level=1 4707 4753 65 0.0 47073.3 1.0X
-ZSTANDARD: zstandard.level=1, zstandard.bufferPool.enabled=true 4701 4717 23 0.0 47006.8 1.0X
-ZSTANDARD: zstandard.level=3 4810 4857 67 0.0 48096.6 1.0X
-ZSTANDARD: zstandard.level=3, zstandard.bufferPool.enabled=true 4768 4799 44 0.0 47684.0 1.0X
-ZSTANDARD: zstandard.level=5 5093 5134 59 0.0 50928.2 0.9X
-ZSTANDARD: zstandard.level=5, zstandard.bufferPool.enabled=true 5033 5052 27 0.0 50331.4 0.9X
-ZSTANDARD: zstandard.level=7 5538 5554 22 0.0 55381.9 0.8X
-ZSTANDARD: zstandard.level=7, zstandard.bufferPool.enabled=true 5479 5484 7 0.0 54790.8 0.9X
-ZSTANDARD: zstandard.level=9 6233 6347 161 0.0 62331.1 0.8X
-ZSTANDARD: zstandard.level=9, zstandard.bufferPool.enabled=true 6266 6319 74 0.0 62663.4 0.8X
+ZSTANDARD: zstandard.level=1 4733 4760 37 0.0 47334.4 1.0X
+ZSTANDARD: zstandard.level=1, zstandard.bufferPool.enabled=true 4650 4664 19 0.0 46501.3 1.0X
+ZSTANDARD: zstandard.level=3 4799 4835 51 0.0 47986.4 1.0X
+ZSTANDARD: zstandard.level=3, zstandard.bufferPool.enabled=true 4722 4748 36 0.0 47224.8 1.0X
+ZSTANDARD: zstandard.level=5 5118 5178 85 0.0 51184.0 0.9X
+ZSTANDARD: zstandard.level=5, zstandard.bufferPool.enabled=true 5002 5008 8 0.0 50019.6 0.9X
+ZSTANDARD: zstandard.level=7 5492 5540 69 0.0 54916.5 0.9X
+ZSTANDARD: zstandard.level=7, zstandard.bufferPool.enabled=true 5380 5381 0 0.0 53801.9 0.9X
+ZSTANDARD: zstandard.level=9 6198 6207 13 0.0 61975.9 0.8X
+ZSTANDARD: zstandard.level=9, zstandard.bufferPool.enabled=true 5839 5885 65 0.0 58392.7 0.8X
diff --git a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 6f345e069ff78..c78bc62126620 100644
--- a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -32,7 +32,6 @@ import org.apache.avro.Schema.Type._
import org.apache.avro.file.{DataFileReader, DataFileWriter}
import org.apache.avro.generic.{GenericData, GenericDatumReader, GenericDatumWriter, GenericRecord}
import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed}
-import org.apache.commons.io.FileUtils
import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf, SparkException, SparkRuntimeException, SparkThrowable, SparkUpgradeException}
import org.apache.spark.TestUtils.assertExceptionMsg
@@ -640,7 +639,7 @@ abstract class AvroSuite
private def createDummyCorruptFile(dir: File): Unit = {
Utils.tryWithResource {
- FileUtils.forceMkdir(dir)
+ Files.createDirectories(dir.toPath)
val corruptFile = new File(dir, "corrupt.avro")
new BufferedWriter(new FileWriter(corruptFile))
} { writer =>
@@ -760,12 +759,12 @@ abstract class AvroSuite
spark.conf.set(SQLConf.AVRO_COMPRESSION_CODEC.key, ZSTANDARD.lowerCaseName())
df.write.format("avro").save(zstandardDir)
- val uncompressSize = FileUtils.sizeOfDirectory(new File(uncompressDir))
- val bzip2Size = FileUtils.sizeOfDirectory(new File(bzip2Dir))
- val xzSize = FileUtils.sizeOfDirectory(new File(xzDir))
- val deflateSize = FileUtils.sizeOfDirectory(new File(deflateDir))
- val snappySize = FileUtils.sizeOfDirectory(new File(snappyDir))
- val zstandardSize = FileUtils.sizeOfDirectory(new File(zstandardDir))
+ val uncompressSize = Utils.sizeOf(new File(uncompressDir))
+ val bzip2Size = Utils.sizeOf(new File(bzip2Dir))
+ val xzSize = Utils.sizeOf(new File(xzDir))
+ val deflateSize = Utils.sizeOf(new File(deflateDir))
+ val snappySize = Utils.sizeOf(new File(snappyDir))
+ val zstandardSize = Utils.sizeOf(new File(zstandardDir))
assert(uncompressSize > deflateSize)
assert(snappySize > deflateSize)
@@ -1875,7 +1874,7 @@ abstract class AvroSuite
intercept[FileNotFoundException] {
withTempDir { dir =>
- FileUtils.touch(new File(dir, "test"))
+ Utils.touch(new File(dir, "test"))
withSQLConf(AvroFileFormat.IgnoreFilesWithoutExtensionProperty -> "true") {
spark.read.format("avro").load(dir.toString)
}
@@ -1884,7 +1883,7 @@ abstract class AvroSuite
intercept[FileNotFoundException] {
withTempDir { dir =>
- FileUtils.touch(new File(dir, "test"))
+ Utils.touch(new File(dir, "test"))
spark
.read
@@ -1899,7 +1898,7 @@ abstract class AvroSuite
withTempPath { tempDir =>
val tempEmptyDir = s"$tempDir/sqlOverwrite"
// Create a temp directory for table that will be overwritten
- new File(tempEmptyDir).mkdirs()
+ Utils.createDirectory(tempEmptyDir)
spark.sql(
s"""
|CREATE TEMPORARY VIEW episodes
diff --git a/connector/docker-integration-tests/README.md b/connector/docker-integration-tests/README.md
index 385218097a3cb..0e77a2816c3aa 100644
--- a/connector/docker-integration-tests/README.md
+++ b/connector/docker-integration-tests/README.md
@@ -45,7 +45,7 @@ the container bootstrapping. To run an individual Docker integration test, use t
Besides the default Docker images, the integration tests can be run with custom Docker images. For example,
- ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-free:23.7-slim ./build/sbt -Pdocker-integration-tests "docker-integration-tests/testOnly *OracleIntegrationSuite"
+ ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-free:23.9-slim ./build/sbt -Pdocker-integration-tests "docker-integration-tests/testOnly *OracleIntegrationSuite"
The following environment variables can be used to specify the custom Docker images for different databases:
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
index 25695f144f558..c2d83943e3222 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DB2IntegrationSuite.scala
@@ -37,7 +37,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class DB2IntegrationSuite extends DockerJDBCIntegrationSuite with SharedJDBCIntegrationTests {
+class DB2IntegrationSuite extends SharedJDBCIntegrationSuite {
override val db = new DB2DatabaseOnDocker
override def dataPreparation(conn: Connection): Unit = {
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
index 1df01bd3bfb62..0062b03575930 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
@@ -37,7 +37,6 @@ import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
import org.scalatest.time.SpanSugar._
import org.apache.spark.internal.LogKeys.{CLASS_NAME, CONTAINER, STATUS}
-import org.apache.spark.internal.MDC
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.util.{DockerUtils, Utils}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala
index 53e45baa99679..4f782bed77922 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MariaDBKrbIntegrationSuite.scala
@@ -33,8 +33,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class MariaDBKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite
- with SharedJDBCIntegrationTests {
+class MariaDBKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite {
override protected val userName = s"mariadb/$dockerIp"
override protected val keytabFileName = "mariadb.keytab"
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
index f72f3ecca6929..0950fd7330c5a 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
@@ -40,8 +40,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite
- with SharedJDBCIntegrationTests {
+class MsSqlServerIntegrationSuite extends SharedJDBCIntegrationSuite {
override val db = new MsSQLServerDatabaseOnDocker
override def dataPreparation(conn: Connection): Unit = {
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
index 7244c5547abb3..f74a24c6319da 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
@@ -39,7 +39,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite {
+class MySQLIntegrationSuite extends SharedJDBCIntegrationSuite {
override val db = new MySQLDatabaseOnDocker
override def dataPreparation(conn: Connection): Unit = {
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala
index 8656d23a2e6a0..e32563c7e94bd 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleDatabaseOnDocker.scala
@@ -21,7 +21,7 @@ import org.apache.spark.internal.Logging
class OracleDatabaseOnDocker extends DatabaseOnDocker with Logging {
lazy override val imageName =
- sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.7-slim")
+ sys.env.getOrElse("ORACLE_DOCKER_IMAGE_NAME", "gvenzl/oracle-free:23.9-slim")
val oracle_password = "Th1s1sThe0racle#Pass"
override val env = Map(
"ORACLE_PWD" -> oracle_password, // oracle images uses this
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
index 33a89f4f20c4c..d44e2d1e748fd 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
@@ -61,9 +61,8 @@ import org.apache.spark.tags.DockerTest
* and with Oracle Express Edition versions 18.4.0 and 21.4.0
*/
@DockerTest
-class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
- with SharedSparkSession
- with SharedJDBCIntegrationTests {
+class OracleIntegrationSuite extends SharedJDBCIntegrationSuite
+ with SharedSparkSession {
import testImplicits._
override val db = new OracleDatabaseOnDocker
@@ -458,7 +457,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
.load()
df1.logicalPlan match {
- case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+ case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
assert(whereClauses === Set(
""""D" < '2018-07-11' or "D" is null""",
@@ -481,7 +480,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
.load()
df2.logicalPlan match {
- case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+ case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
assert(whereClauses === Set(
""""T" < '2018-07-15 20:50:32.5' or "T" is null""",
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
index 9d1186463ca2a..050f74f369034 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
@@ -40,7 +40,7 @@ import org.apache.spark.tags.DockerTest
* }}}
*/
@DockerTest
-class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite with SharedJDBCIntegrationTests {
+class PostgresIntegrationSuite extends SharedJDBCIntegrationSuite {
override val db = new PostgresDatabaseOnDocker
override def dataPreparation(conn: Connection): Unit = {
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala
new file mode 100644
index 0000000000000..e42d04ead0871
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.sql.Connection
+
+import org.scalatest.time.SpanSugar._
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.Row
+
+abstract class SharedJDBCIntegrationSuite extends DockerJDBCIntegrationSuite {
+
+ override def beforeAll(): Unit = runIfTestsEnabled(s"Prepare for ${this.getClass.getName}") {
+ super.beforeAll()
+ var conn: Connection = null
+ eventually(connectionTimeout, interval(1.second)) {
+ conn = getConnection()
+ }
+ try {
+ createSharedTable(conn)
+ } finally {
+ conn.close()
+ }
+ }
+
+ /**
+ * Create a table with the same name that can be used to test common functionality
+ * in
+ * @param conn
+ */
+ def createSharedTable(conn: Connection): Unit = {
+ val batchStmt = conn.createStatement()
+
+ batchStmt.addBatch("CREATE TABLE tbl_shared (x INTEGER)")
+ batchStmt.addBatch("INSERT INTO tbl_shared VALUES(1)")
+
+ batchStmt.executeBatch()
+ batchStmt.close()
+ }
+
+ test("SPARK-52184: Wrap external engine syntax error") {
+ val e = intercept[SparkException] {
+
+ spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", "THIS IS NOT VALID SQL").load()
+ }
+ assert(e.getCondition.startsWith("JDBC_EXTERNAL_ENGINE_SYNTAX_ERROR"))
+ }
+
+ test("SPARK-53386: Parameter `query` should work when ending with semicolons") {
+ val dfSingle = spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", "SELECT x FROM tbl_shared; ")
+ .load()
+ checkAnswer(dfSingle, Seq(Row(1)))
+
+ val dfMultiple = spark.read.format("jdbc")
+ .option("url", jdbcUrl)
+ .option("query", "SELECT x FROM tbl_shared;;;")
+ .load()
+ checkAnswer(dfMultiple, Seq(Row(1)))
+ }
+}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala
index a693920c10032..c959aae8e91e6 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala
@@ -14,9 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.jdbc
+package org.apache.spark.sql.jdbc.querytest
import java.io.File
+import java.nio.file.Files
import java.sql.ResultSet
import scala.collection.mutable.ArrayBuffer
@@ -24,7 +25,7 @@ import scala.util.control.NonFatal
import org.apache.spark.sql.Row
import org.apache.spark.sql.SQLQueryTestHelper
-import org.apache.spark.sql.catalyst.util.fileToString
+import org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite
/**
* This suite builds off of that to allow us to run other DBMS against the SQL test golden files (on
@@ -76,7 +77,7 @@ trait CrossDbmsQueryTestSuite extends DockerJDBCIntegrationSuite with SQLQueryTe
}
protected def runSqlTestCase(testCase: TestCase, listTestCases: Seq[TestCase]): Unit = {
- val input = fileToString(new File(testCase.inputFile))
+ val input = Files.readString(new File(testCase.inputFile).toPath)
val (comments, code) = splitCommentsAndCodes(input)
val queries = getQueries(code, comments, listTestCases)
@@ -143,7 +144,7 @@ trait CrossDbmsQueryTestSuite extends DockerJDBCIntegrationSuite with SQLQueryTe
// Read back the golden files.
var curSegment = 0
val expectedOutputs: Seq[QueryTestOutput] = {
- val goldenOutput = fileToString(new File(testCase.resultFile))
+ val goldenOutput = Files.readString(new File(testCase.resultFile).toPath)
val segments = goldenOutput.split("-- !query.*\n")
outputs.map { output =>
val result =
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala
index 3a1d5e18b7e5a..eabdb033ebc24 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.jdbc
+package org.apache.spark.sql.jdbc.querytest
import java.sql.{Connection, ResultSet, Statement}
import java.util.Locale
@@ -24,6 +24,7 @@ import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.{QueryGeneratorHelper, QueryTest}
import org.apache.spark.sql.Row
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, PostgresDatabaseOnDocker}
import org.apache.spark.tags.DockerTest
/**
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala
index 28320a9e0a949..4b7fc0c3e20cd 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala
@@ -14,11 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.jdbc
+package org.apache.spark.sql.jdbc.querytest
import java.io.File
import java.sql.Connection
+import org.apache.spark.sql.jdbc.PostgresDatabaseOnDocker
import org.apache.spark.tags.DockerTest
/**
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DockerJDBCIntegrationV2Suite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DockerJDBCIntegrationV2Suite.scala
index 60345257f2dc4..2a66845f46ed5 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DockerJDBCIntegrationV2Suite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/DockerJDBCIntegrationV2Suite.scala
@@ -19,10 +19,17 @@ package org.apache.spark.sql.jdbc.v2
import java.sql.Connection
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.jdbc.DockerJDBCIntegrationSuite
abstract class DockerJDBCIntegrationV2Suite extends DockerJDBCIntegrationSuite {
+ override def sparkConf: SparkConf = super.sparkConf
+ // DS V2 relies on ANSI mode to translate expressions, we should always
+ // run JDBC v2 tests with ANSI on.
+ .set(SQLConf.ANSI_ENABLED, true)
+
/**
* Prepare databases and tables for testing.
*/
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala
index 7a58fca17970f..c71f9ae7688f3 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala
@@ -118,11 +118,28 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTes
"CREATE TABLE employee (dept NUMBER(32), name VARCHAR2(32), salary NUMBER(20, 2)," +
" bonus BINARY_DOUBLE)").executeUpdate()
connection.prepareStatement(
- s"""CREATE TABLE pattern_testing_table (
- |pattern_testing_col VARCHAR(50)
- |)
- """.stripMargin
+ """CREATE TABLE pattern_testing_table (
+ |pattern_testing_col VARCHAR(50)
+ |)
+ """.stripMargin
).executeUpdate()
+ connection.prepareStatement(
+ "CREATE TABLE datetime (name VARCHAR(32), date1 DATE, time1 TIMESTAMP)")
+ .executeUpdate()
+ }
+
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation(connection)
+ connection.prepareStatement(
+ "INSERT INTO datetime VALUES ('amy', TO_DATE('2022-05-19', 'YYYY-MM-DD')," +
+ " TO_TIMESTAMP('2022-05-19 00:00:00', 'YYYY-MM-DD HH24:MI:SS'))").executeUpdate()
+ connection.prepareStatement(
+ "INSERT INTO datetime VALUES ('alex', TO_DATE('2022-05-18', 'YYYY-MM-DD')," +
+ " TO_TIMESTAMP('2022-05-18 00:00:00', 'YYYY-MM-DD HH24:MI:SS'))").executeUpdate()
+ // '2022-01-01' is Saturday and is in ISO year 2021.
+ connection.prepareStatement(
+ "INSERT INTO datetime VALUES ('tom', TO_DATE('2022-01-01', 'YYYY-MM-DD')," +
+ " TO_TIMESTAMP('2022-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS'))").executeUpdate()
}
override def testUpdateColumnType(tbl: String): Unit = {
@@ -185,4 +202,143 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTes
checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row("Eason", "Y ")))
}
}
+
+ override def testDatetime(tbl: String): Unit = {
+ val df1 = sql(s"SELECT name FROM $tbl WHERE " +
+ "dayofyear(date1) > 100 AND dayofmonth(date1) > 10 ")
+ checkFilterPushed(df1, false)
+ val rows1 = df1.collect()
+ assert(rows1.length === 2)
+ assert(rows1(0).getString(0) === "amy")
+ assert(rows1(1).getString(0) === "alex")
+
+ val df2 = sql(s"SELECT name FROM $tbl WHERE year(date1) = 2022 AND quarter(date1) = 2")
+ checkFilterPushed(df2, false)
+ val rows2 = df2.collect()
+ assert(rows2.length === 2)
+ assert(rows2(0).getString(0) === "amy")
+ assert(rows2(1).getString(0) === "alex")
+
+ val df3 = sql(s"SELECT name FROM $tbl WHERE month(date1) = 5")
+ checkFilterPushed(df3)
+ val rows3 = df3.collect()
+ assert(rows3.length === 2)
+ assert(rows3(0).getString(0) === "amy")
+ assert(rows3(1).getString(0) === "alex")
+
+ val df4 = sql(s"SELECT name FROM $tbl WHERE hour(time1) = 0 AND minute(time1) = 0")
+ checkFilterPushed(df4)
+ val rows4 = df4.collect()
+ assert(rows4.length === 3)
+ assert(rows4(0).getString(0) === "amy")
+ assert(rows4(1).getString(0) === "alex")
+ assert(rows4(2).getString(0) === "tom")
+
+ val df5 = sql(s"SELECT name FROM $tbl WHERE " +
+ "extract(WEEK from date1) > 10 AND extract(YEAR from date1) = 2022")
+ checkFilterPushed(df5, false)
+ val rows5 = df5.collect()
+ assert(rows5.length === 3)
+ assert(rows5(0).getString(0) === "amy")
+ assert(rows5(1).getString(0) === "alex")
+ assert(rows5(2).getString(0) === "tom")
+
+ val df6 = sql(s"SELECT name FROM $tbl WHERE date_add(date1, 1) = date'2022-05-20' " +
+ "AND datediff(date1, '2022-05-10') > 0")
+ checkFilterPushed(df6, false)
+ val rows6 = df6.collect()
+ assert(rows6.length === 1)
+ assert(rows6(0).getString(0) === "amy")
+
+ val df7 = sql(s"SELECT name FROM $tbl WHERE weekday(date1) = 2")
+ checkFilterPushed(df7, false)
+ val rows7 = df7.collect()
+ assert(rows7.length === 1)
+ assert(rows7(0).getString(0) === "alex")
+
+ withClue("weekofyear") {
+ val woy = sql(s"SELECT weekofyear(date1) FROM $tbl WHERE name = 'tom'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE weekofyear(date1) = $woy")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "tom")
+ }
+
+ withClue("dayofweek") {
+ val dow = sql(s"SELECT dayofweek(date1) FROM $tbl WHERE name = 'alex'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE dayofweek(date1) = $dow")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "alex")
+ }
+
+ withClue("yearofweek") {
+ val yow = sql(s"SELECT extract(YEAROFWEEK from date1) FROM $tbl WHERE name = 'tom'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE extract(YEAROFWEEK from date1) = $yow")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "tom")
+ }
+
+ withClue("dayofyear") {
+ val doy = sql(s"SELECT dayofyear(date1) FROM $tbl WHERE name = 'amy'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE dayofyear(date1) = $doy")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "amy")
+ }
+
+ withClue("dayofmonth") {
+ val dom = sql(s"SELECT dayofmonth(date1) FROM $tbl WHERE name = 'amy'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE dayofmonth(date1) = $dom")
+ checkFilterPushed(df)
+ val rows = df.collect()
+ assert(rows.length === 1)
+ assert(rows(0).getString(0) === "amy")
+ }
+
+ withClue("year") {
+ val year = sql(s"SELECT year(date1) FROM $tbl WHERE name = 'amy'")
+ .collect().head.getInt(0)
+ val df = sql(s"SELECT name FROM $tbl WHERE year(date1) = $year")
+ checkFilterPushed(df)
+ val rows = df.collect()
+ assert(rows.length === 3)
+ assert(rows(0).getString(0) === "amy")
+ assert(rows5(1).getString(0) === "alex")
+ assert(rows5(2).getString(0) === "tom")
+ }
+
+ withClue("second") {
+ val df = sql(s"SELECT name FROM $tbl WHERE second(time1) = 0 AND month(date1) = 5")
+ checkFilterPushed(df, false)
+ val rows = df.collect()
+ assert(rows.length === 2)
+ assert(rows(0).getString(0) === "amy")
+ assert(rows(1).getString(0) === "alex")
+ }
+
+ val df9 = sql(s"SELECT name FROM $tbl WHERE " +
+ "dayofyear(date1) > 100 order by dayofyear(date1) limit 1")
+ checkFilterPushed(df9, false)
+ val rows9 = df9.collect()
+ assert(rows9.length === 1)
+ assert(rows9(0).getString(0) === "alex")
+
+ val df10 = sql(s"SELECT name FROM $tbl WHERE trunc(date1, 'week') = date'2022-05-16'")
+ checkFilterPushed(df10)
+ val rows10 = df10.collect()
+ assert(rows10.length === 2)
+ assert(rows10(0).getString(0) === "amy")
+ assert(rows10(1).getString(0) === "alex")
+ }
}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala
index ee4a479960145..fb18dca97994e 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala
@@ -22,20 +22,21 @@ import org.apache.logging.log4j.Level
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{AnalysisException, DataFrame}
import org.apache.spark.sql.catalyst.analysis.{IndexAlreadyExistsException, NoSuchIndexException}
-import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, GlobalLimit, LocalLimit, Offset, Sample, Sort}
+import org.apache.spark.sql.connector.DataSourcePushdownTestUtils
import org.apache.spark.sql.connector.catalog.{Catalogs, Identifier, TableCatalog}
import org.apache.spark.sql.connector.catalog.index.SupportsIndex
import org.apache.spark.sql.connector.expressions.NullOrdering
-import org.apache.spark.sql.connector.expressions.aggregate.GeneralAggregateFunc
import org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD
-import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanRelation, V1ScanWrapper}
import org.apache.spark.sql.jdbc.DockerIntegrationFunSuite
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types._
import org.apache.spark.tags.DockerTest
@DockerTest
-private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFunSuite {
+private[v2] trait V2JDBCTest
+ extends DataSourcePushdownTestUtils
+ with DockerIntegrationFunSuite
+ with SharedSparkSession {
import testImplicits._
val catalogName: String
@@ -468,56 +469,6 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu
def supportsTableSample: Boolean = false
- private def checkSamplePushed(df: DataFrame, pushed: Boolean = true): Unit = {
- val sample = df.queryExecution.optimizedPlan.collect {
- case s: Sample => s
- }
- if (pushed) {
- assert(sample.isEmpty)
- } else {
- assert(sample.nonEmpty)
- }
- }
-
- protected def checkFilterPushed(df: DataFrame, pushed: Boolean = true): Unit = {
- val filter = df.queryExecution.optimizedPlan.collect {
- case f: Filter => f
- }
- if (pushed) {
- assert(filter.isEmpty)
- } else {
- assert(filter.nonEmpty)
- }
- }
-
- protected def checkLimitRemoved(df: DataFrame, pushed: Boolean = true): Unit = {
- val limit = df.queryExecution.optimizedPlan.collect {
- case l: LocalLimit => l
- case g: GlobalLimit => g
- }
- if (pushed) {
- assert(limit.isEmpty)
- } else {
- assert(limit.nonEmpty)
- }
- }
-
- private def checkLimitPushed(df: DataFrame, limit: Option[Int]): Unit = {
- df.queryExecution.optimizedPlan.collect {
- case relation: DataSourceV2ScanRelation => relation.scan match {
- case v1: V1ScanWrapper =>
- assert(v1.pushedDownOperators.limit == limit)
- }
- }
- }
-
- private def checkColumnPruned(df: DataFrame, col: String): Unit = {
- val scan = df.queryExecution.optimizedPlan.collectFirst {
- case s: DataSourceV2ScanRelation => s
- }.get
- assert(scan.schema.names.sameElements(Seq(col)))
- }
-
test("SPARK-48172: Test CONTAINS") {
val df1 = spark.sql(
s"""
@@ -841,39 +792,6 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu
}
}
- private def checkSortRemoved(df: DataFrame, pushed: Boolean = true): Unit = {
- val sorts = df.queryExecution.optimizedPlan.collect {
- case s: Sort => s
- }
-
- if (pushed) {
- assert(sorts.isEmpty)
- } else {
- assert(sorts.nonEmpty)
- }
- }
-
- private def checkOffsetRemoved(df: DataFrame, pushed: Boolean = true): Unit = {
- val offsets = df.queryExecution.optimizedPlan.collect {
- case o: Offset => o
- }
-
- if (pushed) {
- assert(offsets.isEmpty)
- } else {
- assert(offsets.nonEmpty)
- }
- }
-
- private def checkOffsetPushed(df: DataFrame, offset: Option[Int]): Unit = {
- df.queryExecution.optimizedPlan.collect {
- case relation: DataSourceV2ScanRelation => relation.scan match {
- case v1: V1ScanWrapper =>
- assert(v1.pushedDownOperators.offset == offset)
- }
- }
- }
-
gridTest("simple scan")(partitioningEnabledTestCase) { partitioningEnabled =>
val (tableOptions, partitionInfo) = getTableOptions("employee", partitioningEnabled)
val df = sql(s"SELECT name, salary, bonus FROM $catalogAndNamespace." +
@@ -1028,27 +946,6 @@ private[v2] trait V2JDBCTest extends SharedSparkSession with DockerIntegrationFu
}
}
- private def checkAggregateRemoved(df: DataFrame): Unit = {
- val aggregates = df.queryExecution.optimizedPlan.collect {
- case agg: Aggregate => agg
- }
- assert(aggregates.isEmpty)
- }
-
- private def checkAggregatePushed(df: DataFrame, funcName: String): Unit = {
- df.queryExecution.optimizedPlan.collect {
- case DataSourceV2ScanRelation(_, scan, _, _, _) =>
- assert(scan.isInstanceOf[V1ScanWrapper])
- val wrapper = scan.asInstanceOf[V1ScanWrapper]
- assert(wrapper.pushedDownOperators.aggregation.isDefined)
- val aggregationExpressions =
- wrapper.pushedDownOperators.aggregation.get.aggregateExpressions()
- assert(aggregationExpressions.length == 1)
- assert(aggregationExpressions(0).isInstanceOf[GeneralAggregateFunc])
- assert(aggregationExpressions(0).asInstanceOf[GeneralAggregateFunc].name() == funcName)
- }
- }
-
protected def caseConvert(tableName: String): String = tableName
Seq(true, false).foreach { isDistinct =>
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MsSqlServerJoinPushdownIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MsSqlServerJoinPushdownIntegrationSuite.scala
new file mode 100644
index 0000000000000..d8f6fa4b1b44e
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MsSqlServerJoinPushdownIntegrationSuite.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2.join
+
+import java.sql.Connection
+import java.util.Locale
+
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, JdbcDialect, MsSQLServerDatabaseOnDocker, MsSqlServerDialect}
+import org.apache.spark.sql.jdbc.v2.JDBCV2JoinPushdownIntegrationSuiteBase
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., 2022-CU15-ubuntu-22.04):
+ * {{{
+ * ENABLE_DOCKER_INTEGRATION_TESTS=1
+ * MSSQLSERVER_DOCKER_IMAGE_NAME=mcr.microsoft.com/mssql/server:2022-CU15-ubuntu-22.04
+ * ./build/sbt -Pdocker-integration-tests "testOnly *v2*MsSqlServerIntegrationSuite"
+ * }}}
+ */
+@DockerTest
+class MsSqlServerJoinPushdownIntegrationSuite
+ extends DockerJDBCIntegrationSuite
+ with JDBCV2JoinPushdownIntegrationSuiteBase {
+ override val db = new MsSQLServerDatabaseOnDocker
+
+ override val url = db.getJdbcUrl(dockerIp, externalPort)
+
+ override val jdbcDialect: JdbcDialect = MsSqlServerDialect()
+
+ override def caseConvert(identifier: String): String = identifier.toUpperCase(Locale.ROOT)
+
+ // This method comes from DockerJDBCIntegrationSuite
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation()
+ }
+}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MySQLJoinPushdownIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MySQLJoinPushdownIntegrationSuite.scala
new file mode 100644
index 0000000000000..1a1536b6a2929
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/MySQLJoinPushdownIntegrationSuite.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2.join
+
+import java.sql.Connection
+import java.util.Locale
+
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, JdbcDialect, MySQLDatabaseOnDocker, MySQLDialect}
+import org.apache.spark.sql.jdbc.v2.JDBCV2JoinPushdownIntegrationSuiteBase
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., mysql:9.2.0):
+ * {{{
+ * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:9.2.0
+ * ./build/sbt -Pdocker-integration-tests "testOnly *v2*MySQLIntegrationSuite"
+ * }}}
+ */
+@DockerTest
+class MySQLJoinPushdownIntegrationSuite
+ extends DockerJDBCIntegrationSuite
+ with JDBCV2JoinPushdownIntegrationSuiteBase {
+ override val db = new MySQLDatabaseOnDocker
+
+ override val url = db.getJdbcUrl(dockerIp, externalPort)
+
+ override val jdbcDialect: JdbcDialect = MySQLDialect()
+
+ override def caseConvert(identifier: String): String = identifier.toUpperCase(Locale.ROOT)
+
+ override def remainColumnCase(identifier: String): String = "`" + identifier + "`"
+
+ // This method comes from DockerJDBCIntegrationSuite
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation()
+ }
+}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/OracleJoinPushdownIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/OracleJoinPushdownIntegrationSuite.scala
new file mode 100644
index 0000000000000..ad9e5d002bf50
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/OracleJoinPushdownIntegrationSuite.scala
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2.join
+
+import java.sql.Connection
+import java.util.Locale
+
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, JdbcDialect, OracleDatabaseOnDocker, OracleDialect}
+import org.apache.spark.sql.jdbc.v2.JDBCV2JoinPushdownIntegrationSuiteBase
+import org.apache.spark.sql.types.DataTypes
+import org.apache.spark.tags.DockerTest
+
+/**
+ * The following are the steps to test this:
+ *
+ * 1. Choose to use a prebuilt image or build Oracle database in a container
+ * - The documentation on how to build Oracle RDBMS in a container is at
+ * https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md
+ * - Official Oracle container images can be found at https://container-registry.oracle.com
+ * - Trustable and streamlined Oracle Database Free images can be found on Docker Hub at
+ * https://hub.docker.com/r/gvenzl/oracle-free
+ * see also https://github.com/gvenzl/oci-oracle-free
+ * 2. Run: export ORACLE_DOCKER_IMAGE_NAME=image_you_want_to_use_for_testing
+ * - Example: export ORACLE_DOCKER_IMAGE_NAME=gvenzl/oracle-free:latest
+ * 3. Run: export ENABLE_DOCKER_INTEGRATION_TESTS=1
+ * 4. Start docker: sudo service docker start
+ * - Optionally, docker pull $ORACLE_DOCKER_IMAGE_NAME
+ * 5. Run Spark integration tests for Oracle with: ./build/sbt -Pdocker-integration-tests
+ * "testOnly org.apache.spark.sql.jdbc.v2.OracleIntegrationSuite"
+ *
+ * A sequence of commands to build the Oracle Database Free container image:
+ * $ git clone https://github.com/oracle/docker-images.git
+ * $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles0
+ * $ ./buildContainerImage.sh -v 23.4.0 -f
+ * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:23.4.0-free
+ *
+ * This procedure has been validated with Oracle Database Free version 23.4.0,
+ * and with Oracle Express Edition versions 18.4.0 and 21.4.0
+ */
+@DockerTest
+class OracleJoinPushdownIntegrationSuite
+ extends DockerJDBCIntegrationSuite
+ with JDBCV2JoinPushdownIntegrationSuiteBase {
+ override def excluded: Seq[String] = Seq(
+ // Following tests are harder to be supported for Oracle because Oracle connector does
+ // casts in predicates. There is a separate test in this suite that is similar to
+ // "Test explain formatted" test from base suite.
+ "Test self join with condition",
+ "Test multi-way self join with conditions",
+ "Test explain formatted"
+ )
+
+ override val namespace: String = "SYSTEM"
+
+ override val db = new OracleDatabaseOnDocker
+
+ override val url = db.getJdbcUrl(dockerIp, externalPort)
+
+ override val jdbcDialect: JdbcDialect = OracleDialect()
+
+ override val integerType = DataTypes.createDecimalType(10, 0)
+
+ override def caseConvert(identifier: String): String = identifier.toUpperCase(Locale.ROOT)
+
+ override def schemaPreparation(): Unit = {}
+
+ // This method comes from DockerJDBCIntegrationSuite
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation()
+ }
+
+ test("Test explain formatted - Oracle compatible") {
+ val sqlQuery =
+ s"""
+ |SELECT * FROM $catalogAndNamespace.$casedJoinTableName1 a
+ |JOIN $catalogAndNamespace.$casedJoinTableName2 b
+ |ON a.id = b.id + 1
+ |JOIN $catalogAndNamespace.$casedJoinTableName3 c
+ |ON b.id = c.id + 1
+ |JOIN $catalogAndNamespace.$casedJoinTableName4 d
+ |ON c.id = d.id + 1
+ |""".stripMargin
+
+ withSQLConf(SQLConf.DATA_SOURCE_V2_JOIN_PUSHDOWN.key -> "true") {
+ val df = sql(sqlQuery)
+
+ // scalastyle:off line.size.limit
+ checkJoinPushed(
+ df,
+ s"""PushedFilters: [CAST(id_3 AS decimal(11,0)) = (id_4 + 1)], PushedJoins:\u0020
+ |[L]: PushedFilters: [CAST(ID_1 AS decimal(11,0)) = (id_3 + 1)]
+ | PushedJoins:
+ | [L]: PushedFilters: [CAST(ID AS decimal(11,0)) = (ID_1 + 1)]
+ | PushedJoins:
+ | [L]: Relation: $catalogAndNamespace.${caseConvert(joinTableName1)}
+ | PushedFilters: [${caseConvert("id")} IS NOT NULL]
+ | [R]: Relation: $catalogAndNamespace.${caseConvert(joinTableName2)}
+ | PushedFilters: [${caseConvert("id")} IS NOT NULL]
+ | [R]: Relation: $catalogAndNamespace.${caseConvert(joinTableName3)}
+ | PushedFilters: [id IS NOT NULL]
+ |[R]: Relation: $catalogAndNamespace.${caseConvert(joinTableName4)}
+ | PushedFilters: [id IS NOT NULL]""".stripMargin
+ )
+ // scalastyle:on line.size.limit
+ }
+ }
+}
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/PostgresJoinPushdownIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/PostgresJoinPushdownIntegrationSuite.scala
new file mode 100644
index 0000000000000..e515eb698f60c
--- /dev/null
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/join/PostgresJoinPushdownIntegrationSuite.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc.v2.join
+
+import java.sql.Connection
+
+import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, JdbcDialect, PostgresDatabaseOnDocker, PostgresDialect}
+import org.apache.spark.sql.jdbc.v2.JDBCV2JoinPushdownIntegrationSuiteBase
+import org.apache.spark.tags.DockerTest
+
+/**
+ * To run this test suite for a specific version (e.g., postgres:17.2-alpine)
+ * {{{
+ * ENABLE_DOCKER_INTEGRATION_TESTS=1 POSTGRES_DOCKER_IMAGE_NAME=postgres:17.2-alpine
+ * ./build/sbt -Pdocker-integration-tests "testOnly *v2.PostgresIntegrationSuite"
+ * }}}
+ */
+@DockerTest
+class PostgresJoinPushdownIntegrationSuite
+ extends DockerJDBCIntegrationSuite
+ with JDBCV2JoinPushdownIntegrationSuiteBase {
+ override val db = new PostgresDatabaseOnDocker
+
+ override val url = db.getJdbcUrl(dockerIp, externalPort)
+
+ override val jdbcDialect: JdbcDialect = PostgresDialect()
+
+ // This method comes from DockerJDBCIntegrationSuite
+ override def dataPreparation(connection: Connection): Unit = {
+ super.dataPreparation()
+ }
+}
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala
index cb1c7055483b3..02568aa89eb1d 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaBatchPartitionReader.scala
@@ -20,13 +20,13 @@ package org.apache.spark.sql.kafka010
import java.{util => ju}
import org.apache.spark.TaskContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
import org.apache.spark.sql.connector.metric.CustomTaskMetric
import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory}
-import org.apache.spark.sql.execution.streaming.{MicroBatchExecution, StreamExecution}
+import org.apache.spark.sql.execution.streaming.runtime.{MicroBatchExecution, StreamExecution}
import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer
/** A [[InputPartition]] for reading Kafka data in a batch based streaming query. */
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousStream.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousStream.scala
index 10bdbb1d9d447..041fe074f7e40 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousStream.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousStream.scala
@@ -24,7 +24,7 @@ import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord, Offset
import org.apache.kafka.common.TopicPartition
import org.apache.spark.TaskContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ERROR, OFFSETS, TIP}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.UnsafeRow
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
index c79da13017b97..19a5d7cb739f5 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import org.apache.kafka.common.TopicPartition
import org.apache.spark.SparkEnv
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ERROR, OFFSETS, TIP}
import org.apache.spark.internal.config.Network.NETWORK_TIMEOUT
import org.apache.spark.sql.SparkSession
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala
index 86cec6fc041cd..7420c2c1055b2 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderAdmin.scala
@@ -30,7 +30,7 @@ import org.apache.kafka.common.{IsolationLevel, TopicPartition}
import org.apache.kafka.common.requests.OffsetFetchResponse
import org.apache.spark.SparkEnv
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NUM_RETRY, OFFSETS, TOPIC_PARTITION_OFFSET}
import org.apache.spark.scheduler.ExecutorCacheTaskLocation
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala
index fb06797d1fe34..b7ac9a171c575 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaOffsetReaderConsumer.scala
@@ -27,7 +27,7 @@ import org.apache.kafka.clients.consumer.{Consumer, ConsumerConfig, OffsetAndTim
import org.apache.kafka.common.TopicPartition
import org.apache.spark.SparkEnv
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NUM_RETRY, OFFSETS, TOPIC_PARTITION_OFFSET}
import org.apache.spark.scheduler.ExecutorCacheTaskLocation
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
index ff884310f660c..7e98446567640 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaRelation.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.kafka010
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.TOPIC_PARTITIONS
import org.apache.spark.internal.config.Network.NETWORK_TIMEOUT
import org.apache.spark.rdd.RDD
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSink.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSink.scala
index fb473e71d5a75..d3ea5332f4e44 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSink.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSink.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.kafka010
import java.{util => ju}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.BATCH_ID
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.execution.streaming.Sink
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
index 1b52046b14833..b2679e27827fe 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
@@ -22,7 +22,7 @@ import java.{util => ju}
import org.apache.kafka.common.TopicPartition
import org.apache.spark.SparkContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ERROR, FROM_OFFSET, OFFSETS, TIP, TOPIC_PARTITIONS, UNTIL_OFFSET}
import org.apache.spark.internal.config.Network.NETWORK_TIMEOUT
import org.apache.spark.scheduler.ExecutorCacheTaskLocation
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceInitialOffsetWriter.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceInitialOffsetWriter.scala
index f9b7e23df49b6..cef712fafd888 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceInitialOffsetWriter.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceInitialOffsetWriter.scala
@@ -20,10 +20,10 @@ package org.apache.spark.sql.kafka010
import java.io._
import java.nio.charset.StandardCharsets
-import org.apache.commons.io.IOUtils
-
import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.execution.streaming.{HDFSMetadataLog, SerializedOffset}
+import org.apache.spark.sql.execution.streaming.checkpointing.HDFSMetadataLog
+import org.apache.spark.sql.execution.streaming.runtime.SerializedOffset
+import org.apache.spark.util.Utils
/** A version of [[HDFSMetadataLog]] specialized for saving the initial offsets. */
private[kafka010] class KafkaSourceInitialOffsetWriter(
@@ -43,7 +43,7 @@ private[kafka010] class KafkaSourceInitialOffsetWriter(
override def deserialize(in: InputStream): KafkaSourceOffset = {
in.read() // A zero byte is read to support Spark 2.1.0 (SPARK-19517)
- val content = IOUtils.toString(new InputStreamReader(in, StandardCharsets.UTF_8))
+ val content = Utils.toString(in)
// HDFSMetadataLog guarantees that it never creates a partial file.
require(content.nonEmpty)
if (content(0) == 'v') {
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala
index 8790a77e5d895..d2c0c5d6fba85 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceOffset.scala
@@ -21,7 +21,8 @@ import org.apache.kafka.common.TopicPartition
import org.apache.spark.sql.connector.read.streaming
import org.apache.spark.sql.connector.read.streaming.PartitionOffset
-import org.apache.spark.sql.execution.streaming.{Offset, SerializedOffset}
+import org.apache.spark.sql.execution.streaming.Offset
+import org.apache.spark.sql.execution.streaming.runtime.SerializedOffset
/**
* An [[Offset]] for the [[KafkaSource]]. This one tracks all partitions of subscribed topics and
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
index 82ad75e028afe..ff2f16d26b932 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
@@ -27,7 +27,7 @@ import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer}
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.kafka010.KafkaConfigUpdater
import org.apache.spark.sql.{AnalysisException, DataFrame, SaveMode, SQLContext}
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
index 0f962a29588a0..9bae686a65bf8 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceRDD.scala
@@ -23,7 +23,6 @@ import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.spark.{Partition, SparkContext, SparkException, TaskContext}
import org.apache.spark.internal.LogKeys.{FROM_OFFSET, PARTITION_ID, TOPIC}
-import org.apache.spark.internal.MDC
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer
import org.apache.spark.storage.StorageLevel
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala
index 9f68cb6fd0882..2ea1104177a54 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/FetchedDataPool.scala
@@ -26,7 +26,7 @@ import scala.collection.mutable
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{DATA, KEY}
import org.apache.spark.sql.kafka010.{FETCHED_DATA_CACHE_EVICTOR_THREAD_RUN_INTERVAL, FETCHED_DATA_CACHE_TIMEOUT}
import org.apache.spark.sql.kafka010.consumer.KafkaDataConsumer.{AvailableOffsetRange, CacheKey, UNKNOWN_OFFSET}
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
index ceb9d96660ae3..2d1125294df27 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
@@ -30,7 +30,7 @@ import org.apache.kafka.common.TopicPartition
import org.apache.spark.{SparkEnv, TaskContext}
import org.apache.spark.deploy.security.HadoopDelegationTokenManager
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.kafka010.{KafkaConfigUpdater, KafkaTokenUtil}
import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_MILLIS
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/CachedKafkaProducer.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/CachedKafkaProducer.scala
index c3457cf8982d9..9b40bcf8ce779 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/CachedKafkaProducer.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/CachedKafkaProducer.scala
@@ -23,7 +23,7 @@ import scala.util.control.NonFatal
import org.apache.kafka.clients.producer.KafkaProducer
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PRODUCER_ID
private[kafka010] class CachedKafkaProducer(
diff --git a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/InternalKafkaProducerPool.scala b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/InternalKafkaProducerPool.scala
index 79e0a91dd8968..ca9a0059ba1e4 100644
--- a/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/InternalKafkaProducerPool.scala
+++ b/connector/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/producer/InternalKafkaProducerPool.scala
@@ -27,7 +27,7 @@ import scala.jdk.CollectionConverters._
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.spark.{SparkConf, SparkEnv}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PRODUCER_ID
import org.apache.spark.kafka010.{KafkaConfigUpdater, KafkaRedactionUtil}
import org.apache.spark.sql.kafka010.{PRODUCER_CACHE_EVICTOR_THREAD_RUN_INTERVAL, PRODUCER_CACHE_TIMEOUT}
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
index 4b6a5b899fc8e..2dff58ff87875 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousTest.scala
@@ -22,8 +22,8 @@ import java.util.concurrent.atomic.AtomicInteger
import org.apache.spark.SparkContext
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd, SparkListenerTaskStart}
import org.apache.spark.sql.execution.datasources.v2.ContinuousScanExec
-import org.apache.spark.sql.execution.streaming.StreamExecution
import org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution
+import org.apache.spark.sql.execution.streaming.runtime.StreamExecution
import org.apache.spark.sql.streaming.Trigger
import org.apache.spark.sql.test.TestSparkSession
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
index 702bd4f6ebdad..a55b6e0068519 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
@@ -26,7 +26,7 @@ import org.apache.kafka.common.security.auth.SecurityProtocol.SASL_PLAINTEXT
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.security.HadoopDelegationTokenManager
import org.apache.spark.internal.config.{KEYTAB, PRINCIPAL}
-import org.apache.spark.sql.execution.streaming.MemoryStream
+import org.apache.spark.sql.execution.streaming.runtime.MemoryStream
import org.apache.spark.sql.streaming.{OutputMode, StreamTest}
import org.apache.spark.sql.test.SharedSparkSession
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
index e738abf21f597..2ef6c992ad238 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
@@ -29,7 +29,6 @@ import scala.io.Source
import scala.jdk.CollectionConverters._
import scala.util.Random
-import org.apache.commons.io.FileUtils
import org.apache.kafka.clients.producer.{ProducerRecord, RecordMetadata}
import org.apache.kafka.common.TopicPartition
import org.scalatest.concurrent.PatienceConfiguration.Timeout
@@ -43,8 +42,10 @@ import org.apache.spark.sql.connector.read.streaming.SparkDataStream
import org.apache.spark.sql.execution.datasources.v2.StreamingDataSourceV2ScanRelation
import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
import org.apache.spark.sql.execution.streaming._
-import org.apache.spark.sql.execution.streaming.AsyncProgressTrackingMicroBatchExecution.{ASYNC_PROGRESS_TRACKING_CHECKPOINTING_INTERVAL_MS, ASYNC_PROGRESS_TRACKING_ENABLED}
+import org.apache.spark.sql.execution.streaming.checkpointing.OffsetSeq
import org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution
+import org.apache.spark.sql.execution.streaming.runtime.{MicroBatchExecution, StreamExecution, StreamingExecutionRelation}
+import org.apache.spark.sql.execution.streaming.runtime.AsyncProgressTrackingMicroBatchExecution.{ASYNC_PROGRESS_TRACKING_CHECKPOINTING_INTERVAL_MS, ASYNC_PROGRESS_TRACKING_ENABLED}
import org.apache.spark.sql.functions.{count, expr, window}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.kafka010.KafkaSourceProvider._
@@ -1729,7 +1730,7 @@ abstract class KafkaMicroBatchV2SourceSuite extends KafkaMicroBatchSourceSuiteBa
val checkpointDir = Utils.createTempDir().getCanonicalFile
// Copy the checkpoint to a temp dir to prevent changes to the original.
// Not doing this will lead to the test passing on the first run, but fail subsequent runs.
- FileUtils.copyDirectory(new File(resourceUri), checkpointDir)
+ Utils.copyDirectory(new File(resourceUri), checkpointDir)
testStream(query)(
StartStream(checkpointLocation = checkpointDir.getAbsolutePath),
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
index 82edba59995ec..a444128cb2c0b 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
@@ -32,7 +32,7 @@ import org.scalatest.time.SpanSugar._
import org.apache.spark.{SparkConf, SparkContext, TestUtils}
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, SpecificInternalRow, UnsafeProjection}
-import org.apache.spark.sql.execution.streaming.{MemoryStream, MemoryStreamBase}
+import org.apache.spark.sql.execution.streaming.runtime.{MemoryStream, MemoryStreamBase}
import org.apache.spark.sql.execution.streaming.sources.ContinuousMemoryStream
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala
index 553ab42b9c8b1..18330feac861c 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSourceOffsetSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.sql.kafka010
import java.io.File
-import org.apache.spark.sql.execution.streaming._
+import org.apache.spark.sql.execution.streaming.checkpointing.{OffsetSeq, OffsetSeqLog}
+import org.apache.spark.sql.execution.streaming.runtime._
import org.apache.spark.sql.streaming.OffsetSuite
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.util.Utils
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 60de3705636ec..0e1ca7af14a43 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.kafka010
import java.io.{File, IOException}
import java.net.InetSocketAddress
-import java.nio.charset.StandardCharsets
+import java.nio.file.Files
import java.util.{Collections, Properties, UUID}
import java.util.concurrent.TimeUnit
import javax.security.auth.login.Configuration
@@ -27,7 +27,6 @@ import javax.security.auth.login.Configuration
import scala.io.Source
import scala.jdk.CollectionConverters._
-import com.google.common.io.Files
import kafka.log.LogManager
import kafka.server.{HostedPartition, KafkaConfig, KafkaServer}
import kafka.server.checkpoints.OffsetCheckpointFile
@@ -53,7 +52,7 @@ import org.scalatest.concurrent.Eventually._
import org.scalatest.time.SpanSugar._
import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.kafka010.KafkaTokenUtil
import org.apache.spark.util.{SecurityUtils, ShutdownHookManager, Utils}
@@ -176,7 +175,7 @@ class KafkaTestUtils(
}
kdc.getKrb5conf.delete()
- Files.asCharSink(kdc.getKrb5conf, StandardCharsets.UTF_8).write(krb5confStr)
+ Files.writeString(kdc.getKrb5conf.toPath, krb5confStr)
logDebug(s"krb5.conf file content: $krb5confStr")
}
@@ -240,7 +239,7 @@ class KafkaTestUtils(
| principal="$kafkaServerUser@$realm";
|};
""".stripMargin.trim
- Files.asCharSink(file, StandardCharsets.UTF_8).write(content)
+ Files.writeString(file.toPath, content)
logDebug(s"Created JAAS file: ${file.getPath}")
logDebug(s"JAAS file content: $content")
file.getAbsolutePath()
diff --git a/connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaDelegationTokenProvider.scala b/connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaDelegationTokenProvider.scala
index 8acdd01a15bab..bce318811dc27 100644
--- a/connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaDelegationTokenProvider.scala
+++ b/connector/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaDelegationTokenProvider.scala
@@ -24,7 +24,7 @@ import org.apache.hadoop.security.Credentials
import org.apache.kafka.common.security.auth.SecurityProtocol.{SASL_PLAINTEXT, SASL_SSL, SSL}
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLUSTER_ID, SERVICE_NAME}
import org.apache.spark.security.HadoopDelegationTokenProvider
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala
index 2320f1908da5a..24c3dd0cddf17 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/ConsumerStrategy.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import org.apache.kafka.clients.consumer._
import org.apache.kafka.common.TopicPartition
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.CONFIG
import org.apache.spark.kafka010.KafkaConfigUpdater
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala
index f7bea064d2d6c..beaac35720445 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/DirectKafkaInputDStream.scala
@@ -27,7 +27,7 @@ import scala.jdk.CollectionConverters._
import org.apache.kafka.clients.consumer._
import org.apache.kafka.common.TopicPartition
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{OFFSET, TIME, TOPIC_PARTITION, TOPIC_PARTITION_OFFSET_RANGE}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{StreamingContext, Time}
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala
index 75b046430ef50..03701dbda0947 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaDataConsumer.scala
@@ -26,7 +26,7 @@ import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord, KafkaC
import org.apache.kafka.common.{KafkaException, TopicPartition}
import org.apache.spark.TaskContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.kafka010.KafkaConfigUpdater
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala
index 2637034766574..af97d5464e004 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaRDD.scala
@@ -23,7 +23,7 @@ import org.apache.kafka.clients.consumer.{ ConsumerConfig, ConsumerRecord }
import org.apache.kafka.common.TopicPartition
import org.apache.spark.{Partition, SparkContext, TaskContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{FROM_OFFSET, PARTITION_ID, TOPIC, UNTIL_OFFSET}
import org.apache.spark.internal.config.Network._
import org.apache.spark.partial.{BoundedDouble, PartialResult}
diff --git a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaUtils.scala b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaUtils.scala
index d15e5e25f561d..54cb9ff39d993 100644
--- a/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaUtils.scala
+++ b/connector/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/KafkaUtils.scala
@@ -24,7 +24,7 @@ import org.apache.kafka.common.TopicPartition
import org.apache.spark.SparkContext
import org.apache.spark.api.java.{ JavaRDD, JavaSparkContext }
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CONFIG, GROUP_ID}
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.StreamingContext
diff --git a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
index 212693f6e02cc..a1264af021aa1 100644
--- a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
+++ b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
@@ -38,6 +38,7 @@ import org.scalatest.concurrent.Eventually.{eventually, interval, timeout}
import org.apache.spark._
import org.apache.spark.scheduler.ExecutorCacheTaskLocation
import org.apache.spark.streaming.kafka010.mocks.MockTime
+import org.apache.spark.util.Utils
class KafkaRDDSuite extends SparkFunSuite {
@@ -91,7 +92,7 @@ class KafkaRDDSuite extends SparkFunSuite {
val logs = new Pool[TopicPartition, UnifiedLog]()
val logDir = kafkaTestUtils.brokerLogDir
val dir = new File(logDir, topic + "-" + partition)
- dir.mkdirs()
+ Utils.createDirectory(dir)
val logProps = new ju.Properties()
logProps.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT)
logProps.put(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG, java.lang.Float.valueOf(0.1f))
diff --git a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
index b391203b4b968..ac3622f93321a 100644
--- a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
+++ b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
@@ -29,7 +29,7 @@ import com.amazonaws.services.kinesis.clientlibrary.types.UserRecord
import com.amazonaws.services.kinesis.model._
import org.apache.spark._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ERROR, NUM_RETRY}
import org.apache.spark.rdd.{BlockRDD, BlockRDDPartition}
import org.apache.spark.storage.BlockId
diff --git a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
index c52eeca1e48a1..b259a5337f37e 100644
--- a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
+++ b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
@@ -22,7 +22,7 @@ import scala.util.control.NonFatal
import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{SHARD_ID, WORKER_URL}
import org.apache.spark.streaming.Duration
import org.apache.spark.streaming.util.RecurringTimer
diff --git a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
index 953817e625e48..ab91431035fef 100644
--- a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
+++ b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
@@ -28,7 +28,7 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.{KinesisClientLib
import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel
import com.amazonaws.services.kinesis.model.Record
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.WORKER_URL
import org.apache.spark.storage.{StorageLevel, StreamBlockId}
import org.apache.spark.streaming.Duration
diff --git a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala
index cd740f971e484..8304ddda96dfa 100644
--- a/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala
+++ b/connector/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala
@@ -26,7 +26,7 @@ import com.amazonaws.services.kinesis.clientlibrary.interfaces.{IRecordProcessor
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason
import com.amazonaws.services.kinesis.model.Record
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{KINESIS_REASON, RETRY_INTERVAL, SHARD_ID, WORKER_URL}
/**
diff --git a/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala b/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
index 652822c5fdc97..3f06d476f08dc 100644
--- a/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
+++ b/connector/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
@@ -33,7 +33,7 @@ import com.amazonaws.services.dynamodbv2.document.DynamoDB
import com.amazonaws.services.kinesis.{AmazonKinesis, AmazonKinesisClient}
import com.amazonaws.services.kinesis.model._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{STREAM_NAME, TABLE_NAME}
/**
diff --git a/connector/profiler/README.md b/connector/profiler/README.md
index 9e0a58444c605..742b1a8bb5da3 100644
--- a/connector/profiler/README.md
+++ b/connector/profiler/README.md
@@ -16,7 +16,7 @@ or
## Executor Code Profiling
-The spark profiler module enables code profiling of executors in cluster mode based on the [async profiler](https://github.com/async-profiler/async-profiler/blob/v3.0/README.md), a low overhead sampling profiler. This allows a Spark application to capture CPU and memory profiles for application running on a cluster which can later be analyzed for performance issues. The profiler captures [Java Flight Recorder (jfr)](https://access.redhat.com/documentation/es-es/red_hat_build_of_openjdk/17/html/using_jdk_flight_recorder_with_red_hat_build_of_openjdk/openjdk-flight-recorded-overview) files for each executor; these can be read by many tools including Java Mission Control and Intellij.
+The spark profiler module enables code profiling of executors in cluster mode based on the [async profiler](https://github.com/async-profiler/async-profiler/blob/v4.0/README.md), a low overhead sampling profiler. This allows a Spark application to capture CPU and memory profiles for application running on a cluster which can later be analyzed for performance issues. The profiler captures [Java Flight Recorder (jfr)](https://access.redhat.com/documentation/es-es/red_hat_build_of_openjdk/17/html/using_jdk_flight_recorder_with_red_hat_build_of_openjdk/openjdk-flight-recorded-overview) files for each executor; these can be read by many tools including Java Mission Control and Intellij.
The profiler writes the jfr files to the executor's working directory in the executor's local file system and the files can grow to be large, so it is advisable that the executor machines have adequate storage. The profiler can be configured to copy the jfr files to a hdfs location before the executor shuts down.
@@ -95,7 +95,7 @@ Then enable the profiling in the configuration.
| event=wall,interval=10ms,alloc=2m,lock=10ms,chunktime=300s |
Arguments to pass to the Async Profiler. Detailed options are documented in the comments here:
- Profiler arguments.
+ Profiler options.
Note that the arguments to start, stop, specify output format, and output file do not have to be specified.
|
4.0.0 |
diff --git a/connector/profiler/src/main/scala/org/apache/spark/profiler/ProfilerPlugin.scala b/connector/profiler/src/main/scala/org/apache/spark/profiler/ProfilerPlugin.scala
index db3de90099f7c..cb341aef3a0d8 100644
--- a/connector/profiler/src/main/scala/org/apache/spark/profiler/ProfilerPlugin.scala
+++ b/connector/profiler/src/main/scala/org/apache/spark/profiler/ProfilerPlugin.scala
@@ -23,7 +23,7 @@ import scala.util.Random
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, PluginContext, SparkPlugin}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.EXECUTOR_ID
/**
diff --git a/connector/profiler/src/main/scala/org/apache/spark/profiler/SparkAsyncProfiler.scala b/connector/profiler/src/main/scala/org/apache/spark/profiler/SparkAsyncProfiler.scala
index 125ce63620b63..02d39a2c435b3 100644
--- a/connector/profiler/src/main/scala/org/apache/spark/profiler/SparkAsyncProfiler.scala
+++ b/connector/profiler/src/main/scala/org/apache/spark/profiler/SparkAsyncProfiler.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.fs.permission.FsPermission
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext.DRIVER_IDENTIFIER
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PATH
import org.apache.spark.util.{ThreadUtils, Utils}
@@ -36,7 +36,6 @@ import org.apache.spark.util.{ThreadUtils, Utils}
private[spark] class SparkAsyncProfiler(conf: SparkConf, executorId: String) extends Logging {
private var running = false
- private val enableProfiler = conf.get(PROFILER_EXECUTOR_ENABLED)
private val profilerOptions = conf.get(PROFILER_ASYNC_PROFILER_OPTIONS)
private val profilerDfsDirOpt = conf.get(PROFILER_DFS_DIR)
private val profilerLocalDir = conf.get(PROFILER_LOCAL_DIR)
@@ -66,9 +65,15 @@ private[spark] class SparkAsyncProfiler(conf: SparkConf, executorId: String) ext
private var threadpool: ScheduledExecutorService = _
@volatile private var writing: Boolean = false
+ lazy private val extractionDir =
+ Utils.createTempDir(Utils.getLocalDir(conf), "asyncProfiler").toPath
+
val profiler: Option[AsyncProfiler] = {
Option(
- if (enableProfiler && AsyncProfilerLoader.isSupported) AsyncProfilerLoader.load() else null
+ if (AsyncProfilerLoader.isSupported) {
+ AsyncProfilerLoader.setExtractionDirectory(extractionDir)
+ AsyncProfilerLoader.load()
+ } else null
)
}
diff --git a/connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java b/connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java
index 48c61e80d6655..3a38a52f7ab5d 100644
--- a/connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java
+++ b/connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java
@@ -295,7 +295,7 @@ private void reportTimer(String name, Timer timer) {
reportMetered(sanitizedName, timer, group, "calls");
} catch (GangliaException e) {
LOGGER.warn("Unable to report timer {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, sanitizedName));
+ MDC.of(LogKeys.METRIC_NAME, sanitizedName));
}
}
@@ -306,7 +306,7 @@ private void reportMeter(String name, Meter meter) {
reportMetered(sanitizedName, meter, group, "events");
} catch (GangliaException e) {
LOGGER.warn("Unable to report meter {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, name));
+ MDC.of(LogKeys.METRIC_NAME, name));
}
}
@@ -338,7 +338,7 @@ private void reportHistogram(String name, Histogram histogram) {
announceIfEnabled(P999, sanitizedName, group, snapshot.get999thPercentile(), "");
} catch (GangliaException e) {
LOGGER.warn("Unable to report histogram {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, sanitizedName));
+ MDC.of(LogKeys.METRIC_NAME, sanitizedName));
}
}
@@ -349,7 +349,7 @@ private void reportCounter(String name, Counter counter) {
announce(prefix(sanitizedName, COUNT.getCode()), group, Long.toString(counter.getCount()), GMetricType.DOUBLE, "");
} catch (GangliaException e) {
LOGGER.warn("Unable to report counter {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, name));
+ MDC.of(LogKeys.METRIC_NAME, name));
}
}
@@ -363,7 +363,7 @@ private void reportGauge(String name, Gauge gauge) {
announce(name(prefix, sanitizedName), group, value, type, "");
} catch (GangliaException e) {
LOGGER.warn("Unable to report gauge {}", e,
- MDC.of(LogKeys.METRIC_NAME$.MODULE$, name));
+ MDC.of(LogKeys.METRIC_NAME, name));
}
}
diff --git a/core/benchmarks/ChecksumBenchmark-jdk21-results.txt b/core/benchmarks/ChecksumBenchmark-jdk21-results.txt
index 0065bd684beba..c622b13efc483 100644
--- a/core/benchmarks/ChecksumBenchmark-jdk21-results.txt
+++ b/core/benchmarks/ChecksumBenchmark-jdk21-results.txt
@@ -2,12 +2,12 @@
Benchmark Checksum Algorithms
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Checksum Algorithms: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Adler32 11110 11117 9 0.0 10849608.9 1.0X
-CRC32 2763 2768 4 0.0 2698564.7 4.0X
-CRC32C 2249 2303 48 0.0 2195990.4 4.9X
+Adler32 11108 11126 25 0.0 10847856.7 1.0X
+CRC32 2757 2770 19 0.0 2692382.2 4.0X
+CRC32C 1986 2008 19 0.0 1939837.0 5.6X
diff --git a/core/benchmarks/ChecksumBenchmark-results.txt b/core/benchmarks/ChecksumBenchmark-results.txt
index e907066d649c3..e7515b83cdfff 100644
--- a/core/benchmarks/ChecksumBenchmark-results.txt
+++ b/core/benchmarks/ChecksumBenchmark-results.txt
@@ -2,12 +2,12 @@
Benchmark Checksum Algorithms
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Checksum Algorithms: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Adler32 11096 11101 4 0.0 10835715.4 1.0X
-CRC32 2715 2716 1 0.0 2651776.1 4.1X
-CRC32C 1629 1647 15 0.0 1591225.5 6.8X
+Adler32 11125 11135 15 0.0 10863804.5 1.0X
+CRC32 2778 2783 6 0.0 2713168.9 4.0X
+CRC32C 2268 2513 391 0.0 2214567.2 4.9X
diff --git a/core/benchmarks/CoalescedRDDBenchmark-jdk21-results.txt b/core/benchmarks/CoalescedRDDBenchmark-jdk21-results.txt
index 88142e681790d..e4e7342c1d0ca 100644
--- a/core/benchmarks/CoalescedRDDBenchmark-jdk21-results.txt
+++ b/core/benchmarks/CoalescedRDDBenchmark-jdk21-results.txt
@@ -2,39 +2,39 @@
Coalesced RDD , large scale
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Coalesced RDD: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------
-Coalesce Num Partitions: 100 Num Hosts: 1 255 263 12 0.4 2548.3 1.0X
-Coalesce Num Partitions: 100 Num Hosts: 5 133 135 2 0.8 1332.7 1.9X
-Coalesce Num Partitions: 100 Num Hosts: 10 99 103 4 1.0 994.6 2.6X
-Coalesce Num Partitions: 100 Num Hosts: 20 104 106 2 1.0 1035.4 2.5X
-Coalesce Num Partitions: 100 Num Hosts: 40 92 104 16 1.1 919.5 2.8X
-Coalesce Num Partitions: 100 Num Hosts: 80 99 99 1 1.0 986.7 2.6X
-Coalesce Num Partitions: 500 Num Hosts: 1 880 892 13 0.1 8796.0 0.3X
-Coalesce Num Partitions: 500 Num Hosts: 5 247 254 7 0.4 2474.6 1.0X
-Coalesce Num Partitions: 500 Num Hosts: 10 167 169 1 0.6 1672.5 1.5X
-Coalesce Num Partitions: 500 Num Hosts: 20 127 128 1 0.8 1269.1 2.0X
-Coalesce Num Partitions: 500 Num Hosts: 40 107 107 1 0.9 1065.5 2.4X
-Coalesce Num Partitions: 500 Num Hosts: 80 109 112 3 0.9 1090.6 2.3X
-Coalesce Num Partitions: 1000 Num Hosts: 1 1651 1677 23 0.1 16508.5 0.2X
-Coalesce Num Partitions: 1000 Num Hosts: 5 407 418 14 0.2 4073.4 0.6X
-Coalesce Num Partitions: 1000 Num Hosts: 10 249 250 2 0.4 2485.6 1.0X
-Coalesce Num Partitions: 1000 Num Hosts: 20 165 170 7 0.6 1654.8 1.5X
-Coalesce Num Partitions: 1000 Num Hosts: 40 129 130 3 0.8 1287.6 2.0X
-Coalesce Num Partitions: 1000 Num Hosts: 80 108 110 2 0.9 1082.9 2.4X
-Coalesce Num Partitions: 5000 Num Hosts: 1 7737 7757 19 0.0 77368.9 0.0X
-Coalesce Num Partitions: 5000 Num Hosts: 5 1784 1790 5 0.1 17837.1 0.1X
-Coalesce Num Partitions: 5000 Num Hosts: 10 923 938 20 0.1 9226.0 0.3X
-Coalesce Num Partitions: 5000 Num Hosts: 20 520 526 9 0.2 5196.1 0.5X
-Coalesce Num Partitions: 5000 Num Hosts: 40 314 315 1 0.3 3137.0 0.8X
-Coalesce Num Partitions: 5000 Num Hosts: 80 201 213 13 0.5 2012.4 1.3X
-Coalesce Num Partitions: 10000 Num Hosts: 1 14476 14509 28 0.0 144760.1 0.0X
-Coalesce Num Partitions: 10000 Num Hosts: 5 3401 3429 41 0.0 34012.2 0.1X
-Coalesce Num Partitions: 10000 Num Hosts: 10 1753 1759 9 0.1 17526.3 0.1X
-Coalesce Num Partitions: 10000 Num Hosts: 20 927 936 9 0.1 9272.2 0.3X
-Coalesce Num Partitions: 10000 Num Hosts: 40 510 518 12 0.2 5097.7 0.5X
-Coalesce Num Partitions: 10000 Num Hosts: 80 303 307 3 0.3 3034.1 0.8X
+Coalesce Num Partitions: 100 Num Hosts: 1 258 289 44 0.4 2576.1 1.0X
+Coalesce Num Partitions: 100 Num Hosts: 5 134 135 2 0.7 1337.3 1.9X
+Coalesce Num Partitions: 100 Num Hosts: 10 115 119 5 0.9 1153.1 2.2X
+Coalesce Num Partitions: 100 Num Hosts: 20 121 122 2 0.8 1205.8 2.1X
+Coalesce Num Partitions: 100 Num Hosts: 40 104 111 8 1.0 1042.2 2.5X
+Coalesce Num Partitions: 100 Num Hosts: 80 102 109 9 1.0 1023.5 2.5X
+Coalesce Num Partitions: 500 Num Hosts: 1 861 877 21 0.1 8610.7 0.3X
+Coalesce Num Partitions: 500 Num Hosts: 5 274 276 2 0.4 2743.6 0.9X
+Coalesce Num Partitions: 500 Num Hosts: 10 184 185 1 0.5 1838.5 1.4X
+Coalesce Num Partitions: 500 Num Hosts: 20 147 152 6 0.7 1472.2 1.7X
+Coalesce Num Partitions: 500 Num Hosts: 40 125 126 1 0.8 1249.5 2.1X
+Coalesce Num Partitions: 500 Num Hosts: 80 125 126 2 0.8 1250.0 2.1X
+Coalesce Num Partitions: 1000 Num Hosts: 1 1616 1624 10 0.1 16161.1 0.2X
+Coalesce Num Partitions: 1000 Num Hosts: 5 423 426 5 0.2 4229.1 0.6X
+Coalesce Num Partitions: 1000 Num Hosts: 10 264 266 2 0.4 2641.7 1.0X
+Coalesce Num Partitions: 1000 Num Hosts: 20 182 191 15 0.6 1816.1 1.4X
+Coalesce Num Partitions: 1000 Num Hosts: 40 135 140 5 0.7 1352.6 1.9X
+Coalesce Num Partitions: 1000 Num Hosts: 80 137 138 1 0.7 1373.0 1.9X
+Coalesce Num Partitions: 5000 Num Hosts: 1 7384 7391 6 0.0 73841.7 0.0X
+Coalesce Num Partitions: 5000 Num Hosts: 5 1899 1911 16 0.1 18986.8 0.1X
+Coalesce Num Partitions: 5000 Num Hosts: 10 1004 1010 6 0.1 10035.9 0.3X
+Coalesce Num Partitions: 5000 Num Hosts: 20 571 575 3 0.2 5714.0 0.5X
+Coalesce Num Partitions: 5000 Num Hosts: 40 349 351 3 0.3 3485.6 0.7X
+Coalesce Num Partitions: 5000 Num Hosts: 80 232 234 2 0.4 2324.0 1.1X
+Coalesce Num Partitions: 10000 Num Hosts: 1 13891 13911 35 0.0 138910.9 0.0X
+Coalesce Num Partitions: 10000 Num Hosts: 5 3828 3830 2 0.0 38275.2 0.1X
+Coalesce Num Partitions: 10000 Num Hosts: 10 1921 1935 13 0.1 19206.1 0.1X
+Coalesce Num Partitions: 10000 Num Hosts: 20 1018 1025 6 0.1 10184.3 0.3X
+Coalesce Num Partitions: 10000 Num Hosts: 40 571 577 9 0.2 5710.2 0.5X
+Coalesce Num Partitions: 10000 Num Hosts: 80 341 345 3 0.3 3413.2 0.8X
diff --git a/core/benchmarks/CoalescedRDDBenchmark-results.txt b/core/benchmarks/CoalescedRDDBenchmark-results.txt
index d3a2cc1a6e108..c4e9b60f683b4 100644
--- a/core/benchmarks/CoalescedRDDBenchmark-results.txt
+++ b/core/benchmarks/CoalescedRDDBenchmark-results.txt
@@ -2,39 +2,39 @@
Coalesced RDD , large scale
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Coalesced RDD: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------
-Coalesce Num Partitions: 100 Num Hosts: 1 150 156 7 0.7 1501.3 1.0X
-Coalesce Num Partitions: 100 Num Hosts: 5 109 113 4 0.9 1087.1 1.4X
-Coalesce Num Partitions: 100 Num Hosts: 10 94 106 11 1.1 943.2 1.6X
-Coalesce Num Partitions: 100 Num Hosts: 20 92 99 8 1.1 924.6 1.6X
-Coalesce Num Partitions: 100 Num Hosts: 40 91 97 6 1.1 906.9 1.7X
-Coalesce Num Partitions: 100 Num Hosts: 80 92 98 7 1.1 916.3 1.6X
-Coalesce Num Partitions: 500 Num Hosts: 1 317 323 7 0.3 3174.6 0.5X
-Coalesce Num Partitions: 500 Num Hosts: 5 139 140 2 0.7 1394.3 1.1X
-Coalesce Num Partitions: 500 Num Hosts: 10 114 129 19 0.9 1144.9 1.3X
-Coalesce Num Partitions: 500 Num Hosts: 20 107 116 15 0.9 1065.8 1.4X
-Coalesce Num Partitions: 500 Num Hosts: 40 99 104 5 1.0 995.0 1.5X
-Coalesce Num Partitions: 500 Num Hosts: 80 96 101 5 1.0 960.9 1.6X
-Coalesce Num Partitions: 1000 Num Hosts: 1 523 533 9 0.2 5231.6 0.3X
-Coalesce Num Partitions: 1000 Num Hosts: 5 188 188 0 0.5 1882.4 0.8X
-Coalesce Num Partitions: 1000 Num Hosts: 10 131 133 2 0.8 1314.5 1.1X
-Coalesce Num Partitions: 1000 Num Hosts: 20 112 114 3 0.9 1120.6 1.3X
-Coalesce Num Partitions: 1000 Num Hosts: 40 105 105 0 1.0 1047.7 1.4X
-Coalesce Num Partitions: 1000 Num Hosts: 80 98 105 5 1.0 984.4 1.5X
-Coalesce Num Partitions: 5000 Num Hosts: 1 2321 2342 28 0.0 23213.0 0.1X
-Coalesce Num Partitions: 5000 Num Hosts: 5 686 697 17 0.1 6858.9 0.2X
-Coalesce Num Partitions: 5000 Num Hosts: 10 386 391 4 0.3 3863.4 0.4X
-Coalesce Num Partitions: 5000 Num Hosts: 20 252 253 2 0.4 2518.2 0.6X
-Coalesce Num Partitions: 5000 Num Hosts: 40 168 171 4 0.6 1683.5 0.9X
-Coalesce Num Partitions: 5000 Num Hosts: 80 132 137 5 0.8 1322.0 1.1X
-Coalesce Num Partitions: 10000 Num Hosts: 1 4250 4268 16 0.0 42497.7 0.0X
-Coalesce Num Partitions: 10000 Num Hosts: 5 1401 1408 7 0.1 14008.0 0.1X
-Coalesce Num Partitions: 10000 Num Hosts: 10 724 725 2 0.1 7237.0 0.2X
-Coalesce Num Partitions: 10000 Num Hosts: 20 396 402 5 0.3 3959.1 0.4X
-Coalesce Num Partitions: 10000 Num Hosts: 40 255 257 2 0.4 2550.1 0.6X
-Coalesce Num Partitions: 10000 Num Hosts: 80 181 183 2 0.6 1811.0 0.8X
+Coalesce Num Partitions: 100 Num Hosts: 1 139 150 13 0.7 1388.1 1.0X
+Coalesce Num Partitions: 100 Num Hosts: 5 109 110 1 0.9 1089.8 1.3X
+Coalesce Num Partitions: 100 Num Hosts: 10 95 105 15 1.0 954.0 1.5X
+Coalesce Num Partitions: 100 Num Hosts: 20 92 98 9 1.1 921.3 1.5X
+Coalesce Num Partitions: 100 Num Hosts: 40 91 93 2 1.1 912.2 1.5X
+Coalesce Num Partitions: 100 Num Hosts: 80 90 96 8 1.1 897.0 1.5X
+Coalesce Num Partitions: 500 Num Hosts: 1 314 319 5 0.3 3143.2 0.4X
+Coalesce Num Partitions: 500 Num Hosts: 5 149 151 3 0.7 1492.4 0.9X
+Coalesce Num Partitions: 500 Num Hosts: 10 112 116 5 0.9 1120.0 1.2X
+Coalesce Num Partitions: 500 Num Hosts: 20 102 102 0 1.0 1017.6 1.4X
+Coalesce Num Partitions: 500 Num Hosts: 40 92 97 8 1.1 918.3 1.5X
+Coalesce Num Partitions: 500 Num Hosts: 80 90 92 3 1.1 898.4 1.5X
+Coalesce Num Partitions: 1000 Num Hosts: 1 528 531 4 0.2 5282.1 0.3X
+Coalesce Num Partitions: 1000 Num Hosts: 5 177 177 0 0.6 1768.9 0.8X
+Coalesce Num Partitions: 1000 Num Hosts: 10 146 147 2 0.7 1457.7 1.0X
+Coalesce Num Partitions: 1000 Num Hosts: 20 109 110 2 0.9 1085.2 1.3X
+Coalesce Num Partitions: 1000 Num Hosts: 40 99 101 1 1.0 992.1 1.4X
+Coalesce Num Partitions: 1000 Num Hosts: 80 93 94 1 1.1 928.0 1.5X
+Coalesce Num Partitions: 5000 Num Hosts: 1 2321 2328 6 0.0 23205.7 0.1X
+Coalesce Num Partitions: 5000 Num Hosts: 5 687 692 7 0.1 6870.2 0.2X
+Coalesce Num Partitions: 5000 Num Hosts: 10 386 389 3 0.3 3861.2 0.4X
+Coalesce Num Partitions: 5000 Num Hosts: 20 240 242 2 0.4 2399.8 0.6X
+Coalesce Num Partitions: 5000 Num Hosts: 40 173 179 10 0.6 1729.4 0.8X
+Coalesce Num Partitions: 5000 Num Hosts: 80 128 131 3 0.8 1277.8 1.1X
+Coalesce Num Partitions: 10000 Num Hosts: 1 4233 4245 12 0.0 42331.1 0.0X
+Coalesce Num Partitions: 10000 Num Hosts: 5 1376 1382 5 0.1 13757.6 0.1X
+Coalesce Num Partitions: 10000 Num Hosts: 10 702 707 5 0.1 7019.0 0.2X
+Coalesce Num Partitions: 10000 Num Hosts: 20 406 408 2 0.2 4059.0 0.3X
+Coalesce Num Partitions: 10000 Num Hosts: 40 247 248 1 0.4 2466.7 0.6X
+Coalesce Num Partitions: 10000 Num Hosts: 80 165 171 5 0.6 1654.9 0.8X
diff --git a/core/benchmarks/KryoBenchmark-jdk21-results.txt b/core/benchmarks/KryoBenchmark-jdk21-results.txt
index ca03441d01a87..c182d1f38c036 100644
--- a/core/benchmarks/KryoBenchmark-jdk21-results.txt
+++ b/core/benchmarks/KryoBenchmark-jdk21-results.txt
@@ -2,27 +2,27 @@
Benchmark Kryo Unsafe vs safe Serialization
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark Kryo Unsafe vs safe Serialization: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-basicTypes: Int with unsafe:true 166 169 6 6.0 165.7 1.0X
-basicTypes: Long with unsafe:true 178 182 4 5.6 177.5 0.9X
-basicTypes: Float with unsafe:true 183 191 9 5.5 182.7 0.9X
-basicTypes: Double with unsafe:true 186 193 7 5.4 186.4 0.9X
-Array: Int with unsafe:true 1 1 0 745.8 1.3 123.5X
-Array: Long with unsafe:true 2 3 0 451.7 2.2 74.8X
-Array: Float with unsafe:true 1 1 0 743.0 1.3 123.1X
-Array: Double with unsafe:true 2 2 0 475.6 2.1 78.8X
-Map of string->Double with unsafe:true 27 28 1 37.0 27.0 6.1X
-basicTypes: Int with unsafe:false 198 199 1 5.1 197.7 0.8X
-basicTypes: Long with unsafe:false 220 221 1 4.5 219.8 0.8X
-basicTypes: Float with unsafe:false 206 208 1 4.8 206.3 0.8X
-basicTypes: Double with unsafe:false 222 225 2 4.5 221.9 0.7X
-Array: Int with unsafe:false 13 14 1 78.0 12.8 12.9X
-Array: Long with unsafe:false 21 21 1 48.2 20.8 8.0X
-Array: Float with unsafe:false 6 6 0 178.9 5.6 29.6X
-Array: Double with unsafe:false 15 16 0 65.3 15.3 10.8X
-Map of string->Double with unsafe:false 28 29 2 35.2 28.4 5.8X
+basicTypes: Int with unsafe:true 169 170 1 5.9 168.9 1.0X
+basicTypes: Long with unsafe:true 177 182 3 5.7 176.7 1.0X
+basicTypes: Float with unsafe:true 184 185 1 5.4 184.3 0.9X
+basicTypes: Double with unsafe:true 184 186 3 5.4 183.6 0.9X
+Array: Int with unsafe:true 1 1 0 755.5 1.3 127.6X
+Array: Long with unsafe:true 2 2 0 496.5 2.0 83.8X
+Array: Float with unsafe:true 1 1 0 754.2 1.3 127.4X
+Array: Double with unsafe:true 2 2 0 500.0 2.0 84.4X
+Map of string->Double with unsafe:true 27 27 0 37.7 26.5 6.4X
+basicTypes: Int with unsafe:false 204 205 0 4.9 203.9 0.8X
+basicTypes: Long with unsafe:false 216 218 2 4.6 216.1 0.8X
+basicTypes: Float with unsafe:false 195 196 1 5.1 194.7 0.9X
+basicTypes: Double with unsafe:false 197 199 2 5.1 197.4 0.9X
+Array: Int with unsafe:false 13 13 0 77.6 12.9 13.1X
+Array: Long with unsafe:false 20 20 0 49.4 20.2 8.3X
+Array: Float with unsafe:false 8 8 0 132.8 7.5 22.4X
+Array: Double with unsafe:false 9 10 0 106.0 9.4 17.9X
+Map of string->Double with unsafe:false 27 28 2 37.0 27.0 6.2X
diff --git a/core/benchmarks/KryoBenchmark-results.txt b/core/benchmarks/KryoBenchmark-results.txt
index bffc9eafb79e3..d526e5f4c47c2 100644
--- a/core/benchmarks/KryoBenchmark-results.txt
+++ b/core/benchmarks/KryoBenchmark-results.txt
@@ -2,27 +2,27 @@
Benchmark Kryo Unsafe vs safe Serialization
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark Kryo Unsafe vs safe Serialization: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-basicTypes: Int with unsafe:true 172 174 1 5.8 171.9 1.0X
-basicTypes: Long with unsafe:true 196 197 0 5.1 196.2 0.9X
-basicTypes: Float with unsafe:true 193 195 2 5.2 192.7 0.9X
-basicTypes: Double with unsafe:true 193 194 1 5.2 193.2 0.9X
-Array: Int with unsafe:true 1 1 0 715.2 1.4 122.9X
-Array: Long with unsafe:true 2 2 0 474.2 2.1 81.5X
-Array: Float with unsafe:true 1 1 0 718.2 1.4 123.5X
-Array: Double with unsafe:true 2 2 0 475.8 2.1 81.8X
-Map of string->Double with unsafe:true 27 28 0 36.7 27.2 6.3X
-basicTypes: Int with unsafe:false 207 209 5 4.8 207.3 0.8X
-basicTypes: Long with unsafe:false 239 241 2 4.2 238.9 0.7X
-basicTypes: Float with unsafe:false 215 217 2 4.6 215.4 0.8X
-basicTypes: Double with unsafe:false 220 225 7 4.5 220.2 0.8X
-Array: Int with unsafe:false 16 20 7 63.4 15.8 10.9X
-Array: Long with unsafe:false 22 22 0 45.9 21.8 7.9X
-Array: Float with unsafe:false 6 6 1 170.0 5.9 29.2X
-Array: Double with unsafe:false 10 10 0 98.6 10.1 16.9X
-Map of string->Double with unsafe:false 28 29 1 35.9 27.9 6.2X
+basicTypes: Int with unsafe:true 167 169 2 6.0 167.1 1.0X
+basicTypes: Long with unsafe:true 190 192 2 5.3 190.3 0.9X
+basicTypes: Float with unsafe:true 187 189 1 5.3 187.3 0.9X
+basicTypes: Double with unsafe:true 197 198 3 5.1 196.5 0.9X
+Array: Int with unsafe:true 1 1 0 740.3 1.4 123.7X
+Array: Long with unsafe:true 2 2 0 448.5 2.2 74.9X
+Array: Float with unsafe:true 1 1 0 743.0 1.3 124.1X
+Array: Double with unsafe:true 2 2 0 446.8 2.2 74.6X
+Map of string->Double with unsafe:true 27 29 1 36.5 27.4 6.1X
+basicTypes: Int with unsafe:false 223 224 1 4.5 223.0 0.7X
+basicTypes: Long with unsafe:false 244 245 1 4.1 244.0 0.7X
+basicTypes: Float with unsafe:false 225 225 1 4.5 224.7 0.7X
+basicTypes: Double with unsafe:false 222 224 1 4.5 221.9 0.8X
+Array: Int with unsafe:false 14 15 0 69.4 14.4 11.6X
+Array: Long with unsafe:false 20 21 0 49.1 20.4 8.2X
+Array: Float with unsafe:false 6 6 0 170.8 5.9 28.5X
+Array: Double with unsafe:false 10 10 0 102.7 9.7 17.2X
+Map of string->Double with unsafe:false 31 32 1 32.5 30.8 5.4X
diff --git a/core/benchmarks/KryoIteratorBenchmark-jdk21-results.txt b/core/benchmarks/KryoIteratorBenchmark-jdk21-results.txt
index 835b16e24d95f..c853973a3ed1f 100644
--- a/core/benchmarks/KryoIteratorBenchmark-jdk21-results.txt
+++ b/core/benchmarks/KryoIteratorBenchmark-jdk21-results.txt
@@ -2,27 +2,27 @@
Benchmark of kryo asIterator on deserialization stream
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark of kryo asIterator on deserialization stream: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------------------------
-Colletion of int with 1 elements, useIterator: true 6 7 0 1.5 645.9 1.0X
-Colletion of int with 10 elements, useIterator: true 13 14 0 0.8 1330.8 0.5X
-Colletion of int with 100 elements, useIterator: true 80 81 1 0.1 7987.1 0.1X
-Colletion of string with 1 elements, useIterator: true 8 8 0 1.3 787.4 0.8X
-Colletion of string with 10 elements, useIterator: true 21 21 0 0.5 2113.7 0.3X
-Colletion of string with 100 elements, useIterator: true 161 162 1 0.1 16108.9 0.0X
-Colletion of Array[int] with 1 elements, useIterator: true 7 8 0 1.3 747.8 0.9X
-Colletion of Array[int] with 10 elements, useIterator: true 19 19 0 0.5 1879.8 0.3X
-Colletion of Array[int] with 100 elements, useIterator: true 140 141 1 0.1 14008.3 0.0X
-Colletion of int with 1 elements, useIterator: false 6 7 0 1.6 642.6 1.0X
-Colletion of int with 10 elements, useIterator: false 14 15 1 0.7 1414.6 0.5X
-Colletion of int with 100 elements, useIterator: false 87 88 1 0.1 8699.0 0.1X
-Colletion of string with 1 elements, useIterator: false 7 8 0 1.3 746.5 0.9X
-Colletion of string with 10 elements, useIterator: false 22 22 0 0.5 2192.3 0.3X
-Colletion of string with 100 elements, useIterator: false 161 162 2 0.1 16091.2 0.0X
-Colletion of Array[int] with 1 elements, useIterator: false 7 8 0 1.4 719.6 0.9X
-Colletion of Array[int] with 10 elements, useIterator: false 19 19 0 0.5 1869.6 0.3X
-Colletion of Array[int] with 100 elements, useIterator: false 138 139 1 0.1 13766.1 0.0X
+Colletion of int with 1 elements, useIterator: true 6 6 0 1.6 614.1 1.0X
+Colletion of int with 10 elements, useIterator: true 14 14 0 0.7 1353.8 0.5X
+Colletion of int with 100 elements, useIterator: true 83 84 1 0.1 8301.3 0.1X
+Colletion of string with 1 elements, useIterator: true 8 8 0 1.3 780.9 0.8X
+Colletion of string with 10 elements, useIterator: true 23 23 1 0.4 2274.3 0.3X
+Colletion of string with 100 elements, useIterator: true 171 172 1 0.1 17072.6 0.0X
+Colletion of Array[int] with 1 elements, useIterator: true 7 8 0 1.4 736.9 0.8X
+Colletion of Array[int] with 10 elements, useIterator: true 20 20 0 0.5 1976.1 0.3X
+Colletion of Array[int] with 100 elements, useIterator: true 147 148 1 0.1 14712.0 0.0X
+Colletion of int with 1 elements, useIterator: false 6 6 0 1.6 619.4 1.0X
+Colletion of int with 10 elements, useIterator: false 13 14 0 0.7 1343.7 0.5X
+Colletion of int with 100 elements, useIterator: false 84 85 0 0.1 8391.9 0.1X
+Colletion of string with 1 elements, useIterator: false 7 8 0 1.4 721.7 0.9X
+Colletion of string with 10 elements, useIterator: false 22 22 0 0.5 2208.8 0.3X
+Colletion of string with 100 elements, useIterator: false 164 164 0 0.1 16358.2 0.0X
+Colletion of Array[int] with 1 elements, useIterator: false 7 7 0 1.4 702.7 0.9X
+Colletion of Array[int] with 10 elements, useIterator: false 19 20 0 0.5 1911.7 0.3X
+Colletion of Array[int] with 100 elements, useIterator: false 143 144 1 0.1 14274.6 0.0X
diff --git a/core/benchmarks/KryoIteratorBenchmark-results.txt b/core/benchmarks/KryoIteratorBenchmark-results.txt
index 6caa842d0e4d2..016f8e38bd807 100644
--- a/core/benchmarks/KryoIteratorBenchmark-results.txt
+++ b/core/benchmarks/KryoIteratorBenchmark-results.txt
@@ -2,27 +2,27 @@
Benchmark of kryo asIterator on deserialization stream
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark of kryo asIterator on deserialization stream: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------------------------
-Colletion of int with 1 elements, useIterator: true 6 6 0 1.6 629.3 1.0X
-Colletion of int with 10 elements, useIterator: true 14 14 0 0.7 1350.0 0.5X
-Colletion of int with 100 elements, useIterator: true 83 83 1 0.1 8255.8 0.1X
-Colletion of string with 1 elements, useIterator: true 8 8 0 1.3 750.8 0.8X
-Colletion of string with 10 elements, useIterator: true 21 21 1 0.5 2116.4 0.3X
-Colletion of string with 100 elements, useIterator: true 162 163 1 0.1 16191.1 0.0X
-Colletion of Array[int] with 1 elements, useIterator: true 7 8 0 1.4 732.3 0.9X
-Colletion of Array[int] with 10 elements, useIterator: true 19 19 0 0.5 1906.4 0.3X
-Colletion of Array[int] with 100 elements, useIterator: true 142 143 0 0.1 14222.9 0.0X
-Colletion of int with 1 elements, useIterator: false 6 6 0 1.7 604.6 1.0X
-Colletion of int with 10 elements, useIterator: false 13 13 0 0.8 1325.8 0.5X
-Colletion of int with 100 elements, useIterator: false 83 83 0 0.1 8261.2 0.1X
-Colletion of string with 1 elements, useIterator: false 7 8 1 1.4 719.2 0.9X
-Colletion of string with 10 elements, useIterator: false 22 22 0 0.5 2203.0 0.3X
-Colletion of string with 100 elements, useIterator: false 163 163 1 0.1 16294.8 0.0X
-Colletion of Array[int] with 1 elements, useIterator: false 7 7 0 1.5 674.8 0.9X
-Colletion of Array[int] with 10 elements, useIterator: false 18 19 0 0.6 1808.7 0.3X
-Colletion of Array[int] with 100 elements, useIterator: false 135 135 0 0.1 13481.7 0.0X
+Colletion of int with 1 elements, useIterator: true 6 7 0 1.6 641.1 1.0X
+Colletion of int with 10 elements, useIterator: true 14 14 0 0.7 1354.7 0.5X
+Colletion of int with 100 elements, useIterator: true 81 82 0 0.1 8098.5 0.1X
+Colletion of string with 1 elements, useIterator: true 8 8 0 1.3 779.2 0.8X
+Colletion of string with 10 elements, useIterator: true 22 23 1 0.5 2215.9 0.3X
+Colletion of string with 100 elements, useIterator: true 162 162 0 0.1 16186.6 0.0X
+Colletion of Array[int] with 1 elements, useIterator: true 7 8 0 1.3 745.7 0.9X
+Colletion of Array[int] with 10 elements, useIterator: true 20 20 0 0.5 1976.9 0.3X
+Colletion of Array[int] with 100 elements, useIterator: true 149 150 3 0.1 14870.3 0.0X
+Colletion of int with 1 elements, useIterator: false 6 7 0 1.6 630.8 1.0X
+Colletion of int with 10 elements, useIterator: false 13 14 0 0.7 1345.9 0.5X
+Colletion of int with 100 elements, useIterator: false 83 84 1 0.1 8279.3 0.1X
+Colletion of string with 1 elements, useIterator: false 7 8 1 1.3 741.4 0.9X
+Colletion of string with 10 elements, useIterator: false 21 22 1 0.5 2116.2 0.3X
+Colletion of string with 100 elements, useIterator: false 167 167 0 0.1 16683.1 0.0X
+Colletion of Array[int] with 1 elements, useIterator: false 7 7 0 1.4 709.9 0.9X
+Colletion of Array[int] with 10 elements, useIterator: false 19 20 0 0.5 1912.0 0.3X
+Colletion of Array[int] with 100 elements, useIterator: false 139 140 1 0.1 13891.9 0.0X
diff --git a/core/benchmarks/KryoSerializerBenchmark-jdk21-results.txt b/core/benchmarks/KryoSerializerBenchmark-jdk21-results.txt
index e2c03b0acd23c..eaa4579ea6f5a 100644
--- a/core/benchmarks/KryoSerializerBenchmark-jdk21-results.txt
+++ b/core/benchmarks/KryoSerializerBenchmark-jdk21-results.txt
@@ -2,11 +2,11 @@
Benchmark KryoPool vs old"pool of 1" implementation
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark KryoPool vs old"pool of 1" implementation: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------
-KryoPool:true 3610 5082 1510 0.0 7219633.5 1.0X
-KryoPool:false 5886 7699 1454 0.0 11772046.4 0.6X
+KryoPool:true 3317 5001 1551 0.0 6634900.3 1.0X
+KryoPool:false 5484 7586 1475 0.0 10968339.2 0.6X
diff --git a/core/benchmarks/KryoSerializerBenchmark-results.txt b/core/benchmarks/KryoSerializerBenchmark-results.txt
index 54b21654ce054..f0fefa24a833c 100644
--- a/core/benchmarks/KryoSerializerBenchmark-results.txt
+++ b/core/benchmarks/KryoSerializerBenchmark-results.txt
@@ -2,11 +2,11 @@
Benchmark KryoPool vs old"pool of 1" implementation
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1021-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark KryoPool vs old"pool of 1" implementation: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------------
-KryoPool:true 3682 5317 1787 0.0 7363182.2 1.0X
-KryoPool:false 5853 7922 1339 0.0 11705848.9 0.6X
+KryoPool:true 3514 5193 1835 0.0 7028412.4 1.0X
+KryoPool:false 5665 7598 1316 0.0 11329264.8 0.6X
diff --git a/core/benchmarks/LZFBenchmark-jdk21-results.txt b/core/benchmarks/LZFBenchmark-jdk21-results.txt
index b96edb0de4fa8..26adedf06015f 100644
--- a/core/benchmarks/LZFBenchmark-jdk21-results.txt
+++ b/core/benchmarks/LZFBenchmark-jdk21-results.txt
@@ -2,18 +2,18 @@
Benchmark LZFCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Compress small objects: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------
-Compression 256000000 int values in parallel 590 602 11 433.7 2.3 1.0X
-Compression 256000000 int values single-threaded 557 558 1 460.0 2.2 1.1X
+Compression 256000000 int values in parallel 592 604 12 432.7 2.3 1.0X
+Compression 256000000 int values single-threaded 558 560 2 458.7 2.2 1.1X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Compress large objects: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Compression 1024 array values in 1 threads 42 47 3 0.0 41185.8 1.0X
-Compression 1024 array values single-threaded 31 31 0 0.0 30214.3 1.4X
+Compression 1024 array values in 1 threads 44 50 3 0.0 43112.9 1.0X
+Compression 1024 array values single-threaded 32 33 0 0.0 31315.1 1.4X
diff --git a/core/benchmarks/LZFBenchmark-results.txt b/core/benchmarks/LZFBenchmark-results.txt
index d999bb12f1fbc..713482e7a2e3d 100644
--- a/core/benchmarks/LZFBenchmark-results.txt
+++ b/core/benchmarks/LZFBenchmark-results.txt
@@ -2,18 +2,18 @@
Benchmark LZFCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Compress small objects: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------
-Compression 256000000 int values in parallel 608 619 8 420.8 2.4 1.0X
-Compression 256000000 int values single-threaded 584 586 2 438.3 2.3 1.0X
+Compression 256000000 int values in parallel 591 611 14 433.3 2.3 1.0X
+Compression 256000000 int values single-threaded 613 619 6 417.3 2.4 1.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Compress large objects: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Compression 1024 array values in 1 threads 37 46 6 0.0 35682.0 1.0X
-Compression 1024 array values single-threaded 32 33 0 0.0 31517.3 1.1X
+Compression 1024 array values in 1 threads 39 45 5 0.0 38040.2 1.0X
+Compression 1024 array values single-threaded 31 31 1 0.0 29909.9 1.3X
diff --git a/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt b/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt
index 2639b7194286f..3e3baa1708890 100644
--- a/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt
+++ b/core/benchmarks/MapStatusesConvertBenchmark-jdk21-results.txt
@@ -2,12 +2,12 @@
MapStatuses Convert Benchmark
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
MapStatuses Convert: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Num Maps: 50000 Fetch partitions:500 699 706 6 0.0 698685201.0 1.0X
-Num Maps: 50000 Fetch partitions:1000 1587 1596 11 0.0 1586755889.0 0.4X
-Num Maps: 50000 Fetch partitions:1500 2442 2479 33 0.0 2442011440.0 0.3X
+Num Maps: 50000 Fetch partitions:500 717 721 5 0.0 716748683.0 1.0X
+Num Maps: 50000 Fetch partitions:1000 1615 1640 30 0.0 1615066557.0 0.4X
+Num Maps: 50000 Fetch partitions:1500 2485 2497 12 0.0 2484802897.0 0.3X
diff --git a/core/benchmarks/MapStatusesConvertBenchmark-results.txt b/core/benchmarks/MapStatusesConvertBenchmark-results.txt
index bd17fb8c53dee..ebadf6f9bc5d1 100644
--- a/core/benchmarks/MapStatusesConvertBenchmark-results.txt
+++ b/core/benchmarks/MapStatusesConvertBenchmark-results.txt
@@ -2,12 +2,12 @@
MapStatuses Convert Benchmark
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
MapStatuses Convert: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Num Maps: 50000 Fetch partitions:500 750 763 16 0.0 749869953.0 1.0X
-Num Maps: 50000 Fetch partitions:1000 1826 1851 24 0.0 1825734215.0 0.4X
-Num Maps: 50000 Fetch partitions:1500 2846 2863 17 0.0 2845610666.0 0.3X
+Num Maps: 50000 Fetch partitions:500 868 871 3 0.0 867735046.0 1.0X
+Num Maps: 50000 Fetch partitions:1000 1472 1492 29 0.0 1471850953.0 0.6X
+Num Maps: 50000 Fetch partitions:1500 2239 2252 14 0.0 2238836298.0 0.4X
diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk21-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk21-results.txt
index 936af3f1dd149..3304f827bb4ea 100644
--- a/core/benchmarks/MapStatusesSerDeserBenchmark-jdk21-results.txt
+++ b/core/benchmarks/MapStatusesSerDeserBenchmark-jdk21-results.txt
@@ -1,64 +1,64 @@
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization 85 88 4 2.3 425.9 1.0X
-Deserialization 141 148 8 1.4 707.0 0.6X
+Serialization 84 87 4 2.4 420.1 1.0X
+Deserialization 142 150 5 1.4 709.3 0.6X
Compressed Serialized MapStatus sizes: 426.0 B
Compressed Serialized Broadcast MapStatus sizes: 2.5 MiB
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------
-Serialization 75 76 1 2.7 376.2 1.0X
-Deserialization 141 145 9 1.4 706.7 0.5X
+Serialization 82 83 1 2.4 409.4 1.0X
+Deserialization 141 143 4 1.4 704.1 0.6X
Compressed Serialized MapStatus sizes: 2.5 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------
-Serialization 161 166 8 1.2 805.6 1.0X
-Deserialization 158 163 13 1.3 790.6 1.0X
+Serialization 154 160 10 1.3 770.5 1.0X
+Deserialization 157 161 7 1.3 787.2 1.0X
Compressed Serialized MapStatus sizes: 442.0 B
Compressed Serialized Broadcast MapStatus sizes: 13.6 MiB
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Serialization 143 143 0 1.4 712.6 1.0X
-Deserialization 158 164 9 1.3 789.7 0.9X
+Serialization 152 154 1 1.3 761.2 1.0X
+Deserialization 157 159 4 1.3 787.0 1.0X
Compressed Serialized MapStatus sizes: 13.6 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Serialization 699 725 44 0.3 3494.9 1.0X
-Deserialization 327 344 11 0.6 1633.4 2.1X
+Serialization 727 749 37 0.3 3634.8 1.0X
+Deserialization 319 337 18 0.6 1593.8 2.3X
Compressed Serialized MapStatus sizes: 570.0 B
Compressed Serialized Broadcast MapStatus sizes: 122.3 MiB
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------
-Serialization 562 567 5 0.4 2809.3 1.0X
-Deserialization 320 341 18 0.6 1598.2 1.8X
+Serialization 577 579 2 0.3 2884.2 1.0X
+Deserialization 316 335 18 0.6 1579.5 1.8X
Compressed Serialized MapStatus sizes: 122.3 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
diff --git a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt
index 48fe51f7cf938..af0468b055d72 100644
--- a/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt
+++ b/core/benchmarks/MapStatusesSerDeserBenchmark-results.txt
@@ -1,64 +1,64 @@
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 10 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-------------------------------------------------------------------------------------------------------------------------
-Serialization 90 93 3 2.2 449.1 1.0X
-Deserialization 137 144 6 1.5 685.7 0.7X
+Serialization 90 93 3 2.2 450.5 1.0X
+Deserialization 142 146 6 1.4 710.5 0.6X
Compressed Serialized MapStatus sizes: 426.0 B
Compressed Serialized Broadcast MapStatus sizes: 2.5 MiB
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 10 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------
-Serialization 75 76 1 2.7 376.3 1.0X
-Deserialization 137 142 6 1.5 683.4 0.6X
+Serialization 84 85 1 2.4 421.4 1.0X
+Deserialization 141 147 8 1.4 707.0 0.6X
Compressed Serialized MapStatus sizes: 2.5 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 100 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------
-Serialization 153 158 5 1.3 762.6 1.0X
-Deserialization 153 161 11 1.3 765.5 1.0X
+Serialization 164 168 4 1.2 817.8 1.0X
+Deserialization 157 164 8 1.3 786.7 1.0X
Compressed Serialized MapStatus sizes: 442.0 B
Compressed Serialized Broadcast MapStatus sizes: 13.6 MiB
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 100 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Serialization 156 160 3 1.3 779.5 1.0X
-Deserialization 153 160 11 1.3 762.6 1.0X
+Serialization 151 152 2 1.3 752.9 1.0X
+Deserialization 157 164 10 1.3 786.9 1.0X
Compressed Serialized MapStatus sizes: 13.6 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 1000 blocks w/ broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
---------------------------------------------------------------------------------------------------------------------------
-Serialization 691 716 30 0.3 3457.5 1.0X
-Deserialization 330 342 9 0.6 1652.0 2.1X
+Serialization 717 726 8 0.3 3586.0 1.0X
+Deserialization 334 350 13 0.6 1669.7 2.1X
-Compressed Serialized MapStatus sizes: 569.0 B
+Compressed Serialized MapStatus sizes: 568.0 B
Compressed Serialized Broadcast MapStatus sizes: 122.3 MiB
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
200000 MapOutputs, 1000 blocks w/o broadcast: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------
-Serialization 573 581 8 0.3 2866.3 1.0X
-Deserialization 323 336 12 0.6 1616.5 1.8X
+Serialization 578 584 11 0.3 2888.6 1.0X
+Deserialization 332 352 17 0.6 1660.3 1.7X
Compressed Serialized MapStatus sizes: 122.3 MiB
Compressed Serialized Broadcast MapStatus sizes: 0.0 B
diff --git a/core/benchmarks/PercentileHeapBenchmark-jdk21-results.txt b/core/benchmarks/PercentileHeapBenchmark-jdk21-results.txt
index 5d08b56176fbc..b005440163bce 100644
--- a/core/benchmarks/PercentileHeapBenchmark-jdk21-results.txt
+++ b/core/benchmarks/PercentileHeapBenchmark-jdk21-results.txt
@@ -2,40 +2,40 @@
PercentileHeap Operations
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 10000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 1 1 0 10.2 98.3 1.0X
-Percentile: 0.9 1 1 0 17.4 57.3 1.7X
-Percentile: 0.95 0 0 0 22.8 43.9 2.2X
-Percentile: 0.99 0 0 0 30.4 32.9 3.0X
+Percentile: 0.5 1 1 0 9.8 102.2 1.0X
+Percentile: 0.9 1 1 0 17.5 57.2 1.8X
+Percentile: 0.95 0 0 0 22.2 45.0 2.3X
+Percentile: 0.99 0 0 0 30.1 33.2 3.1X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 50000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 5 5 0 9.6 104.6 1.0X
-Percentile: 0.9 3 3 0 16.8 59.6 1.8X
-Percentile: 0.95 2 2 0 20.8 48.1 2.2X
-Percentile: 0.99 2 2 0 29.3 34.2 3.1X
+Percentile: 0.5 5 6 0 9.2 109.2 1.0X
+Percentile: 0.9 3 4 1 15.9 63.0 1.7X
+Percentile: 0.95 2 2 0 21.4 46.8 2.3X
+Percentile: 0.99 2 2 0 29.0 34.4 3.2X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 100000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 12 12 0 8.2 121.5 1.0X
-Percentile: 0.9 6 7 0 15.6 64.1 1.9X
-Percentile: 0.95 5 5 0 19.9 50.1 2.4X
-Percentile: 0.99 4 4 0 28.3 35.3 3.4X
+Percentile: 0.5 13 13 0 7.9 126.5 1.0X
+Percentile: 0.9 7 7 1 14.7 67.9 1.9X
+Percentile: 0.95 5 5 0 20.0 50.1 2.5X
+Percentile: 0.99 4 4 0 28.2 35.4 3.6X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 200000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 26 26 0 7.8 128.2 1.0X
-Percentile: 0.9 14 14 0 14.5 68.8 1.9X
-Percentile: 0.95 11 11 0 19.0 52.7 2.4X
-Percentile: 0.99 7 7 0 27.1 36.9 3.5X
+Percentile: 0.5 27 27 1 7.5 134.0 1.0X
+Percentile: 0.9 14 14 0 14.3 70.1 1.9X
+Percentile: 0.95 11 11 0 18.5 54.1 2.5X
+Percentile: 0.99 7 7 0 27.2 36.8 3.6X
diff --git a/core/benchmarks/PercentileHeapBenchmark-results.txt b/core/benchmarks/PercentileHeapBenchmark-results.txt
index 2967772a8c273..979030652a5b7 100644
--- a/core/benchmarks/PercentileHeapBenchmark-results.txt
+++ b/core/benchmarks/PercentileHeapBenchmark-results.txt
@@ -2,40 +2,40 @@
PercentileHeap Operations
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 10000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 1 1 0 9.9 101.1 1.0X
-Percentile: 0.9 1 1 0 17.2 58.3 1.7X
-Percentile: 0.95 0 0 0 22.9 43.6 2.3X
-Percentile: 0.99 0 0 0 30.5 32.8 3.1X
+Percentile: 0.5 1 1 0 10.3 97.4 1.0X
+Percentile: 0.9 1 1 0 18.3 54.7 1.8X
+Percentile: 0.95 0 0 0 23.4 42.8 2.3X
+Percentile: 0.99 0 0 0 30.8 32.5 3.0X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 50000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
-----------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 5 6 0 9.3 107.0 1.0X
-Percentile: 0.9 3 3 0 16.8 59.4 1.8X
-Percentile: 0.95 2 2 0 21.8 45.9 2.3X
-Percentile: 0.99 2 2 0 28.8 34.8 3.1X
+Percentile: 0.5 5 6 1 9.4 106.8 1.0X
+Percentile: 0.9 3 3 0 16.5 60.5 1.8X
+Percentile: 0.95 2 2 0 22.0 45.5 2.3X
+Percentile: 0.99 2 2 0 29.0 34.5 3.1X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 100000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 12 12 0 8.2 122.2 1.0X
-Percentile: 0.9 6 7 0 15.8 63.5 1.9X
-Percentile: 0.95 5 5 0 20.4 49.0 2.5X
-Percentile: 0.99 3 3 0 28.7 34.9 3.5X
+Percentile: 0.5 12 12 0 8.6 116.5 1.0X
+Percentile: 0.9 7 7 0 15.3 65.3 1.8X
+Percentile: 0.95 5 5 0 20.9 47.8 2.4X
+Percentile: 0.99 4 4 0 28.1 35.5 3.3X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1015-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
PercentileHeap Operations - Input Size: 200000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------
-Percentile: 0.5 26 26 0 7.7 130.1 1.0X
-Percentile: 0.9 14 14 0 14.4 69.4 1.9X
-Percentile: 0.95 10 11 1 19.3 51.8 2.5X
-Percentile: 0.99 7 7 0 27.6 36.2 3.6X
+Percentile: 0.5 25 26 0 7.9 126.6 1.0X
+Percentile: 0.9 14 14 0 14.6 68.6 1.8X
+Percentile: 0.95 10 11 1 19.4 51.6 2.5X
+Percentile: 0.99 7 7 0 27.3 36.7 3.5X
diff --git a/core/benchmarks/PersistenceEngineBenchmark-jdk21-results.txt b/core/benchmarks/PersistenceEngineBenchmark-jdk21-results.txt
index 49e54abebfea6..dbaca446f7782 100644
--- a/core/benchmarks/PersistenceEngineBenchmark-jdk21-results.txt
+++ b/core/benchmarks/PersistenceEngineBenchmark-jdk21-results.txt
@@ -2,17 +2,17 @@
PersistenceEngineBenchmark
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
1000 Workers: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------------------
-ZooKeeperPersistenceEngine with JavaSerializer 6237 6453 236 0.0 6236919.5 1.0X
-FileSystemPersistenceEngine with JavaSerializer 2481 2495 12 0.0 2480642.5 2.5X
-FileSystemPersistenceEngine with JavaSerializer (lz4) 776 805 29 0.0 776056.9 8.0X
-FileSystemPersistenceEngine with JavaSerializer (lzf) 728 733 4 0.0 727884.9 8.6X
-FileSystemPersistenceEngine with JavaSerializer (snappy) 739 753 13 0.0 738838.5 8.4X
-FileSystemPersistenceEngine with JavaSerializer (zstd) 976 989 22 0.0 976199.8 6.4X
-RocksDBPersistenceEngine with JavaSerializer 283 286 4 0.0 282774.2 22.1X
-BlackHolePersistenceEngine 0 0 0 6.0 167.9 37143.3X
+ZooKeeperPersistenceEngine with JavaSerializer 6956 7080 115 0.0 6956379.5 1.0X
+FileSystemPersistenceEngine with JavaSerializer 2402 2447 67 0.0 2401650.8 2.9X
+FileSystemPersistenceEngine with JavaSerializer (lz4) 779 796 21 0.0 778925.0 8.9X
+FileSystemPersistenceEngine with JavaSerializer (lzf) 783 800 16 0.0 782977.7 8.9X
+FileSystemPersistenceEngine with JavaSerializer (snappy) 709 752 38 0.0 709110.4 9.8X
+FileSystemPersistenceEngine with JavaSerializer (zstd) 899 981 78 0.0 898701.3 7.7X
+RocksDBPersistenceEngine with JavaSerializer 280 280 1 0.0 279833.2 24.9X
+BlackHolePersistenceEngine 0 0 0 6.0 165.9 41930.9X
diff --git a/core/benchmarks/PersistenceEngineBenchmark-results.txt b/core/benchmarks/PersistenceEngineBenchmark-results.txt
index 91390591e312b..566ca47a5b7d7 100644
--- a/core/benchmarks/PersistenceEngineBenchmark-results.txt
+++ b/core/benchmarks/PersistenceEngineBenchmark-results.txt
@@ -2,17 +2,17 @@
PersistenceEngineBenchmark
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
1000 Workers: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------------------
-ZooKeeperPersistenceEngine with JavaSerializer 5746 5947 252 0.0 5746489.4 1.0X
-FileSystemPersistenceEngine with JavaSerializer 2520 2545 23 0.0 2520044.8 2.3X
-FileSystemPersistenceEngine with JavaSerializer (lz4) 782 807 35 0.0 782074.1 7.3X
-FileSystemPersistenceEngine with JavaSerializer (lzf) 731 768 37 0.0 731043.3 7.9X
-FileSystemPersistenceEngine with JavaSerializer (snappy) 817 831 17 0.0 816894.1 7.0X
-FileSystemPersistenceEngine with JavaSerializer (zstd) 950 959 14 0.0 950456.4 6.0X
-RocksDBPersistenceEngine with JavaSerializer 285 287 3 0.0 285388.0 20.1X
-BlackHolePersistenceEngine 0 0 0 5.9 170.2 33755.4X
+ZooKeeperPersistenceEngine with JavaSerializer 7368 7576 255 0.0 7368098.5 1.0X
+FileSystemPersistenceEngine with JavaSerializer 2454 2487 37 0.0 2454081.7 3.0X
+FileSystemPersistenceEngine with JavaSerializer (lz4) 784 835 47 0.0 784080.9 9.4X
+FileSystemPersistenceEngine with JavaSerializer (lzf) 743 806 63 0.0 742767.1 9.9X
+FileSystemPersistenceEngine with JavaSerializer (snappy) 787 811 34 0.0 786859.3 9.4X
+FileSystemPersistenceEngine with JavaSerializer (zstd) 967 1004 32 0.0 966791.5 7.6X
+RocksDBPersistenceEngine with JavaSerializer 289 292 3 0.0 288790.5 25.5X
+BlackHolePersistenceEngine 0 0 0 6.0 167.7 43938.0X
diff --git a/core/benchmarks/PropertiesCloneBenchmark-jdk21-results.txt b/core/benchmarks/PropertiesCloneBenchmark-jdk21-results.txt
index 0dda0a89b6119..ee4a3af6d7fbc 100644
--- a/core/benchmarks/PropertiesCloneBenchmark-jdk21-results.txt
+++ b/core/benchmarks/PropertiesCloneBenchmark-jdk21-results.txt
@@ -2,39 +2,39 @@
Properties Cloning
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Empty Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
SerializationUtils.clone 0 0 0 0.3 3145.0 1.0X
-Utils.cloneProperties 0 0 0 11.2 89.0 35.3X
+Utils.cloneProperties 0 0 0 11.1 90.0 34.9X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
System Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.0 148318.0 1.0X
-Utils.cloneProperties 0 0 0 0.2 4708.0 31.5X
+SerializationUtils.clone 0 0 0 0.0 166011.0 1.0X
+Utils.cloneProperties 0 0 0 0.2 4758.0 34.9X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Small Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.0 241993.0 1.0X
-Utils.cloneProperties 0 0 0 0.1 6812.0 35.5X
+SerializationUtils.clone 0 0 0 0.0 241673.0 1.0X
+Utils.cloneProperties 0 0 0 0.2 5450.0 44.3X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Medium Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 1 1 0 0.0 858356.0 1.0X
-Utils.cloneProperties 0 0 0 0.0 29876.0 28.7X
+SerializationUtils.clone 1 1 0 0.0 855822.0 1.0X
+Utils.cloneProperties 0 0 0 0.0 27581.0 31.0X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Large Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 2 2 0 0.0 1612428.0 1.0X
-Utils.cloneProperties 0 0 0 0.0 59872.0 26.9X
+SerializationUtils.clone 2 2 0 0.0 1611288.0 1.0X
+Utils.cloneProperties 0 0 0 0.0 55464.0 29.1X
diff --git a/core/benchmarks/PropertiesCloneBenchmark-results.txt b/core/benchmarks/PropertiesCloneBenchmark-results.txt
index 2cf84524d1c39..2371d8efac012 100644
--- a/core/benchmarks/PropertiesCloneBenchmark-results.txt
+++ b/core/benchmarks/PropertiesCloneBenchmark-results.txt
@@ -2,39 +2,39 @@
Properties Cloning
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Empty Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.3 3146.0 1.0X
-Utils.cloneProperties 0 0 0 11.1 90.0 35.0X
+SerializationUtils.clone 0 0 0 0.3 3056.0 1.0X
+Utils.cloneProperties 0 0 0 11.1 90.0 34.0X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
System Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.0 150510.0 1.0X
-Utils.cloneProperties 0 0 0 0.2 4017.0 37.5X
+SerializationUtils.clone 0 0 0 0.0 178123.0 1.0X
+Utils.cloneProperties 0 0 0 0.2 4167.0 42.7X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Small Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 0 0 0 0.0 250416.0 1.0X
-Utils.cloneProperties 0 0 0 0.1 7914.0 31.6X
+SerializationUtils.clone 0 0 0 0.0 249586.0 1.0X
+Utils.cloneProperties 0 0 0 0.1 7614.0 32.8X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Medium Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 1 1 0 0.0 853339.0 1.0X
-Utils.cloneProperties 0 0 0 0.0 39814.0 21.4X
+SerializationUtils.clone 1 1 0 0.0 862953.0 1.0X
+Utils.cloneProperties 0 0 0 0.0 38932.0 22.2X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Large Properties: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-SerializationUtils.clone 2 2 0 0.0 1604705.0 1.0X
-Utils.cloneProperties 0 0 0 0.0 78226.0 20.5X
+SerializationUtils.clone 2 2 0 0.0 1613866.0 1.0X
+Utils.cloneProperties 0 0 0 0.0 78046.0 20.7X
diff --git a/core/benchmarks/RocksDBBenchmark-jdk21-results.txt b/core/benchmarks/RocksDBBenchmark-jdk21-results.txt
new file mode 100644
index 0000000000000..72342104c1704
--- /dev/null
+++ b/core/benchmarks/RocksDBBenchmark-jdk21-results.txt
@@ -0,0 +1,102 @@
+================================================================================================
+RocksDB Lifecycle Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+RocksDB Lifecycle Operations: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+DB Creation 6 8 2 0.0 5683288.0 1.0X
+DB Close 0 0 0 0.0 151425.0 37.5X
+
+
+================================================================================================
+Sequential Operations Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Writes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 23 24 1 0.0 22705.3 1.0X
+No Index 18 18 0 0.1 17420.4 1.3X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Updates: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 22 23 0 0.0 21721.3 1.0X
+No Index 11 11 0 0.1 10447.7 2.1X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Deletes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 11 12 0 0.1 11053.0 1.0X
+No Index 8 8 0 0.1 7709.9 1.4X
+
+
+================================================================================================
+Random Operations Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Writes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 24 25 1 0.0 23404.3 1.0X
+No Index 18 19 0 0.1 17619.8 1.3X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Updates: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 23 24 0 0.0 22456.5 1.0X
+No Index 11 11 0 0.1 10427.0 2.2X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Deletes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 12 12 0 0.1 11272.5 1.0X
+No Index 8 8 1 0.1 7679.9 1.5X
+
+
+================================================================================================
+Natural Index Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Natural Index - Create Iterator: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 0 0 0 1053.5 0.9 1.0X
+Descending 0 0 0 1012.9 1.0 1.0X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Natural Index - Iteration: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 1 1 0 0.8 1292.4 1.0X
+Descending 1 2 0 0.8 1298.7 1.0X
+
+
+================================================================================================
+Ref Index Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Ref Index - Create Iterator: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 0 0 0 725.2 1.4 1.0X
+Descending 0 0 0 710.1 1.4 1.0X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Ref Index - Iteration: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 3 3 0 0.4 2621.7 1.0X
+Descending 3 3 1 0.4 2635.0 1.0X
+
+
diff --git a/core/benchmarks/RocksDBBenchmark-results.txt b/core/benchmarks/RocksDBBenchmark-results.txt
new file mode 100644
index 0000000000000..52865acf97273
--- /dev/null
+++ b/core/benchmarks/RocksDBBenchmark-results.txt
@@ -0,0 +1,102 @@
+================================================================================================
+RocksDB Lifecycle Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+RocksDB Lifecycle Operations: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+DB Creation 5 6 1 0.0 5171553.0 1.0X
+DB Close 0 0 0 0.0 151443.0 34.1X
+
+
+================================================================================================
+Sequential Operations Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Writes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 24 25 2 0.0 23602.9 1.0X
+No Index 19 19 0 0.1 18194.7 1.3X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Updates: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 23 23 1 0.0 22149.7 1.0X
+No Index 11 11 0 0.1 10640.2 2.1X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Sequential Deletes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 12 12 0 0.1 11319.0 1.0X
+No Index 8 9 0 0.1 8044.0 1.4X
+
+
+================================================================================================
+Random Operations Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Writes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 25 25 1 0.0 24001.2 1.0X
+No Index 19 19 1 0.1 18206.3 1.3X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Updates: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 23 23 1 0.0 22268.3 1.0X
+No Index 11 11 1 0.1 10525.2 2.1X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Random Deletes: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Indexed 12 12 0 0.1 11377.6 1.0X
+No Index 8 9 0 0.1 8007.0 1.4X
+
+
+================================================================================================
+Natural Index Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Natural Index - Create Iterator: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 0 0 0 946.4 1.1 1.0X
+Descending 0 0 0 896.7 1.1 0.9X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Natural Index - Iteration: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 1 1 0 0.7 1419.1 1.0X
+Descending 1 2 0 0.7 1412.8 1.0X
+
+
+================================================================================================
+Ref Index Benchmark
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Ref Index - Create Iterator: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 0 0 0 912.7 1.1 1.0X
+Descending 0 0 0 881.2 1.1 1.0X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Ref Index - Iteration: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Ascending 3 3 0 0.4 2633.6 1.0X
+Descending 3 3 1 0.4 2643.5 1.0X
+
+
diff --git a/core/benchmarks/SorterBenchmark-jdk21-results.txt b/core/benchmarks/SorterBenchmark-jdk21-results.txt
index 91b725a0b493b..072a30a8a8815 100644
--- a/core/benchmarks/SorterBenchmark-jdk21-results.txt
+++ b/core/benchmarks/SorterBenchmark-jdk21-results.txt
@@ -2,25 +2,28 @@
key-value pairs sort
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
key-value pairs sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Tuple-sort using Arrays.sort() 17292 17584 413 1.4 691.7 1.0X
-KV-sort using Sorter 21245 21340 134 1.2 849.8 0.8X
+Tuple-sort using Arrays.sort() 16337 16917 821 1.5 653.5 1.0X
+Tuple-sort using Arrays.parallelSort() 8220 9097 1240 3.0 328.8 2.0X
+KV-sort using Sorter 19974 20183 295 1.3 799.0 0.8X
================================================================================================
primitive int array sort
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
-primitive int array sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------
-Java Arrays.sort() on non-primitive int array 14231 14561 468 1.8 569.2 1.0X
-Java Arrays.sort() on primitive int array 2141 2145 6 11.7 85.6 6.6X
-Sorter without key reuse on primitive int array 8607 8612 7 2.9 344.3 1.7X
-Sorter with key reuse on primitive int array 10602 10621 27 2.4 424.1 1.3X
+primitive int array sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+-------------------------------------------------------------------------------------------------------------------------------------
+Java Arrays.sort() on non-primitive int array 14132 14508 532 1.8 565.3 1.0X
+Java Arrays.parallelSort() on non-primitive int array 6251 6572 453 4.0 250.1 2.3X
+Java Arrays.sort() on primitive int array 2123 2123 0 11.8 84.9 6.7X
+Java Arrays.parallelSort() on primitive int array 823 885 54 30.4 32.9 17.2X
+Sorter without key reuse on primitive int array 8532 8546 18 2.9 341.3 1.7X
+Sorter with key reuse on primitive int array 10609 10615 9 2.4 424.4 1.3X
diff --git a/core/benchmarks/SorterBenchmark-results.txt b/core/benchmarks/SorterBenchmark-results.txt
index 6369727a85ca6..a00cbb61ac804 100644
--- a/core/benchmarks/SorterBenchmark-results.txt
+++ b/core/benchmarks/SorterBenchmark-results.txt
@@ -2,25 +2,28 @@
key-value pairs sort
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
key-value pairs sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Tuple-sort using Arrays.sort() 20545 20560 21 1.2 821.8 1.0X
-KV-sort using Sorter 26696 26797 143 0.9 1067.8 0.8X
+Tuple-sort using Arrays.sort() 17269 17567 421 1.4 690.8 1.0X
+Tuple-sort using Arrays.parallelSort() 8155 9560 1988 3.1 326.2 2.1X
+KV-sort using Sorter 20713 20875 230 1.2 828.5 0.8X
================================================================================================
primitive int array sort
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
-primitive int array sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------
-Java Arrays.sort() on non-primitive int array 15325 15510 263 1.6 613.0 1.0X
-Java Arrays.sort() on primitive int array 2088 2141 76 12.0 83.5 7.3X
-Sorter without key reuse on primitive int array 8254 8262 10 3.0 330.2 1.9X
-Sorter with key reuse on primitive int array 10184 10186 3 2.5 407.3 1.5X
+primitive int array sort 25000000: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+-------------------------------------------------------------------------------------------------------------------------------------
+Java Arrays.sort() on non-primitive int array 15599 15631 45 1.6 624.0 1.0X
+Java Arrays.parallelSort() on non-primitive int array 8045 8066 30 3.1 321.8 1.9X
+Java Arrays.sort() on primitive int array 2174 2176 2 11.5 87.0 7.2X
+Java Arrays.parallelSort() on primitive int array 771 782 18 32.4 30.8 20.2X
+Sorter without key reuse on primitive int array 8380 8397 25 3.0 335.2 1.9X
+Sorter with key reuse on primitive int array 10176 10183 11 2.5 407.0 1.5X
diff --git a/core/benchmarks/XORShiftRandomBenchmark-jdk21-results.txt b/core/benchmarks/XORShiftRandomBenchmark-jdk21-results.txt
index faeb60e44af10..a7e03aa0c450c 100644
--- a/core/benchmarks/XORShiftRandomBenchmark-jdk21-results.txt
+++ b/core/benchmarks/XORShiftRandomBenchmark-jdk21-results.txt
@@ -2,43 +2,43 @@
Pseudo random
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextInt: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 453 453 0 220.7 4.5 1.0X
-XORShiftRandom 186 186 0 536.7 1.9 2.4X
+java.util.Random 453 454 0 220.5 4.5 1.0X
+XORShiftRandom 186 186 0 536.5 1.9 2.4X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextLong: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 901 901 0 111.0 9.0 1.0X
-XORShiftRandom 373 373 0 268.2 3.7 2.4X
+java.util.Random 901 901 0 110.9 9.0 1.0X
+XORShiftRandom 373 373 0 268.1 3.7 2.4X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextDouble: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
java.util.Random 905 905 0 110.5 9.0 1.0X
-XORShiftRandom 373 375 3 268.2 3.7 2.4X
+XORShiftRandom 373 373 0 268.1 3.7 2.4X
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextGaussian: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 3425 3428 3 29.2 34.3 1.0X
-XORShiftRandom 2487 2495 9 40.2 24.9 1.4X
+java.util.Random 3440 3441 1 29.1 34.4 1.0X
+XORShiftRandom 2467 2467 0 40.5 24.7 1.4X
================================================================================================
hash seed
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Hash seed: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-XORShiftRandom.hashSeed 1 1 0 12973.9 0.1 1.0X
+XORShiftRandom.hashSeed 1 1 0 12851.4 0.1 1.0X
diff --git a/core/benchmarks/XORShiftRandomBenchmark-results.txt b/core/benchmarks/XORShiftRandomBenchmark-results.txt
index 93f9adebb1e44..7f51cfc9f5818 100644
--- a/core/benchmarks/XORShiftRandomBenchmark-results.txt
+++ b/core/benchmarks/XORShiftRandomBenchmark-results.txt
@@ -2,43 +2,43 @@
Pseudo random
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextInt: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 441 441 0 226.6 4.4 1.0X
-XORShiftRandom 186 186 0 536.7 1.9 2.4X
+java.util.Random 441 442 0 226.5 4.4 1.0X
+XORShiftRandom 186 186 0 536.8 1.9 2.4X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextLong: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 913 914 0 109.5 9.1 1.0X
-XORShiftRandom 373 373 0 268.4 3.7 2.5X
+java.util.Random 918 918 0 109.0 9.2 1.0X
+XORShiftRandom 373 373 0 268.3 3.7 2.5X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextDouble: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 909 910 1 110.0 9.1 1.0X
-XORShiftRandom 373 373 0 268.3 3.7 2.4X
+java.util.Random 912 916 7 109.7 9.1 1.0X
+XORShiftRandom 373 373 0 268.4 3.7 2.4X
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
nextGaussian: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-java.util.Random 4031 4035 3 24.8 40.3 1.0X
-XORShiftRandom 2952 2965 19 33.9 29.5 1.4X
+java.util.Random 4021 4021 0 24.9 40.2 1.0X
+XORShiftRandom 2942 2942 1 34.0 29.4 1.4X
================================================================================================
hash seed
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Hash seed: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-XORShiftRandom.hashSeed 2 2 0 6455.0 0.2 1.0X
+XORShiftRandom.hashSeed 2 2 0 6452.9 0.2 1.0X
diff --git a/core/benchmarks/ZStandardBenchmark-jdk21-results.txt b/core/benchmarks/ZStandardBenchmark-jdk21-results.txt
index 41993e9cfa479..93a9b0995fee5 100644
--- a/core/benchmarks/ZStandardBenchmark-jdk21-results.txt
+++ b/core/benchmarks/ZStandardBenchmark-jdk21-results.txt
@@ -2,48 +2,81 @@
Benchmark ZStandardCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------------
-Compression 10000 times at level 1 without buffer pool 657 673 15 0.0 65664.9 1.0X
-Compression 10000 times at level 2 without buffer pool 718 726 13 0.0 71830.9 0.9X
-Compression 10000 times at level 3 without buffer pool 815 819 7 0.0 81453.1 0.8X
-Compression 10000 times at level 1 with buffer pool 598 600 1 0.0 59809.3 1.1X
-Compression 10000 times at level 2 with buffer pool 637 639 2 0.0 63710.0 1.0X
-Compression 10000 times at level 3 with buffer pool 754 757 3 0.0 75403.8 0.9X
-
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+Compression 10000 times at level 1 without buffer pool 646 665 13 0.0 64646.5 1.0X
+Compression 10000 times at level 2 without buffer pool 706 706 1 0.0 70582.5 0.9X
+Compression 10000 times at level 3 without buffer pool 802 809 9 0.0 80238.3 0.8X
+Compression 10000 times at level 1 with buffer pool 595 596 1 0.0 59540.8 1.1X
+Compression 10000 times at level 2 with buffer pool 627 628 1 0.0 62663.7 1.0X
+Compression 10000 times at level 3 with buffer pool 736 750 13 0.0 73612.1 0.9X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------------------
-Decompression 10000 times from level 1 without buffer pool 832 835 2 0.0 83247.0 1.0X
-Decompression 10000 times from level 2 without buffer pool 833 840 8 0.0 83277.8 1.0X
-Decompression 10000 times from level 3 without buffer pool 833 833 0 0.0 83280.6 1.0X
-Decompression 10000 times from level 1 with buffer pool 753 755 2 0.0 75333.7 1.1X
-Decompression 10000 times from level 2 with buffer pool 751 752 1 0.0 75115.3 1.1X
-Decompression 10000 times from level 3 with buffer pool 753 754 2 0.0 75254.0 1.1X
-
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+Decompression 10000 times from level 1 without buffer pool 831 832 0 0.0 83137.4 1.0X
+Decompression 10000 times from level 2 without buffer pool 833 838 7 0.0 83253.4 1.0X
+Decompression 10000 times from level 3 without buffer pool 831 832 1 0.0 83113.3 1.0X
+Decompression 10000 times from level 1 with buffer pool 764 765 1 0.0 76368.9 1.1X
+Decompression 10000 times from level 2 with buffer pool 763 764 0 0.0 76313.5 1.1X
+Decompression 10000 times from level 3 with buffer pool 763 764 0 0.0 76331.2 1.1X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 77 78 1 0.0 601317.6 1.0X
-Parallel Compression with 1 workers 64 70 4 0.0 499456.2 1.2X
-Parallel Compression with 2 workers 53 56 3 0.0 410610.6 1.5X
-Parallel Compression with 4 workers 45 47 1 0.0 350847.8 1.7X
-Parallel Compression with 8 workers 47 49 1 0.0 370647.8 1.6X
-Parallel Compression with 16 workers 50 52 1 0.0 390524.8 1.5X
-
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+Parallel Compression with 0 workers 65 66 1 0.0 506864.4 1.0X
+Parallel Compression with 1 workers 55 57 2 0.0 427674.1 1.2X
+Parallel Compression with 2 workers 45 47 1 0.0 352972.0 1.4X
+Parallel Compression with 4 workers 41 42 1 0.0 316529.7 1.6X
+Parallel Compression with 8 workers 43 45 1 0.0 336022.6 1.5X
+Parallel Compression with 16 workers 46 51 2 0.0 358955.3 1.4X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 243 244 2 0.0 1894990.4 1.0X
-Parallel Compression with 1 workers 299 300 2 0.0 2335128.6 0.8X
-Parallel Compression with 2 workers 167 177 11 0.0 1308212.2 1.4X
-Parallel Compression with 4 workers 161 163 3 0.0 1254638.0 1.5X
-Parallel Compression with 8 workers 166 170 4 0.0 1299104.0 1.5X
-Parallel Compression with 16 workers 167 170 2 0.0 1301666.4 1.5X
+Parallel Compression with 0 workers 233 234 1 0.0 1820726.1 1.0X
+Parallel Compression with 1 workers 252 254 2 0.0 1968958.3 0.9X
+Parallel Compression with 2 workers 139 145 7 0.0 1086297.3 1.7X
+Parallel Compression with 4 workers 130 134 2 0.0 1019245.3 1.8X
+Parallel Compression with 8 workers 133 139 3 0.0 1041655.7 1.7X
+Parallel Compression with 16 workers 134 138 2 0.0 1049368.5 1.7X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Compression at level 1: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 53 54 1 0.0 410474.6 1.0X
+Compression by strategy 1 52 54 1 0.0 409782.1 1.0X
+Compression by strategy 3 87 88 1 0.0 677298.8 0.6X
+Compression by strategy 5 232 233 1 0.0 1809977.4 0.2X
+Compression by strategy 7 251 252 1 0.0 1959523.1 0.2X
+Compression by strategy 9 251 252 1 0.0 1958592.9 0.2X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 65 66 1 0.0 504323.2 1.0X
+Compression by strategy 1 49 50 1 0.0 381367.0 1.3X
+Compression by strategy 3 85 87 1 0.0 666905.7 0.8X
+Compression by strategy 5 232 234 1 0.0 1816082.8 0.3X
+Compression by strategy 7 270 272 1 0.0 2110571.7 0.2X
+Compression by strategy 9 270 271 1 0.0 2108787.3 0.2X
+
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 234 235 1 0.0 1824805.4 1.0X
+Compression by strategy 1 49 51 1 0.0 381728.3 4.8X
+Compression by strategy 3 86 87 1 0.0 670661.9 2.7X
+Compression by strategy 5 235 235 1 0.0 1832255.5 1.0X
+Compression by strategy 7 272 273 1 0.0 2125776.3 0.9X
+Compression by strategy 9 272 273 1 0.0 2122237.8 0.9X
diff --git a/core/benchmarks/ZStandardBenchmark-results.txt b/core/benchmarks/ZStandardBenchmark-results.txt
index 66aa149515a2e..0a8f611d4d666 100644
--- a/core/benchmarks/ZStandardBenchmark-results.txt
+++ b/core/benchmarks/ZStandardBenchmark-results.txt
@@ -2,48 +2,81 @@
Benchmark ZStandardCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------------
-Compression 10000 times at level 1 without buffer pool 651 654 3 0.0 65120.6 1.0X
-Compression 10000 times at level 2 without buffer pool 701 702 1 0.0 70069.5 0.9X
-Compression 10000 times at level 3 without buffer pool 788 790 2 0.0 78845.5 0.8X
-Compression 10000 times at level 1 with buffer pool 577 578 2 0.0 57688.1 1.1X
-Compression 10000 times at level 2 with buffer pool 607 608 1 0.0 60653.3 1.1X
-Compression 10000 times at level 3 with buffer pool 716 717 0 0.0 71634.8 0.9X
-
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+Compression 10000 times at level 1 without buffer pool 265 267 1 0.0 26513.9 1.0X
+Compression 10000 times at level 2 without buffer pool 298 300 1 0.0 29837.5 0.9X
+Compression 10000 times at level 3 without buffer pool 809 809 0 0.0 80903.8 0.3X
+Compression 10000 times at level 1 with buffer pool 578 579 0 0.0 57809.5 0.5X
+Compression 10000 times at level 2 with buffer pool 609 609 1 0.0 60892.6 0.4X
+Compression 10000 times at level 3 with buffer pool 717 728 9 0.0 71748.2 0.4X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------------------------
-Decompression 10000 times from level 1 without buffer pool 621 621 0 0.0 62063.9 1.0X
-Decompression 10000 times from level 2 without buffer pool 620 621 1 0.0 62003.0 1.0X
-Decompression 10000 times from level 3 without buffer pool 620 622 1 0.0 62017.4 1.0X
-Decompression 10000 times from level 1 with buffer pool 544 544 0 0.0 54377.6 1.1X
-Decompression 10000 times from level 2 with buffer pool 544 544 1 0.0 54404.3 1.1X
-Decompression 10000 times from level 3 with buffer pool 543 544 1 0.0 54310.2 1.1X
-
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+Decompression 10000 times from level 1 without buffer pool 592 595 4 0.0 59245.4 1.0X
+Decompression 10000 times from level 2 without buffer pool 592 594 1 0.0 59209.6 1.0X
+Decompression 10000 times from level 3 without buffer pool 593 593 0 0.0 59309.5 1.0X
+Decompression 10000 times from level 1 with buffer pool 540 541 0 0.0 54039.4 1.1X
+Decompression 10000 times from level 2 with buffer pool 541 541 0 0.0 54058.9 1.1X
+Decompression 10000 times from level 3 with buffer pool 540 541 0 0.0 54016.9 1.1X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 77 79 1 0.0 603481.7 1.0X
-Parallel Compression with 1 workers 65 68 3 0.0 504835.7 1.2X
-Parallel Compression with 2 workers 50 55 3 0.0 393366.7 1.5X
-Parallel Compression with 4 workers 45 48 2 0.0 352491.0 1.7X
-Parallel Compression with 8 workers 46 49 1 0.0 362934.5 1.7X
-Parallel Compression with 16 workers 50 53 1 0.0 393219.4 1.5X
-
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+Parallel Compression with 0 workers 65 66 0 0.0 509528.4 1.0X
+Parallel Compression with 1 workers 55 58 3 0.0 426851.9 1.2X
+Parallel Compression with 2 workers 44 46 1 0.0 341996.8 1.5X
+Parallel Compression with 4 workers 41 42 1 0.0 322444.2 1.6X
+Parallel Compression with 8 workers 43 45 1 0.0 336123.3 1.5X
+Parallel Compression with 16 workers 46 49 1 0.0 359754.8 1.4X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 237 239 1 0.0 1852985.0 1.0X
-Parallel Compression with 1 workers 299 301 2 0.0 2339270.2 0.8X
-Parallel Compression with 2 workers 167 174 6 0.0 1303651.0 1.4X
-Parallel Compression with 4 workers 160 165 3 0.0 1248593.3 1.5X
-Parallel Compression with 8 workers 166 170 3 0.0 1295084.3 1.4X
-Parallel Compression with 16 workers 165 169 3 0.0 1288571.2 1.4X
+Parallel Compression with 0 workers 228 229 1 0.0 1780109.3 1.0X
+Parallel Compression with 1 workers 247 249 3 0.0 1927634.1 0.9X
+Parallel Compression with 2 workers 140 146 8 0.0 1094789.1 1.6X
+Parallel Compression with 4 workers 130 133 2 0.0 1013913.1 1.8X
+Parallel Compression with 8 workers 136 138 2 0.0 1058972.8 1.7X
+Parallel Compression with 16 workers 135 139 2 0.0 1053766.4 1.7X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Compression at level 1: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 54 54 0 0.0 418444.8 1.0X
+Compression by strategy 1 53 54 0 0.0 417570.9 1.0X
+Compression by strategy 3 88 88 0 0.0 684663.0 0.6X
+Compression by strategy 5 228 229 0 0.0 1784827.6 0.2X
+Compression by strategy 7 256 256 0 0.0 1998418.9 0.2X
+Compression by strategy 9 256 256 0 0.0 1997712.8 0.2X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 65 66 1 0.0 509229.4 1.0X
+Compression by strategy 1 49 50 1 0.0 383259.2 1.3X
+Compression by strategy 3 86 87 0 0.0 674150.1 0.8X
+Compression by strategy 5 227 228 0 0.0 1777012.2 0.3X
+Compression by strategy 7 274 274 0 0.0 2137945.7 0.2X
+Compression by strategy 9 274 275 0 0.0 2141819.7 0.2X
+
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
+AMD EPYC 7763 64-Core Processor
+Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+Compression by strategy -1 228 230 1 0.0 1781070.7 1.0X
+Compression by strategy 1 50 50 1 0.0 387332.5 4.6X
+Compression by strategy 3 86 87 1 0.0 674742.4 2.6X
+Compression by strategy 5 228 229 0 0.0 1783581.6 1.0X
+Compression by strategy 7 275 276 1 0.0 2151470.1 0.8X
+Compression by strategy 9 275 276 1 0.0 2150367.6 0.8X
diff --git a/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt b/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt
index 58be348a69c9b..5e1af70caa2c5 100644
--- a/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt
+++ b/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt
@@ -2,48 +2,48 @@
Benchmark ZStandardCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------------
-Compression 4 times at level 1 without buffer pool 2852 2858 10 0.0 712887837.3 1.0X
-Compression 4 times at level 2 without buffer pool 4417 4418 1 0.0 1104333096.8 0.6X
-Compression 4 times at level 3 without buffer pool 6594 6602 12 0.0 1648568689.0 0.4X
-Compression 4 times at level 1 with buffer pool 2875 2876 2 0.0 718686835.3 1.0X
-Compression 4 times at level 2 with buffer pool 4415 4422 10 0.0 1103705078.0 0.6X
-Compression 4 times at level 3 with buffer pool 6749 6774 35 0.0 1687199785.0 0.4X
+Compression 4 times at level 1 without buffer pool 2764 2764 1 0.0 690899114.5 1.0X
+Compression 4 times at level 2 without buffer pool 4363 4381 25 0.0 1090808911.2 0.6X
+Compression 4 times at level 3 without buffer pool 6602 6604 2 0.0 1650543838.8 0.4X
+Compression 4 times at level 1 with buffer pool 2749 2750 2 0.0 687137810.8 1.0X
+Compression 4 times at level 2 with buffer pool 4439 4444 8 0.0 1109656370.0 0.6X
+Compression 4 times at level 3 with buffer pool 6561 6579 25 0.0 1640364047.0 0.4X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------------
-Decompression 4 times from level 1 without buffer pool 1084 1140 79 0.0 271079599.8 1.0X
-Decompression 4 times from level 2 without buffer pool 1341 1363 31 0.0 335200382.3 0.8X
-Decompression 4 times from level 3 without buffer pool 1620 1632 18 0.0 404881883.0 0.7X
-Decompression 4 times from level 1 with buffer pool 1110 1144 48 0.0 277478874.5 1.0X
-Decompression 4 times from level 2 with buffer pool 1306 1363 80 0.0 326419359.8 0.8X
-Decompression 4 times from level 3 with buffer pool 1566 1585 27 0.0 391420983.5 0.7X
+Decompression 4 times from level 1 without buffer pool 1026 1091 92 0.0 256389848.5 1.0X
+Decompression 4 times from level 2 without buffer pool 1344 1377 48 0.0 335901373.5 0.8X
+Decompression 4 times from level 3 without buffer pool 1556 1610 76 0.0 388926978.3 0.7X
+Decompression 4 times from level 1 with buffer pool 1028 1079 72 0.0 257022260.0 1.0X
+Decompression 4 times from level 2 with buffer pool 1328 1356 40 0.0 331990447.8 0.8X
+Decompression 4 times from level 3 with buffer pool 1596 1652 80 0.0 398930929.8 0.6X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 2160 2166 9 0.0 539942027.3 1.0X
-Parallel Compression with 1 workers 2245 2248 4 0.0 561232751.0 1.0X
-Parallel Compression with 2 workers 1180 1183 4 0.0 295021164.8 1.8X
-Parallel Compression with 4 workers 832 840 12 0.0 207891005.5 2.6X
-Parallel Compression with 8 workers 971 1002 44 0.0 242832737.3 2.2X
-Parallel Compression with 16 workers 1329 1365 51 0.0 332270317.7 1.6X
+Parallel Compression with 0 workers 2080 2092 18 0.0 519889128.5 1.0X
+Parallel Compression with 1 workers 2203 2208 6 0.0 550838598.0 0.9X
+Parallel Compression with 2 workers 1155 1170 21 0.0 288752087.0 1.8X
+Parallel Compression with 4 workers 826 839 11 0.0 206623975.8 2.5X
+Parallel Compression with 8 workers 872 893 18 0.0 218002184.8 2.4X
+Parallel Compression with 16 workers 1225 1256 44 0.0 306297206.5 1.7X
-OpenJDK 64-Bit Server VM 21.0.7+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 9168 9276 154 0.0 2291883601.5 1.0X
-Parallel Compression with 1 workers 8142 8244 145 0.0 2035422527.5 1.1X
-Parallel Compression with 2 workers 4703 4939 334 0.0 1175760328.8 1.9X
-Parallel Compression with 4 workers 4366 4426 84 0.0 1091576971.0 2.1X
-Parallel Compression with 8 workers 5395 5716 453 0.0 1348844072.5 1.7X
-Parallel Compression with 16 workers 6410 6573 230 0.0 1602434231.8 1.4X
+Parallel Compression with 0 workers 9201 9228 37 0.0 2300372887.8 1.0X
+Parallel Compression with 1 workers 7690 7852 229 0.0 1922482505.8 1.2X
+Parallel Compression with 2 workers 4272 4317 63 0.0 1068103607.8 2.2X
+Parallel Compression with 4 workers 3675 3685 14 0.0 918764298.5 2.5X
+Parallel Compression with 8 workers 4731 4762 44 0.0 1182798359.8 1.9X
+Parallel Compression with 16 workers 5834 5851 24 0.0 1458440232.3 1.6X
diff --git a/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt b/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt
index cc439b099a6c5..b7afea04fbc88 100644
--- a/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt
+++ b/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt
@@ -2,48 +2,48 @@
Benchmark ZStandardCompressionCodec
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
----------------------------------------------------------------------------------------------------------------------------------
-Compression 4 times at level 1 without buffer pool 2582 2583 1 0.0 645586401.5 1.0X
-Compression 4 times at level 2 without buffer pool 4128 4132 6 0.0 1032017030.8 0.6X
-Compression 4 times at level 3 without buffer pool 6222 6229 11 0.0 1555453651.0 0.4X
-Compression 4 times at level 1 with buffer pool 2581 2581 1 0.0 645260104.3 1.0X
-Compression 4 times at level 2 with buffer pool 4123 4129 9 0.0 1030789387.7 0.6X
-Compression 4 times at level 3 with buffer pool 6290 6299 13 0.0 1572478346.5 0.4X
+Compression 4 times at level 1 without buffer pool 2554 2556 3 0.0 638404691.8 1.0X
+Compression 4 times at level 2 without buffer pool 4160 4160 1 0.0 1039958809.8 0.6X
+Compression 4 times at level 3 without buffer pool 6259 6272 19 0.0 1564689762.0 0.4X
+Compression 4 times at level 1 with buffer pool 2554 2555 1 0.0 638584854.0 1.0X
+Compression 4 times at level 2 with buffer pool 4173 4174 1 0.0 1043297333.7 0.6X
+Compression 4 times at level 3 with buffer pool 6292 6296 5 0.0 1573072300.0 0.4X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
--------------------------------------------------------------------------------------------------------------------------------------
-Decompression 4 times from level 1 without buffer pool 898 902 4 0.0 224453665.0 1.0X
-Decompression 4 times from level 2 without buffer pool 1170 1174 5 0.0 292566849.8 0.8X
-Decompression 4 times from level 3 without buffer pool 1409 1411 2 0.0 352317771.3 0.6X
-Decompression 4 times from level 1 with buffer pool 900 901 2 0.0 224956421.3 1.0X
-Decompression 4 times from level 2 with buffer pool 1169 1174 7 0.0 292235703.0 0.8X
-Decompression 4 times from level 3 with buffer pool 1400 1403 3 0.0 350101054.0 0.6X
+Decompression 4 times from level 1 without buffer pool 899 913 23 0.0 224716523.7 1.0X
+Decompression 4 times from level 2 without buffer pool 1163 1168 6 0.0 290860752.3 0.8X
+Decompression 4 times from level 3 without buffer pool 1392 1392 0 0.0 347912543.8 0.6X
+Decompression 4 times from level 1 with buffer pool 890 894 3 0.0 222572354.0 1.0X
+Decompression 4 times from level 2 with buffer pool 1157 1161 6 0.0 289278691.8 0.8X
+Decompression 4 times from level 3 with buffer pool 1383 1387 6 0.0 345816264.3 0.6X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 2088 2098 15 0.0 521933319.5 1.0X
-Parallel Compression with 1 workers 2142 2145 4 0.0 535401971.0 1.0X
-Parallel Compression with 2 workers 1086 1090 5 0.0 271554405.7 1.9X
-Parallel Compression with 4 workers 789 799 10 0.0 197186788.0 2.6X
-Parallel Compression with 8 workers 930 951 18 0.0 232624325.3 2.2X
-Parallel Compression with 16 workers 1011 1014 5 0.0 252634970.3 2.1X
+Parallel Compression with 0 workers 2095 2095 0 0.0 523730144.5 1.0X
+Parallel Compression with 1 workers 2117 2117 0 0.0 529351910.2 1.0X
+Parallel Compression with 2 workers 1084 1085 0 0.0 271065080.0 1.9X
+Parallel Compression with 4 workers 754 759 4 0.0 188485738.0 2.8X
+Parallel Compression with 8 workers 784 787 6 0.0 195887107.7 2.7X
+Parallel Compression with 16 workers 882 897 13 0.0 220488661.0 2.4X
-OpenJDK 64-Bit Server VM 17.0.15+6-LTS on Linux 6.11.0-1014-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-Parallel Compression with 0 workers 8766 8812 64 0.0 2191521645.8 1.0X
-Parallel Compression with 1 workers 7808 7810 3 0.0 1951970435.0 1.1X
-Parallel Compression with 2 workers 4057 4096 56 0.0 1014125964.5 2.2X
-Parallel Compression with 4 workers 3508 3528 28 0.0 877027867.8 2.5X
-Parallel Compression with 8 workers 4026 4028 2 0.0 1006539516.7 2.2X
-Parallel Compression with 16 workers 4044 4052 11 0.0 1011070885.5 2.2X
+Parallel Compression with 0 workers 8758 8789 45 0.0 2189469160.5 1.0X
+Parallel Compression with 1 workers 7089 7118 41 0.0 1772262320.5 1.2X
+Parallel Compression with 2 workers 3669 3674 8 0.0 917169770.5 2.4X
+Parallel Compression with 4 workers 3290 3296 9 0.0 822425053.8 2.7X
+Parallel Compression with 8 workers 3657 3685 40 0.0 914190198.7 2.4X
+Parallel Compression with 16 workers 3804 3815 15 0.0 951081198.7 2.3X
diff --git a/core/src/main/java/org/apache/spark/api/java/Optional.java b/core/src/main/java/org/apache/spark/api/java/Optional.java
index 362149c92145e..913ed966668d4 100644
--- a/core/src/main/java/org/apache/spark/api/java/Optional.java
+++ b/core/src/main/java/org/apache/spark/api/java/Optional.java
@@ -20,8 +20,6 @@
import java.io.Serializable;
import java.util.Objects;
-import com.google.common.base.Preconditions;
-
/**
* Like {@code java.util.Optional} in Java 8, {@code scala.Option} in Scala, and
* {@code com.google.common.base.Optional} in Google Guava, this class represents a
@@ -71,8 +69,7 @@ private Optional() {
}
private Optional(T value) {
- Preconditions.checkNotNull(value);
- this.value = value;
+ this.value = Objects.requireNonNull(value);
}
// java.util.Optional API (subset)
@@ -112,8 +109,7 @@ public static Optional ofNullable(T value) {
* @throws NullPointerException if this is empty (contains no value)
*/
public T get() {
- Preconditions.checkNotNull(value);
- return value;
+ return Objects.requireNonNull(value);
}
/**
diff --git a/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java b/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java
index 7dd87df713e6e..91ee2d8ccbe96 100644
--- a/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java
+++ b/core/src/main/java/org/apache/spark/io/ReadAheadInputStream.java
@@ -28,13 +28,11 @@
import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.concurrent.GuardedBy;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-
import org.apache.spark.internal.SparkLogger;
import org.apache.spark.internal.SparkLoggerFactory;
import org.apache.spark.internal.LogKeys;
import org.apache.spark.internal.MDC;
+import org.apache.spark.network.util.JavaUtils;
import org.apache.spark.util.ThreadUtils;
/**
@@ -105,7 +103,7 @@ public class ReadAheadInputStream extends InputStream {
*/
public ReadAheadInputStream(
InputStream inputStream, int bufferSizeInBytes) {
- Preconditions.checkArgument(bufferSizeInBytes > 0,
+ JavaUtils.checkArgument(bufferSizeInBytes > 0,
"bufferSizeInBytes should be greater than 0, but the value is " + bufferSizeInBytes);
activeBuffer = ByteBuffer.allocate(bufferSizeInBytes);
readAheadBuffer = ByteBuffer.allocate(bufferSizeInBytes);
@@ -120,8 +118,10 @@ private boolean isEndOfStream() {
private void checkReadException() throws IOException {
if (readAborted) {
- Throwables.throwIfInstanceOf(readException, IOException.class);
- Throwables.throwIfUnchecked(readException);
+ if (readException == null) throw new NullPointerException("readException is not captured.");
+ if (readException instanceof IOException ie) throw ie;
+ if (readException instanceof Error error) throw error;
+ if (readException instanceof RuntimeException re) throw re;
throw new IOException(readException);
}
}
@@ -213,7 +213,7 @@ private void closeUnderlyingInputStreamIfNecessary() {
try {
underlyingInputStream.close();
} catch (IOException e) {
- logger.warn("{}", e, MDC.of(LogKeys.ERROR$.MODULE$, e.getMessage()));
+ logger.warn("{}", e, MDC.of(LogKeys.ERROR, e.getMessage()));
}
}
}
diff --git a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
index e98554db22524..ab9e470e0c2c0 100644
--- a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
+++ b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java
@@ -279,11 +279,11 @@ private long trySpillAndAcquire(
} catch (ClosedByInterruptException | InterruptedIOException e) {
// This called by user to kill a task (e.g: speculative task).
logger.error("Error while calling spill() on {}", e,
- MDC.of(LogKeys.MEMORY_CONSUMER$.MODULE$, consumerToSpill));
+ MDC.of(LogKeys.MEMORY_CONSUMER, consumerToSpill));
throw new RuntimeException(e.getMessage());
} catch (IOException e) {
logger.error("Error while calling spill() on {}", e,
- MDC.of(LogKeys.MEMORY_CONSUMER$.MODULE$, consumerToSpill));
+ MDC.of(LogKeys.MEMORY_CONSUMER, consumerToSpill));
// checkstyle.off: RegexpSinglelineJava
throw new SparkOutOfMemoryError(
"SPILL_OUT_OF_MEMORY",
@@ -320,7 +320,7 @@ public void releaseExecutionMemory(long size, MemoryConsumer consumer) {
*/
public void showMemoryUsage() {
logger.info("Memory used in task {}",
- MDC.of(LogKeys.TASK_ATTEMPT_ID$.MODULE$, taskAttemptId));
+ MDC.of(LogKeys.TASK_ATTEMPT_ID, taskAttemptId));
synchronized (this) {
long memoryAccountedForByConsumers = 0;
for (MemoryConsumer c: consumers) {
@@ -328,20 +328,20 @@ public void showMemoryUsage() {
memoryAccountedForByConsumers += totalMemUsage;
if (totalMemUsage > 0) {
logger.info("Acquired by {}: {}",
- MDC.of(LogKeys.MEMORY_CONSUMER$.MODULE$, c),
- MDC.of(LogKeys.MEMORY_SIZE$.MODULE$, Utils.bytesToString(totalMemUsage)));
+ MDC.of(LogKeys.MEMORY_CONSUMER, c),
+ MDC.of(LogKeys.MEMORY_SIZE, Utils.bytesToString(totalMemUsage)));
}
}
long memoryNotAccountedFor =
memoryManager.getExecutionMemoryUsageForTask(taskAttemptId) - memoryAccountedForByConsumers;
logger.info(
"{} bytes of memory were used by task {} but are not associated with specific consumers",
- MDC.of(LogKeys.MEMORY_SIZE$.MODULE$, memoryNotAccountedFor),
- MDC.of(LogKeys.TASK_ATTEMPT_ID$.MODULE$, taskAttemptId));
+ MDC.of(LogKeys.MEMORY_SIZE, memoryNotAccountedFor),
+ MDC.of(LogKeys.TASK_ATTEMPT_ID, taskAttemptId));
logger.info(
"{} bytes of memory are used for execution and {} bytes of memory are used for storage",
- MDC.of(LogKeys.EXECUTION_MEMORY_SIZE$.MODULE$, memoryManager.executionMemoryUsed()),
- MDC.of(LogKeys.STORAGE_MEMORY_SIZE$.MODULE$, memoryManager.storageMemoryUsed()));
+ MDC.of(LogKeys.EXECUTION_MEMORY_SIZE, memoryManager.executionMemoryUsed()),
+ MDC.of(LogKeys.STORAGE_MEMORY_SIZE, memoryManager.storageMemoryUsed()));
}
}
@@ -388,7 +388,7 @@ public MemoryBlock allocatePage(long size, MemoryConsumer consumer) {
page = memoryManager.tungstenMemoryAllocator().allocate(acquired);
} catch (OutOfMemoryError e) {
logger.warn("Failed to allocate a page ({} bytes), try again.",
- MDC.of(LogKeys.PAGE_SIZE$.MODULE$, acquired));
+ MDC.of(LogKeys.PAGE_SIZE, acquired));
// there is no enough memory actually, it means the actual free memory is smaller than
// MemoryManager thought, we should keep the acquired memory.
synchronized (this) {
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java
index 86f7d5143eff5..8072a432ab110 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java
@@ -227,7 +227,7 @@ private long[] writePartitionedData(ShuffleMapOutputWriter mapOutputWriter) thro
}
if (!file.delete()) {
logger.error("Unable to delete file for partition {}",
- MDC.of(LogKeys.PARTITION_ID$.MODULE$, i));
+ MDC.of(LogKeys.PARTITION_ID, i));
}
}
}
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java
index de3c41a4b526b..bed241b1e03a9 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java
@@ -89,6 +89,11 @@ final class ShuffleExternalSorter extends MemoryConsumer implements ShuffleCheck
*/
private final int numElementsForSpillThreshold;
+ /**
+ * Force this sorter to spill when the size in memory is beyond this threshold.
+ */
+ private final long recordsSizeForSpillThreshold;
+
/** The buffer size to use when writing spills using DiskBlockObjectWriter */
private final int fileBufferSizeBytes;
@@ -112,6 +117,7 @@ final class ShuffleExternalSorter extends MemoryConsumer implements ShuffleCheck
@Nullable private ShuffleInMemorySorter inMemSorter;
@Nullable private MemoryBlock currentPage = null;
private long pageCursor = -1;
+ private long inMemRecordsSize = 0;
// Checksum calculator for each partition. Empty when shuffle checksum disabled.
private final Checksum[] partitionChecksums;
@@ -136,6 +142,8 @@ final class ShuffleExternalSorter extends MemoryConsumer implements ShuffleCheck
(int) (long) conf.get(package$.MODULE$.SHUFFLE_FILE_BUFFER_SIZE()) * 1024;
this.numElementsForSpillThreshold =
(int) conf.get(package$.MODULE$.SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD());
+ this.recordsSizeForSpillThreshold =
+ (long) conf.get(package$.MODULE$.SHUFFLE_SPILL_MAX_SIZE_FORCE_SPILL_THRESHOLD());
this.writeMetrics = writeMetrics;
this.inMemSorter = new ShuffleInMemorySorter(
this, initialSize, (boolean) conf.get(package$.MODULE$.SHUFFLE_SORT_USE_RADIXSORT()));
@@ -162,11 +170,11 @@ private void writeSortedFile(boolean isFinalFile) {
if (!isFinalFile) {
logger.info(
"Task {} on Thread {} spilling sort data of {} to disk ({} {} so far)",
- MDC.of(LogKeys.TASK_ATTEMPT_ID$.MODULE$, taskContext.taskAttemptId()),
- MDC.of(LogKeys.THREAD_ID$.MODULE$, Thread.currentThread().getId()),
- MDC.of(LogKeys.MEMORY_SIZE$.MODULE$, Utils.bytesToString(getMemoryUsage())),
- MDC.of(LogKeys.NUM_SPILLS$.MODULE$, spills.size()),
- MDC.of(LogKeys.SPILL_TIMES$.MODULE$, spills.size() != 1 ? "times" : "time"));
+ MDC.of(LogKeys.TASK_ATTEMPT_ID, taskContext.taskAttemptId()),
+ MDC.of(LogKeys.THREAD_ID, Thread.currentThread().getId()),
+ MDC.of(LogKeys.MEMORY_SIZE, Utils.bytesToString(getMemoryUsage())),
+ MDC.of(LogKeys.NUM_SPILLS, spills.size()),
+ MDC.of(LogKeys.SPILL_TIMES, spills.size() != 1 ? "times" : "time"));
}
// This call performs the actual sort.
@@ -338,6 +346,7 @@ private long freeMemory() {
allocatedPages.clear();
currentPage = null;
pageCursor = 0;
+ inMemRecordsSize = 0;
return memoryFreed;
}
@@ -353,7 +362,7 @@ public void cleanupResources() {
for (SpillInfo spill : spills) {
if (spill.file.exists() && !spill.file.delete()) {
logger.error("Unable to delete spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, spill.file.getPath()));
+ MDC.of(LogKeys.PATH, spill.file.getPath()));
}
}
}
@@ -417,11 +426,16 @@ private void acquireNewPageIfNecessary(int required) {
public void insertRecord(Object recordBase, long recordOffset, int length, int partitionId)
throws IOException {
- // for tests
assert(inMemSorter != null);
if (inMemSorter.numRecords() >= numElementsForSpillThreshold) {
- logger.info("Spilling data because number of spilledRecords crossed the threshold {}" +
- MDC.of(LogKeys.NUM_ELEMENTS_SPILL_THRESHOLD$.MODULE$, numElementsForSpillThreshold));
+ logger.info("Spilling data because number of spilledRecords ({}) crossed the threshold {}",
+ MDC.of(LogKeys.NUM_ELEMENTS_SPILL_RECORDS, inMemSorter.numRecords()),
+ MDC.of(LogKeys.NUM_ELEMENTS_SPILL_THRESHOLD, numElementsForSpillThreshold));
+ spill();
+ } else if (inMemRecordsSize >= recordsSizeForSpillThreshold) {
+ logger.info("Spilling data because size of spilledRecords ({}) crossed the size threshold {}",
+ MDC.of(LogKeys.SPILL_RECORDS_SIZE, inMemRecordsSize),
+ MDC.of(LogKeys.SPILL_RECORDS_SIZE_THRESHOLD, recordsSizeForSpillThreshold));
spill();
}
@@ -439,6 +453,7 @@ public void insertRecord(Object recordBase, long recordOffset, int length, int p
Platform.copyMemory(recordBase, recordOffset, base, pageCursor, length);
pageCursor += length;
inMemSorter.insertRecord(recordAddress, partitionId);
+ inMemRecordsSize += required;
}
/**
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
index ac9d335d63591..36a1487627367 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
@@ -33,7 +33,6 @@
import scala.reflect.ClassTag$;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.ByteStreams;
import com.google.common.io.Closeables;
import org.apache.spark.*;
@@ -229,7 +228,7 @@ void closeAndWriteOutput() throws IOException {
for (SpillInfo spill : spills) {
if (spill.file.exists() && !spill.file.delete()) {
logger.error("Error while deleting spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, spill.file.getPath()));
+ MDC.of(LogKeys.PATH, spill.file.getPath()));
}
}
}
@@ -404,7 +403,7 @@ private void mergeSpillsWithFileStream(
partitionInputStream = compressionCodec.compressedInputStream(
partitionInputStream);
}
- ByteStreams.copy(partitionInputStream, partitionOutput);
+ partitionInputStream.transferTo(partitionOutput);
copySpillThrewException = false;
} finally {
Closeables.close(partitionInputStream, copySpillThrewException);
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
index c0b9018c770a0..e51a48b019854 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
@@ -125,7 +125,7 @@ public void abort(Throwable error) throws IOException {
cleanUp();
if (outputTempFile != null && outputTempFile.exists() && !outputTempFile.delete()) {
log.warn("Failed to delete temporary shuffle file at {}",
- MDC.of(LogKeys.PATH$.MODULE$, outputTempFile.getAbsolutePath()));
+ MDC.of(LogKeys.PATH, outputTempFile.getAbsolutePath()));
}
}
diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 2a8e15cd09ccf..486bfd62bc97a 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -395,7 +395,7 @@ private void handleFailedDelete() {
File file = spillWriters.removeFirst().getFile();
if (file != null && file.exists() && !file.delete()) {
logger.error("Was unable to delete spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, file.getAbsolutePath()));
+ MDC.of(LogKeys.PATH, file.getAbsolutePath()));
}
}
}
@@ -897,7 +897,7 @@ public void free() {
if (file != null && file.exists()) {
if (!file.delete()) {
logger.error("Was unable to delete spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, file.getAbsolutePath()));
+ MDC.of(LogKeys.PATH, file.getAbsolutePath()));
}
}
}
diff --git a/core/src/main/java/org/apache/spark/util/EnumUtil.java b/core/src/main/java/org/apache/spark/util/EnumUtil.java
index c40c7e727613c..6c397640590fe 100644
--- a/core/src/main/java/org/apache/spark/util/EnumUtil.java
+++ b/core/src/main/java/org/apache/spark/util/EnumUtil.java
@@ -16,7 +16,8 @@
*/
package org.apache.spark.util;
-import com.google.common.base.Joiner;
+import java.util.StringJoiner;
+
import org.apache.spark.annotation.Private;
@Private
@@ -33,6 +34,14 @@ public static > E parseIgnoreCase(Class clz, String str) {
}
throw new IllegalArgumentException(
String.format("Illegal type='%s'. Supported type values: %s",
- str, Joiner.on(", ").join(constants)));
+ str, joinToString(constants)));
+ }
+
+ private static > String joinToString(E[] enums) {
+ StringJoiner stringJoiner = new StringJoiner(", ");
+ for (E e : enums) {
+ stringJoiner.add(e.name());
+ }
+ return stringJoiner.toString();
}
}
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/RadixSort.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/RadixSort.java
index 3dd318471008b..c8f984e856cef 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/RadixSort.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/RadixSort.java
@@ -17,10 +17,9 @@
package org.apache.spark.util.collection.unsafe.sort;
-import com.google.common.primitives.Ints;
-
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.array.LongArray;
+import org.apache.spark.network.util.JavaUtils;
public class RadixSort {
@@ -63,7 +62,7 @@ public static int sort(
}
}
}
- return Ints.checkedCast(inIndex);
+ return JavaUtils.checkedCast(inIndex);
}
/**
@@ -204,7 +203,7 @@ public static int sortKeyPrefixArray(
}
}
}
- return Ints.checkedCast(inIndex);
+ return JavaUtils.checkedCast(inIndex);
}
/**
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
index af421e903ba3f..94c37e187131f 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
@@ -27,7 +27,6 @@
import java.util.function.Supplier;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.io.IOUtils;
import org.apache.spark.TaskContext;
import org.apache.spark.executor.ShuffleWriteMetrics;
@@ -80,6 +79,11 @@ public final class UnsafeExternalSorter extends MemoryConsumer {
*/
private final int numElementsForSpillThreshold;
+ /**
+ * Force this sorter to spill when the size in memory is beyond this threshold.
+ */
+ private final long recordsSizeForSpillThreshold;
+
/**
* Memory pages that hold the records being sorted. The pages in this list are freed when
* spilling, although in principle we could recycle these pages across spills (on the other hand,
@@ -92,6 +96,7 @@ public final class UnsafeExternalSorter extends MemoryConsumer {
// These variables are reset after spilling:
@Nullable private volatile UnsafeInMemorySorter inMemSorter;
+ private long inMemRecordsSize = 0;
private MemoryBlock currentPage = null;
private long pageCursor = -1;
@@ -110,11 +115,13 @@ public static UnsafeExternalSorter createWithExistingInMemorySorter(
int initialSize,
long pageSizeBytes,
int numElementsForSpillThreshold,
+ long recordsSizeForSpillThreshold,
UnsafeInMemorySorter inMemorySorter,
long existingMemoryConsumption) throws IOException {
UnsafeExternalSorter sorter = new UnsafeExternalSorter(taskMemoryManager, blockManager,
serializerManager, taskContext, recordComparatorSupplier, prefixComparator, initialSize,
- pageSizeBytes, numElementsForSpillThreshold, inMemorySorter, false /* ignored */);
+ pageSizeBytes, numElementsForSpillThreshold, recordsSizeForSpillThreshold,
+ inMemorySorter, false /* ignored */);
sorter.spill(Long.MAX_VALUE, sorter);
taskContext.taskMetrics().incMemoryBytesSpilled(existingMemoryConsumption);
sorter.totalSpillBytes += existingMemoryConsumption;
@@ -133,10 +140,11 @@ public static UnsafeExternalSorter create(
int initialSize,
long pageSizeBytes,
int numElementsForSpillThreshold,
+ long recordsSizeForSpillThreshold,
boolean canUseRadixSort) {
return new UnsafeExternalSorter(taskMemoryManager, blockManager, serializerManager,
taskContext, recordComparatorSupplier, prefixComparator, initialSize, pageSizeBytes,
- numElementsForSpillThreshold, null, canUseRadixSort);
+ numElementsForSpillThreshold, recordsSizeForSpillThreshold, null, canUseRadixSort);
}
private UnsafeExternalSorter(
@@ -149,6 +157,7 @@ private UnsafeExternalSorter(
int initialSize,
long pageSizeBytes,
int numElementsForSpillThreshold,
+ long recordsSizeForSpillThreshold,
@Nullable UnsafeInMemorySorter existingInMemorySorter,
boolean canUseRadixSort) {
super(taskMemoryManager, pageSizeBytes, taskMemoryManager.getTungstenMemoryMode());
@@ -178,6 +187,7 @@ private UnsafeExternalSorter(
this.inMemSorter = existingInMemorySorter;
}
this.peakMemoryUsedBytes = getMemoryUsage();
+ this.recordsSizeForSpillThreshold = recordsSizeForSpillThreshold;
this.numElementsForSpillThreshold = numElementsForSpillThreshold;
// Register a cleanup task with TaskContext to ensure that memory is guaranteed to be freed at
@@ -220,10 +230,10 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
}
logger.info("Thread {} spilling sort data of {} to disk ({} {} so far)",
- MDC.of(LogKeys.THREAD_ID$.MODULE$, Thread.currentThread().getId()),
- MDC.of(LogKeys.MEMORY_SIZE$.MODULE$, Utils.bytesToString(getMemoryUsage())),
- MDC.of(LogKeys.NUM_SPILL_WRITERS$.MODULE$, spillWriters.size()),
- MDC.of(LogKeys.SPILL_TIMES$.MODULE$, spillWriters.size() > 1 ? "times" : "time"));
+ MDC.of(LogKeys.THREAD_ID, Thread.currentThread().getId()),
+ MDC.of(LogKeys.MEMORY_SIZE, Utils.bytesToString(getMemoryUsage())),
+ MDC.of(LogKeys.NUM_SPILL_WRITERS, spillWriters.size()),
+ MDC.of(LogKeys.SPILL_TIMES, spillWriters.size() > 1 ? "times" : "time"));
ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics();
@@ -238,6 +248,7 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
// pages will currently be counted as memory spilled even though that space isn't actually
// written to disk. This also counts the space needed to store the sorter's pointer array.
inMemSorter.freeMemory();
+ inMemRecordsSize = 0;
// Reset the in-memory sorter's pointer array only after freeing up the memory pages holding the
// records. Otherwise, if the task is over allocated memory, then without freeing the memory
// pages, we might not be able to get memory for the pointer array.
@@ -339,7 +350,7 @@ private void deleteSpillFiles() {
if (file != null && file.exists()) {
if (!file.delete()) {
logger.error("Was unable to delete spill file {}",
- MDC.of(LogKeys.PATH$.MODULE$, file.getAbsolutePath()));
+ MDC.of(LogKeys.PATH, file.getAbsolutePath()));
}
}
}
@@ -480,8 +491,14 @@ public void insertRecord(
assert(inMemSorter != null);
if (inMemSorter.numRecords() >= numElementsForSpillThreshold) {
- logger.info("Spilling data because number of spilledRecords crossed the threshold {}",
- MDC.of(LogKeys.NUM_ELEMENTS_SPILL_THRESHOLD$.MODULE$, numElementsForSpillThreshold));
+ logger.info("Spilling data because number of spilledRecords ({}) crossed the threshold {}",
+ MDC.of(LogKeys.NUM_ELEMENTS_SPILL_RECORDS, inMemSorter.numRecords()),
+ MDC.of(LogKeys.NUM_ELEMENTS_SPILL_THRESHOLD, numElementsForSpillThreshold));
+ spill();
+ } else if (inMemRecordsSize >= recordsSizeForSpillThreshold) {
+ logger.info("Spilling data because size of spilledRecords ({}) crossed the size threshold {}",
+ MDC.of(LogKeys.SPILL_RECORDS_SIZE, inMemRecordsSize),
+ MDC.of(LogKeys.SPILL_RECORDS_SIZE_THRESHOLD, recordsSizeForSpillThreshold));
spill();
}
@@ -497,6 +514,7 @@ public void insertRecord(
Platform.copyMemory(recordBase, recordOffset, base, pageCursor, length);
pageCursor += length;
inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull);
+ inMemRecordsSize += required;
}
/**
@@ -867,7 +885,7 @@ public void close() throws IOException {
private void closeIfPossible(UnsafeSorterIterator iterator) {
if (iterator instanceof Closeable closeable) {
- IOUtils.closeQuietly((closeable));
+ Utils.closeQuietly((closeable));
}
}
}
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
index 0693f8cb1a808..6674fcd73b92b 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
@@ -17,7 +17,6 @@
package org.apache.spark.util.collection.unsafe.sort;
-import com.google.common.io.ByteStreams;
import com.google.common.io.Closeables;
import org.apache.spark.SparkEnv;
import org.apache.spark.TaskContext;
@@ -27,6 +26,7 @@
import org.apache.spark.internal.SparkLoggerFactory;
import org.apache.spark.io.NioBufferedFileInputStream;
import org.apache.spark.io.ReadAheadInputStream;
+import org.apache.spark.network.util.JavaUtils;
import org.apache.spark.serializer.SerializerManager;
import org.apache.spark.storage.BlockId;
import org.apache.spark.unsafe.Platform;
@@ -128,7 +128,7 @@ public void loadNext() throws IOException {
arr = new byte[recordLength];
baseObject = arr;
}
- ByteStreams.readFully(in, arr, 0, recordLength);
+ JavaUtils.readFully(in, arr, 0, recordLength);
numRecordsRemaining--;
if (numRecordsRemaining == 0) {
close();
diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index 3f95515c04d29..2fdb8debf1909 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -24,7 +24,7 @@ import java.util.function.Consumer
import scala.collection.mutable.{ArrayBuffer, HashSet}
import scala.jdk.CollectionConverters._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerStageCompleted}
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 47f287293974f..ee1e4dafcfc1a 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -27,7 +27,7 @@ import scala.util.{Failure, Success => ScalaSuccess, Try}
import org.apache.spark.annotation.{Experimental, Since}
import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.memory.TaskMemoryManager
import org.apache.spark.metrics.source.Source
diff --git a/core/src/main/scala/org/apache/spark/ContextCleaner.scala b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
index fb56389cde77e..54ea8c94daac1 100644
--- a/core/src/main/scala/org/apache/spark/ContextCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
@@ -24,7 +24,7 @@ import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue, Scheduled
import scala.jdk.CollectionConverters._
import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ACCUMULATOR_ID, BROADCAST_ID, LISTENER, RDD_ID, SHUFFLE_ID}
import org.apache.spark.internal.config._
import org.apache.spark.rdd.{RDD, ReliableRDDCheckpointData}
diff --git a/core/src/main/scala/org/apache/spark/Dependency.scala b/core/src/main/scala/org/apache/spark/Dependency.scala
index 573608c4327e0..745faf866cebf 100644
--- a/core/src/main/scala/org/apache/spark/Dependency.scala
+++ b/core/src/main/scala/org/apache/spark/Dependency.scala
@@ -24,7 +24,7 @@ import scala.reflect.ClassTag
import org.roaringbitmap.RoaringBitmap
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rdd.RDD
import org.apache.spark.serializer.Serializer
diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index dd131e443135f..16ec7689d5261 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -26,7 +26,7 @@ import scala.util.control.NonFatal
import com.codahale.metrics.{Counter, Gauge, MetricRegistry}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.DECOMMISSION_ENABLED
@@ -614,7 +614,7 @@ private[spark] class ExecutorAllocationManager(
} else {
executorMonitor.executorsKilled(executorsRemoved.toSeq)
}
- logInfo(log"Executors ${MDC(EXECUTOR_IDS, executorsRemoved.mkString(","))}" +
+ logInfo(log"Executors ${MDC(EXECUTOR_IDS, executorsRemoved.mkString(","))} " +
log"removed due to idle timeout.")
executorsRemoved.toSeq
} else {
diff --git a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
index 92aea5959aab7..40ea5c59ec04e 100644
--- a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
+++ b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
@@ -23,7 +23,7 @@ import scala.collection.mutable.{HashMap, Map}
import scala.concurrent.Future
import org.apache.spark.executor.ExecutorMetrics
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.Network
import org.apache.spark.rpc.{IsolatedThreadSafeRpcEndpoint, RpcCallContext, RpcEnv}
diff --git a/core/src/main/scala/org/apache/spark/Heartbeater.scala b/core/src/main/scala/org/apache/spark/Heartbeater.scala
index 090458eecf182..8302aa17a9942 100644
--- a/core/src/main/scala/org/apache/spark/Heartbeater.scala
+++ b/core/src/main/scala/org/apache/spark/Heartbeater.scala
@@ -48,6 +48,13 @@ private[spark] class Heartbeater(
heartbeater.scheduleAtFixedRate(heartbeatTask, initialDelay, intervalMs, TimeUnit.MILLISECONDS)
}
+ /**
+ * Reports a heartbeat.
+ */
+ def doReportHeartbeat(): Unit = {
+ reportHeartbeat()
+ }
+
/** Stops the heartbeat thread. */
def stop(): Unit = {
heartbeater.shutdown()
diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
index a660bccd2e68f..9b2d3d748ed4d 100644
--- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
+++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
@@ -34,7 +34,7 @@ import org.apache.commons.io.output.{ByteArrayOutputStream => ApacheByteArrayOut
import org.roaringbitmap.RoaringBitmap
import org.apache.spark.broadcast.{Broadcast, BroadcastManager}
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.io.CompressionCodec
diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala b/core/src/main/scala/org/apache/spark/SecurityManager.scala
index c951876e62034..1461677219bc1 100644
--- a/core/src/main/scala/org/apache/spark/SecurityManager.scala
+++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.io.Text
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.UI._
import org.apache.spark.launcher.SparkLauncher
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index bbf2809b7db23..1d1901ae4f688 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import org.apache.avro.{Schema, SchemaNormalization}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.History._
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 2e61773251b67..d65b5f297dad4 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -46,14 +46,14 @@ import org.apache.spark.deploy.{LocalSparkCluster, SparkHadoopUtil}
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.{Executor, ExecutorMetrics, ExecutorMetricsSource}
import org.apache.spark.input.{FixedLengthBinaryInputFormat, PortableDataStream, StreamInputFormat, WholeTextFileInputFormat}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.Tests._
import org.apache.spark.internal.config.UI._
import org.apache.spark.internal.plugin.PluginContainer
import org.apache.spark.io.CompressionCodec
-import org.apache.spark.launcher.JavaModuleOptions
+import org.apache.spark.launcher.{JavaModuleOptions, SparkLauncher}
import org.apache.spark.metrics.source.JVMCPUSource
import org.apache.spark.partial.{ApproximateEvaluator, PartialResult}
import org.apache.spark.rdd._
@@ -201,10 +201,10 @@ class SparkContext(config: SparkConf) extends Logging {
// log out Spark Version in Spark driver log
logInfo(log"Running Spark version ${MDC(LogKeys.SPARK_VERSION, SPARK_VERSION)}")
- logInfo(log"OS info ${MDC(LogKeys.OS_NAME, System.getProperty("os.name"))}," +
- log" ${MDC(LogKeys.OS_VERSION, System.getProperty("os.version"))}, " +
- log"${MDC(LogKeys.OS_ARCH, System.getProperty("os.arch"))}")
- logInfo(log"Java version ${MDC(LogKeys.JAVA_VERSION, System.getProperty("java.version"))}")
+ logInfo(log"OS info ${MDC(LogKeys.OS_NAME, Utils.osName)}," +
+ log" ${MDC(LogKeys.OS_VERSION, Utils.osVersion)}, " +
+ log"${MDC(LogKeys.OS_ARCH, Utils.osArch)}")
+ logInfo(log"Java version ${MDC(LogKeys.JAVA_VERSION, Utils.javaVersion)}")
/* ------------------------------------------------------------------------------------- *
| Private variables. These variables keep the internal state of the context, and are |
@@ -483,7 +483,6 @@ class SparkContext(config: SparkConf) extends Logging {
}
_listenerBus = new LiveListenerBus(_conf)
- _resourceProfileManager = new ResourceProfileManager(_conf, _listenerBus)
// Initialize the app status store and listener before SparkEnv is created so that it gets
// all events.
@@ -584,8 +583,9 @@ class SparkContext(config: SparkConf) extends Logging {
_heartbeatReceiver = env.rpcEnv.setupEndpoint(
HeartbeatReceiver.ENDPOINT_NAME, new HeartbeatReceiver(this))
- // Initialize any plugins before the task scheduler is initialized.
+ // Initialize any plugins before initializing the task scheduler and resource profile manager.
_plugins = PluginContainer(this, _resources.asJava)
+ _resourceProfileManager = new ResourceProfileManager(_conf, _listenerBus)
_env.initializeShuffleManager()
_env.initializeMemoryManager(SparkContext.numDriverCores(master, conf))
@@ -2934,8 +2934,14 @@ class SparkContext(config: SparkConf) extends Logging {
_driverLogger.foreach(_.startSync(_hadoopConfiguration))
}
- /** Post the application end event */
+ /** Post the application end event and report the final heartbeat */
private def postApplicationEnd(exitCode: Int): Unit = {
+ try {
+ _heartbeater.doReportHeartbeat()
+ } catch {
+ case t: Throwable =>
+ logInfo("Unable to report driver heartbeat metrics when stopping spark context", t);
+ }
listenerBus.post(SparkListenerApplicationEnd(System.currentTimeMillis, Some(exitCode)))
}
@@ -3413,27 +3419,21 @@ object SparkContext extends Logging {
* `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions`.
*/
private def supplementJavaModuleOptions(conf: SparkConf): Unit = {
- def supplement(key: OptionalConfigEntry[String]): Unit = {
- val v = conf.get(key) match {
- case Some(opts) => s"${JavaModuleOptions.defaultModuleOptions()} $opts"
- case None => JavaModuleOptions.defaultModuleOptions()
- }
- conf.set(key.key, v)
+ def supplement(key: String): Unit = {
+ val v = s"${JavaModuleOptions.defaultModuleOptions()} ${conf.get(key, "")}".trim()
+ conf.set(key, v)
}
- supplement(DRIVER_JAVA_OPTIONS)
- supplement(EXECUTOR_JAVA_OPTIONS)
+ supplement(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
+ supplement(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS)
}
private def supplementJavaIPv6Options(conf: SparkConf): Unit = {
- def supplement(key: OptionalConfigEntry[String]): Unit = {
- val v = conf.get(key) match {
- case Some(opts) => s"-Djava.net.preferIPv6Addresses=${Utils.preferIPv6} $opts"
- case None => s"-Djava.net.preferIPv6Addresses=${Utils.preferIPv6}"
- }
- conf.set(key.key, v)
+ def supplement(key: String): Unit = {
+ val v = s"-Djava.net.preferIPv6Addresses=${Utils.preferIPv6} ${conf.get(key, "")}".trim()
+ conf.set(key, v)
}
- supplement(DRIVER_JAVA_OPTIONS)
- supplement(EXECUTOR_JAVA_OPTIONS)
+ supplement(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
+ supplement(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS)
}
}
diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala
index bf6e30f5afade..796dbf4b6d5f8 100644
--- a/core/src/main/scala/org/apache/spark/SparkEnv.scala
+++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala
@@ -32,7 +32,7 @@ import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.api.python.{PythonWorker, PythonWorkerFactory}
import org.apache.spark.broadcast.BroadcastManager
import org.apache.spark.executor.ExecutorBackend
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.memory.{MemoryManager, UnifiedMemoryManager}
diff --git a/core/src/main/scala/org/apache/spark/TaskContext.scala b/core/src/main/scala/org/apache/spark/TaskContext.scala
index 5384fd86a8f19..e4cf5f4bf01a5 100644
--- a/core/src/main/scala/org/apache/spark/TaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/TaskContext.scala
@@ -50,6 +50,15 @@ object TaskContext {
}
}
+ def withTaskContext[T](context: TaskContext)(task: => T): T = {
+ try {
+ TaskContext.setTaskContext(context)
+ task
+ } finally {
+ TaskContext.unset()
+ }
+ }
+
private[this] val taskContext: ThreadLocal[TaskContext] = new ThreadLocal[TaskContext]
// Note: protected[spark] instead of private[spark] to prevent the following two from
diff --git a/core/src/main/scala/org/apache/spark/TaskContextImpl.scala b/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
index f0e844289b9db..a7f09028597af 100644
--- a/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
+++ b/core/src/main/scala/org/apache/spark/TaskContextImpl.scala
@@ -25,7 +25,7 @@ import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.LISTENER
import org.apache.spark.memory.TaskMemoryManager
import org.apache.spark.metrics.MetricsSystem
diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala
index fed15a067c00f..aadfb2125cd69 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -20,7 +20,7 @@ package org.apache.spark
import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream}
import java.net.{HttpURLConnection, InetSocketAddress, URL}
import java.nio.charset.StandardCharsets
-import java.nio.file.{Files => JavaFiles, Paths}
+import java.nio.file.{Files, Paths}
import java.nio.file.attribute.PosixFilePermission.{OWNER_EXECUTE, OWNER_READ, OWNER_WRITE}
import java.security.SecureRandom
import java.security.cert.X509Certificate
@@ -37,8 +37,6 @@ import scala.reflect.{classTag, ClassTag}
import scala.sys.process.Process
import scala.util.Try
-import com.google.common.io.{ByteStreams, Files}
-import org.apache.commons.lang3.StringUtils
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.core.LoggerContext
import org.apache.logging.log4j.core.appender.ConsoleAppender
@@ -97,7 +95,7 @@ private[spark] object TestUtils extends SparkTestUtils {
files.foreach { case (k, v) =>
val entry = new JarEntry(k)
jarStream.putNextEntry(entry)
- ByteStreams.copy(new ByteArrayInputStream(v.getBytes(StandardCharsets.UTF_8)), jarStream)
+ new ByteArrayInputStream(v.getBytes(StandardCharsets.UTF_8)).transferTo(jarStream)
}
jarStream.close()
jarFile.toURI.toURL
@@ -133,7 +131,7 @@ private[spark] object TestUtils extends SparkTestUtils {
jarStream.putNextEntry(jarEntry)
val in = new FileInputStream(file)
- ByteStreams.copy(in, jarStream)
+ in.transferTo(jarStream)
in.close()
}
jarStream.close()
@@ -229,13 +227,13 @@ private[spark] object TestUtils extends SparkTestUtils {
def getAbsolutePathFromExecutable(executable: String): Option[String] = {
val command = if (Utils.isWindows) s"$executable.exe" else executable
if (command.split(File.separator, 2).length == 1 &&
- JavaFiles.isRegularFile(Paths.get(command)) &&
- JavaFiles.isExecutable(Paths.get(command))) {
+ Files.isRegularFile(Paths.get(command)) &&
+ Files.isExecutable(Paths.get(command))) {
Some(Paths.get(command).toAbsolutePath.toString)
} else {
sys.env("PATH").split(Pattern.quote(File.pathSeparator))
- .map(path => Paths.get(s"${StringUtils.strip(path, "\"")}${File.separator}$command"))
- .find(p => JavaFiles.isRegularFile(p) && JavaFiles.isExecutable(p))
+ .map(path => Paths.get(s"${Utils.strip(path, "\"")}${File.separator}$command"))
+ .find(p => Files.isRegularFile(p) && Files.isExecutable(p))
.map(_.toString)
}
}
@@ -413,7 +411,7 @@ private[spark] object TestUtils extends SparkTestUtils {
/** Creates a temp JSON file that contains the input JSON record. */
def createTempJsonFile(dir: File, prefix: String, jsonValue: JValue): String = {
val file = File.createTempFile(prefix, ".json", dir)
- JavaFiles.write(file.toPath, compact(render(jsonValue)).getBytes())
+ Files.write(file.toPath, compact(render(jsonValue)).getBytes())
file.getPath
}
@@ -421,8 +419,8 @@ private[spark] object TestUtils extends SparkTestUtils {
def createTempScriptWithExpectedOutput(dir: File, prefix: String, output: String): String = {
val file = File.createTempFile(prefix, ".sh", dir)
val script = s"cat < "NON_UDF"
@@ -91,6 +94,7 @@ private[spark] object PythonEvalType {
case SQL_COGROUPED_MAP_ARROW_UDF => "SQL_COGROUPED_MAP_ARROW_UDF"
case SQL_TABLE_UDF => "SQL_TABLE_UDF"
case SQL_ARROW_TABLE_UDF => "SQL_ARROW_TABLE_UDF"
+ case SQL_ARROW_UDTF => "SQL_ARROW_UDTF"
case SQL_TRANSFORM_WITH_STATE_PANDAS_UDF => "SQL_TRANSFORM_WITH_STATE_PANDAS_UDF"
case SQL_TRANSFORM_WITH_STATE_PANDAS_INIT_STATE_UDF =>
"SQL_TRANSFORM_WITH_STATE_PANDAS_INIT_STATE_UDF"
@@ -101,6 +105,8 @@ private[spark] object PythonEvalType {
// Arrow UDFs
case SQL_SCALAR_ARROW_UDF => "SQL_SCALAR_ARROW_UDF"
case SQL_SCALAR_ARROW_ITER_UDF => "SQL_SCALAR_ARROW_ITER_UDF"
+ case SQL_GROUPED_AGG_ARROW_UDF => "SQL_GROUPED_AGG_ARROW_UDF"
+ case SQL_WINDOW_AGG_ARROW_UDF => "SQL_WINDOW_AGG_ARROW_UDF"
}
}
@@ -170,6 +176,8 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
protected val faultHandlerEnabled: Boolean = conf.get(PYTHON_WORKER_FAULTHANLDER_ENABLED)
protected val idleTimeoutSeconds: Long = conf.get(PYTHON_WORKER_IDLE_TIMEOUT_SECONDS)
protected val killOnIdleTimeout: Boolean = conf.get(PYTHON_WORKER_KILL_ON_IDLE_TIMEOUT)
+ protected val tracebackDumpIntervalSeconds: Long =
+ conf.get(PYTHON_WORKER_TRACEBACK_DUMP_INTERVAL_SECONDS)
protected val hideTraceback: Boolean = false
protected val simplifiedTraceback: Boolean = false
@@ -267,6 +275,9 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
if (faultHandlerEnabled) {
envVars.put("PYTHON_FAULTHANDLER_DIR", faultHandlerLogDir.toString)
}
+ if (tracebackDumpIntervalSeconds > 0L) {
+ envVars.put("PYTHON_TRACEBACK_DUMP_INTERVAL_SECONDS", tracebackDumpIntervalSeconds.toString)
+ }
// allow the user to set the batch size for the BatchedSerializer on UDFs
envVars.put("PYTHON_UDF_BATCH_SIZE", batchSizeForPythonUDF.toString)
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
index 816ceea327aae..52ad6a93ce4c6 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
@@ -28,7 +28,7 @@ import scala.sys.process.Process
import org.apache.spark.{SparkContext, SparkEnv}
import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{PATH, PYTHON_PACKAGES, PYTHON_VERSION}
import org.apache.spark.util.ArrayImplicits.SparkArrayOps
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
index 64b29585a0d92..e02f10cc3fe69 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
@@ -32,9 +32,9 @@ import scala.jdk.OptionConverters._
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
-import org.apache.spark.internal.config.Python.{PYTHON_UNIX_DOMAIN_SOCKET_DIR, PYTHON_UNIX_DOMAIN_SOCKET_ENABLED}
+import org.apache.spark.internal.config.Python.PYTHON_FACTORY_IDLE_WORKER_MAX_POOL_SIZE
import org.apache.spark.security.SocketAuthHelper
import org.apache.spark.util.{RedirectThread, Utils}
@@ -95,11 +95,12 @@ private[spark] class PythonWorkerFactory(
// so we can also fall back to launching workers, pyspark/worker.py (by default) directly.
private val useDaemon = {
// This flag is ignored on Windows as it's unable to fork.
- !System.getProperty("os.name").startsWith("Windows") && useDaemonEnabled
+ !Utils.isWindows && useDaemonEnabled
}
- private val authHelper = new SocketAuthHelper(SparkEnv.get.conf)
- private val isUnixDomainSock = authHelper.conf.get(PYTHON_UNIX_DOMAIN_SOCKET_ENABLED)
+ private val conf = SparkEnv.get.conf
+ private val authHelper = new SocketAuthHelper(conf)
+ private val isUnixDomainSock = authHelper.isUnixDomainSock
@GuardedBy("self")
private var daemon: Process = null
@@ -111,7 +112,11 @@ private[spark] class PythonWorkerFactory(
@GuardedBy("self")
private var daemonSockPath: String = _
@GuardedBy("self")
- private val idleWorkers = new mutable.Queue[PythonWorker]()
+ // Visible for testing
+ private[spark] val idleWorkers = new mutable.Queue[PythonWorker]()
+ @GuardedBy("self")
+ private val maxIdleWorkerPoolSize =
+ conf.get(PYTHON_FACTORY_IDLE_WORKER_MAX_POOL_SIZE)
@GuardedBy("self")
private var lastActivityNs = 0L
new MonitorThread().start()
@@ -127,15 +132,16 @@ private[spark] class PythonWorkerFactory(
def create(): (PythonWorker, Option[ProcessHandle]) = {
if (useDaemon) {
self.synchronized {
- // Pull from idle workers until we one that is alive, otherwise create a new one.
+ // Pull from idle workers until we get one that is alive, otherwise create a new one.
while (idleWorkers.nonEmpty) {
val worker = idleWorkers.dequeue()
- val workerHandle = daemonWorkers(worker)
- if (workerHandle.isAlive()) {
- try {
- return (worker.refresh(), Some(workerHandle))
- } catch {
- case c: CancelledKeyException => /* pass */
+ daemonWorkers.get(worker).foreach { workerHandle =>
+ if (workerHandle.isAlive()) {
+ try {
+ return (worker.refresh(), Some(workerHandle))
+ } catch {
+ case _: CancelledKeyException => /* pass */
+ }
}
}
logWarning(log"Worker ${MDC(WORKER, worker)} " +
@@ -202,8 +208,7 @@ private[spark] class PythonWorkerFactory(
blockingMode: Boolean): (PythonWorker, Option[ProcessHandle]) = {
var serverSocketChannel: ServerSocketChannel = null
lazy val sockPath = new File(
- authHelper.conf.get(PYTHON_UNIX_DOMAIN_SOCKET_DIR)
- .getOrElse(System.getProperty("java.io.tmpdir")),
+ authHelper.sockDir,
s".${UUID.randomUUID()}.sock")
try {
if (isUnixDomainSock) {
@@ -306,8 +311,7 @@ private[spark] class PythonWorkerFactory(
if (isUnixDomainSock) {
workerEnv.put(
"PYTHON_WORKER_FACTORY_SOCK_DIR",
- authHelper.conf.get(PYTHON_UNIX_DOMAIN_SOCKET_DIR)
- .getOrElse(System.getProperty("java.io.tmpdir")))
+ authHelper.sockDir)
workerEnv.put("PYTHON_UNIX_DOMAIN_ENABLED", "True")
} else {
workerEnv.put("PYTHON_WORKER_FACTORY_SECRET", authHelper.secret)
@@ -482,6 +486,15 @@ private[spark] class PythonWorkerFactory(
if (useDaemon) {
self.synchronized {
lastActivityNs = System.nanoTime()
+ if (maxIdleWorkerPoolSize.exists(idleWorkers.size >= _)) {
+ val oldestWorker = idleWorkers.dequeue()
+ try {
+ stopWorker(oldestWorker)
+ } catch {
+ case e: Exception =>
+ logWarning("Failed to stop evicted worker", e)
+ }
+ }
idleWorkers.enqueue(worker)
}
} else {
diff --git a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
index 0fe57dd0bb0ae..e6f2a68567648 100644
--- a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
@@ -28,7 +28,7 @@ import net.razorvine.pickle.{Pickler, Unpickler}
import org.apache.spark.SparkException
import org.apache.spark.api.java.JavaRDD
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rdd.RDD
import org.apache.spark.util.ArrayImplicits._
diff --git a/core/src/main/scala/org/apache/spark/api/python/StreamingPythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/StreamingPythonRunner.scala
index 7eba574751b46..e61fa01db113b 100644
--- a/core/src/main/scala/org/apache/spark/api/python/StreamingPythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/StreamingPythonRunner.scala
@@ -23,7 +23,7 @@ import java.nio.channels.Channels
import scala.jdk.CollectionConverters._
import org.apache.spark.{SparkEnv, SparkPythonException}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{PYTHON_WORKER_MODULE, PYTHON_WORKER_RESPONSE, SESSION_ID}
import org.apache.spark.internal.config.BUFFER_SIZE
import org.apache.spark.internal.config.Python.{PYTHON_AUTH_SOCKET_TIMEOUT, PYTHON_UNIX_DOMAIN_SOCKET_ENABLED}
diff --git a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
index c3d01ec47458e..622833c9cd6af 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
@@ -26,7 +26,7 @@ import io.netty.handler.timeout.ReadTimeoutException
import org.apache.spark.{SparkConf, SparkEnv}
import org.apache.spark.api.r.SerDe._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.R._
import org.apache.spark.util.{ThreadUtils, Utils}
diff --git a/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala
index 3adb540a7ad18..ad1a274170a2a 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala
@@ -22,7 +22,7 @@ import java.io.Serializable
import scala.reflect.ClassTag
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.util.Utils
/**
diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
index 0c7ec5c1a98a7..0f93db9b96b58 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
@@ -27,7 +27,7 @@ import scala.reflect.ClassTag
import scala.util.Random
import org.apache.spark._
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.io.CompressionCodec
import org.apache.spark.serializer.Serializer
diff --git a/core/src/main/scala/org/apache/spark/deploy/Client.scala b/core/src/main/scala/org/apache/spark/deploy/Client.scala
index 226a6dcd36a16..2891247f20104 100644
--- a/core/src/main/scala/org/apache/spark/deploy/Client.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/Client.scala
@@ -31,7 +31,7 @@ import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.DeployMessages._
import org.apache.spark.deploy.master.{DriverState, Master}
import org.apache.spark.deploy.master.DriverState.DriverState
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.Network.RPC_ASK_TIMEOUT
import org.apache.spark.resource.ResourceUtils
diff --git a/core/src/main/scala/org/apache/spark/deploy/DriverTimeoutPlugin.scala b/core/src/main/scala/org/apache/spark/deploy/DriverTimeoutPlugin.scala
index 736c23556ec15..c1db3ddebc8ca 100644
--- a/core/src/main/scala/org/apache/spark/deploy/DriverTimeoutPlugin.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/DriverTimeoutPlugin.scala
@@ -23,7 +23,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.SparkContext
import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, PluginContext, SparkPlugin}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.DRIVER_TIMEOUT
import org.apache.spark.util.{SparkExitCode, ThreadUtils}
diff --git a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
index e21c772c00779..fec69aed48981 100644
--- a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.CountDownLatch
import scala.jdk.CollectionConverters._
import org.apache.spark.{SecurityManager, SparkConf}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.{AUTH_ENABLED, PORT, SHUFFLE_DB_BACKEND_KEY, SHUFFLE_DB_BACKEND_NAME}
import org.apache.spark.metrics.{MetricsSystem, MetricsSystemInstances}
import org.apache.spark.network.TransportContext
diff --git a/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala b/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
index 263b1a233b808..e03dc2ff00a08 100644
--- a/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
@@ -24,7 +24,7 @@ import scala.collection.mutable.ArrayBuffer
import org.apache.spark.SparkConf
import org.apache.spark.deploy.master.Master
import org.apache.spark.deploy.worker.Worker
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.rpc.RpcEnv
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala b/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala
index 5d996381a485e..f14c2dd693975 100644
--- a/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala
@@ -24,10 +24,8 @@ import java.util.zip.{ZipEntry, ZipOutputStream}
import scala.jdk.CollectionConverters._
-import com.google.common.io.{ByteStreams, Files}
-
import org.apache.spark.api.r.RUtils
-import org.apache.spark.internal.{LogEntry, Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{LogEntry, Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.{RedirectThread, Utils}
@@ -153,11 +151,11 @@ private[deploy] object RPackageUtils extends Logging {
if (verbose) {
print(log"Creating directory: ${MDC(PATH, dir)}", printStream)
}
- dir.mkdirs
+ Utils.createDirectory(dir)
} else {
val inStream = jar.getInputStream(entry)
val outPath = new File(tempDir, entryPath)
- Files.createParentDirs(outPath)
+ Utils.createParentDirs(outPath)
val outStream = new FileOutputStream(outPath)
if (verbose) {
print(log"Extracting ${MDC(JAR_ENTRY, entry)} to ${MDC(PATH, outPath)}", printStream)
@@ -251,7 +249,7 @@ private[deploy] object RPackageUtils extends Logging {
val fis = new FileInputStream(file)
val zipEntry = new ZipEntry(relPath)
zipOutputStream.putNextEntry(zipEntry)
- ByteStreams.copy(fis, zipOutputStream)
+ fis.transferTo(zipOutputStream)
zipOutputStream.closeEntry()
fis.close()
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/RedirectConsolePlugin.scala b/core/src/main/scala/org/apache/spark/deploy/RedirectConsolePlugin.scala
new file mode 100644
index 0000000000000..cc1995a264feb
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/deploy/RedirectConsolePlugin.scala
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy
+
+import java.io.{ByteArrayOutputStream, PrintStream}
+import java.util.{Collections, Map => JMap}
+
+import org.apache.spark.SparkContext
+import org.apache.spark.api.plugin.{DriverPlugin, ExecutorPlugin, PluginContext, SparkPlugin}
+import org.apache.spark.internal.{Logging, SparkLoggerFactory}
+import org.apache.spark.internal.config._
+
+/**
+ * A built-in plugin to allow redirecting stdout/stderr to logging system (SLF4J).
+ */
+class RedirectConsolePlugin extends SparkPlugin {
+ override def driverPlugin(): DriverPlugin = new DriverRedirectConsolePlugin()
+
+ override def executorPlugin(): ExecutorPlugin = new ExecRedirectConsolePlugin()
+}
+
+object RedirectConsolePlugin {
+
+ def redirectStdoutToLog(): Unit = {
+ val stdoutLogger = SparkLoggerFactory.getLogger("stdout")
+ System.setOut(new LoggingPrintStream(stdoutLogger.info))
+ }
+
+ def redirectStderrToLog(): Unit = {
+ val stderrLogger = SparkLoggerFactory.getLogger("stderr")
+ System.setErr(new LoggingPrintStream(stderrLogger.error))
+ }
+}
+
+class DriverRedirectConsolePlugin extends DriverPlugin with Logging {
+
+ override def init(sc: SparkContext, ctx: PluginContext): JMap[String, String] = {
+ val outputs = sc.conf.get(DRIVER_REDIRECT_CONSOLE_OUTPUTS)
+ if (outputs.contains("stdout")) {
+ logInfo("Redirect driver's stdout to logging system.")
+ RedirectConsolePlugin.redirectStdoutToLog()
+ }
+ if (outputs.contains("stderr")) {
+ logInfo("Redirect driver's stderr to logging system.")
+ RedirectConsolePlugin.redirectStderrToLog()
+ }
+ Collections.emptyMap
+ }
+}
+
+class ExecRedirectConsolePlugin extends ExecutorPlugin with Logging {
+
+ override def init(ctx: PluginContext, extraConf: JMap[String, String]): Unit = {
+ val outputs = ctx.conf.get(EXEC_REDIRECT_CONSOLE_OUTPUTS)
+ if (outputs.contains("stdout")) {
+ logInfo("Redirect executor's stdout to logging system.")
+ RedirectConsolePlugin.redirectStdoutToLog()
+ }
+ if (outputs.contains("stderr")) {
+ logInfo("Redirect executor's stderr to logging system.")
+ RedirectConsolePlugin.redirectStderrToLog()
+ }
+ }
+}
+
+private[spark] class LoggingPrintStream(redirect: String => Unit)
+ extends PrintStream(new LineBuffer(4 * 1024 * 1024)) {
+
+ override def write(b: Int): Unit = {
+ super.write(b)
+ tryLogCurrentLine()
+ }
+
+ override def write(buf: Array[Byte], off: Int, len: Int): Unit = {
+ super.write(buf, off, len)
+ tryLogCurrentLine()
+ }
+
+ private def tryLogCurrentLine(): Unit = this.synchronized {
+ out.asInstanceOf[LineBuffer].tryGenerateContext.foreach { logContext =>
+ redirect(logContext)
+ }
+ }
+}
+
+/**
+ * Cache bytes before line ending. When current line is ended or the bytes size reaches the
+ * threshold, it can generate the line.
+ */
+private[spark] object LineBuffer {
+ private val LF_BYTES = System.lineSeparator.getBytes
+ private val LF_LENGTH = LF_BYTES.length
+}
+
+private[spark] class LineBuffer(lineMaxBytes: Long) extends ByteArrayOutputStream {
+
+ import LineBuffer._
+
+ def tryGenerateContext: Option[String] =
+ if (isLineEnded) {
+ try Some(new String(buf, 0, count - LF_LENGTH)) finally reset()
+ } else if (count >= lineMaxBytes) {
+ try Some(new String(buf, 0, count)) finally reset()
+ } else {
+ None
+ }
+
+ private def isLineEnded: Boolean = {
+ if (count < LF_LENGTH) return false
+ // fast return in UNIX-like OS when LF is single char '\n'
+ if (LF_LENGTH == 1) return LF_BYTES(0) == buf(count - 1)
+
+ var i = 0
+ do {
+ if (LF_BYTES(i) != buf(count - LF_LENGTH + i)) {
+ return false
+ }
+ i = i + 1
+ } while (i < LF_LENGTH)
+ true
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index ca932ef5dc05c..9c40f4cd1cf75 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -37,7 +37,7 @@ import org.apache.hadoop.security.token.{Token, TokenIdentifier}
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier
import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.BUFFER_SIZE
import org.apache.spark.util.ArrayImplicits._
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkPipelines.scala b/core/src/main/scala/org/apache/spark/deploy/SparkPipelines.scala
new file mode 100644
index 0000000000000..2df83df3e291b
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkPipelines.scala
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy
+
+import java.util
+
+import scala.collection.mutable.ArrayBuffer
+import scala.jdk.CollectionConverters._
+
+import org.apache.spark.SparkUserAppException
+import org.apache.spark.internal.Logging
+import org.apache.spark.launcher.SparkSubmitArgumentsParser
+import org.apache.spark.util.SparkExitCode
+
+/**
+ * Outer implementation of the spark-pipelines command line interface. Responsible for routing
+ * spark-submit args to spark-submit, and pipeline-specific args to the inner Python CLI
+ * implementation that loads the user code and submits it to the backend.
+ */
+object SparkPipelines extends Logging {
+ def main(args: Array[String]): Unit = {
+ val sparkHome = sys.env("SPARK_HOME")
+ SparkSubmit.main(constructSparkSubmitArgs(args, sparkHome).toArray)
+ }
+
+ protected[deploy] def constructSparkSubmitArgs(
+ args: Array[String],
+ sparkHome: String): Seq[String] = {
+ val (sparkSubmitArgs, pipelinesArgs) = splitArgs(args)
+ val pipelinesCliFile = s"$sparkHome/python/pyspark/pipelines/cli.py"
+ (sparkSubmitArgs ++ Seq(pipelinesCliFile) ++ pipelinesArgs)
+ }
+
+ /**
+ * Split the arguments into spark-submit args (--master, --remote, etc.) and pipeline args
+ * (run, --spec, etc.).
+ */
+ private def splitArgs(args: Array[String]): (Seq[String], Seq[String]) = {
+ val sparkSubmitArgs = new ArrayBuffer[String]()
+ val pipelinesArgs = new ArrayBuffer[String]()
+ var remote = "local"
+
+ new SparkSubmitArgumentsParser() {
+ parse(util.Arrays.asList(args: _*))
+
+ override protected def handle(opt: String, value: String): Boolean = {
+ if (opt == "--remote") {
+ remote = value
+ } else if (opt == "--class") {
+ logInfo("--class argument not supported.")
+ throw SparkUserAppException(SparkExitCode.EXIT_FAILURE)
+ } else if (opt == "--conf" &&
+ value.startsWith("spark.api.mode=") &&
+ value != "spark.api.mode=connect") {
+ logInfo(
+ "--spark.api.mode must be 'connect'. " +
+ "Declarative Pipelines currently only supports Spark Connect."
+ )
+ throw SparkUserAppException(SparkExitCode.EXIT_FAILURE)
+ } else if (Seq("--name", "-h", "--help").contains(opt)) {
+ pipelinesArgs += opt
+ if (value != null && value.nonEmpty) {
+ pipelinesArgs += value
+ }
+ } else {
+ sparkSubmitArgs += opt
+ if (value != null) {
+ sparkSubmitArgs += value
+ }
+ }
+
+ true
+ }
+
+ override protected def handleExtraArgs(extra: util.List[String]): Unit = {
+ pipelinesArgs.appendAll(extra.asScala)
+ }
+
+ override protected def handleUnknown(opt: String): Boolean = {
+ pipelinesArgs += opt
+ true
+ }
+ }
+
+ sparkSubmitArgs += "--conf"
+ sparkSubmitArgs += "spark.api.mode=connect"
+ sparkSubmitArgs += "--remote"
+ sparkSubmitArgs += remote
+ (sparkSubmitArgs.toSeq, pipelinesArgs.toSeq)
+ }
+
+}
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index c529e37e7e1bf..fe74f034e2933 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -30,7 +30,6 @@ import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
import scala.util.{Properties, Try}
-import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.conf.{Configuration => HadoopConfiguration}
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.security.UserGroupInformation
@@ -39,7 +38,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.spark._
import org.apache.spark.api.r.RUtils
import org.apache.spark.deploy.rest._
-import org.apache.spark.internal.{LogEntry, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{LogEntry, Logging, LogKeys}
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.UI._
import org.apache.spark.launcher.SparkLauncher
@@ -368,7 +367,7 @@ private[spark] class SparkSubmit extends Logging {
// install any R packages that may have been passed through --jars or --packages.
// Spark Packages may contain R source code inside the jar.
- if (args.isR && !StringUtils.isBlank(args.jars)) {
+ if (args.isR && !SparkStringUtils.isBlank(args.jars)) {
RPackageUtils.checkAndBuildRPackage(args.jars, printStream, args.verbose)
}
}
@@ -997,14 +996,14 @@ private[spark] class SparkSubmit extends Logging {
// TODO(SPARK-42375): Should point out the user-facing page here instead.
logInfo("You need to specify Spark Connect jars with --jars or --packages.")
}
- throw new SparkUserAppException(CLASS_NOT_FOUND_EXIT_STATUS)
+ throw new SparkUserAppException(SparkExitCode.CLASS_NOT_FOUND)
case e: NoClassDefFoundError =>
logError(log"Failed to load ${MDC(LogKeys.CLASS_NAME, childMainClass)}", e)
if (e.getMessage.contains("org/apache/hadoop/hive")) {
logInfo("Failed to load hive class.")
logInfo("You need to build Spark with -Phive and -Phive-thriftserver.")
}
- throw new SparkUserAppException(CLASS_NOT_FOUND_EXIT_STATUS)
+ throw new SparkUserAppException(SparkExitCode.CLASS_NOT_FOUND)
}
val app: SparkApplication = if (classOf[SparkApplication].isAssignableFrom(mainClass)) {
@@ -1023,11 +1022,19 @@ private[spark] class SparkSubmit extends Logging {
e
}
+ var exitCode: Int = 1
try {
app.start(childArgs.toArray, sparkConf)
+ exitCode = 0
} catch {
case t: Throwable =>
- throw findCause(t)
+ val cause = findCause(t)
+ cause match {
+ case e: SparkUserAppException =>
+ exitCode = e.exitCode
+ case _ =>
+ }
+ throw cause
} finally {
if (args.master.startsWith("k8s") && !isShell(args.primaryResource) &&
!isSqlShell(args.mainClass) && !isThriftServer(args.mainClass) &&
@@ -1038,6 +1045,12 @@ private[spark] class SparkSubmit extends Logging {
case e: Throwable => logError("Failed to close SparkContext", e)
}
}
+ if (sparkConf.get(SUBMIT_CALL_SYSTEM_EXIT_ON_MAIN_EXIT)) {
+ logInfo(
+ log"Calling System.exit() with exit code ${MDC(LogKeys.EXIT_CODE, exitCode)} " +
+ log"because ${MDC(LogKeys.CONFIG, SUBMIT_CALL_SYSTEM_EXIT_ON_MAIN_EXIT.key)}=true")
+ exitFn(exitCode)
+ }
}
}
@@ -1081,8 +1094,6 @@ object SparkSubmit extends CommandLineUtils with Logging {
private val SPARKR_PACKAGE_ARCHIVE = "sparkr.zip"
private val R_PACKAGE_ARCHIVE = "rpkg.zip"
- private val CLASS_NOT_FOUND_EXIT_STATUS = 101
-
// Following constants are visible for testing.
private[deploy] val YARN_CLUSTER_SUBMIT_CLASS =
"org.apache.spark.deploy.yarn.YarnClusterApplication"
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index f4884385555f6..3eb2fd2a0e4d2 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -28,7 +28,7 @@ import scala.util.Try
import org.apache.spark.{SparkConf, SparkException, SparkUserAppException}
import org.apache.spark.deploy.SparkSubmitAction._
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.DYN_ALLOCATION_ENABLED
import org.apache.spark.launcher.SparkSubmitArgumentsParser
diff --git a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
index 7b98461b01acf..5a1d58897703d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
@@ -27,7 +27,7 @@ import org.json4s.{DefaultFormats, Extraction, Formats}
import org.json4s.jackson.JsonMethods.{compact, render}
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.COMPONENT
import org.apache.spark.resource.{ResourceAllocation, ResourceID, ResourceInformation, ResourceRequirement}
import org.apache.spark.util.ArrayImplicits._
diff --git a/core/src/main/scala/org/apache/spark/deploy/Utils.scala b/core/src/main/scala/org/apache/spark/deploy/Utils.scala
index b3d871d75e6c7..9ff1333b03e62 100644
--- a/core/src/main/scala/org/apache/spark/deploy/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/Utils.scala
@@ -22,7 +22,7 @@ import java.io.File
import jakarta.servlet.http.HttpServletRequest
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{LOG_TYPE, PATH}
import org.apache.spark.ui.JettyUtils.createServletHandler
import org.apache.spark.ui.WebUI
diff --git a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
index b34e5c408c3be..2d742b31f99c2 100644
--- a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala
@@ -29,7 +29,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.deploy.{ApplicationDescription, ExecutorState}
import org.apache.spark.deploy.DeployMessages._
import org.apache.spark.deploy.master.Master
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.resource.ResourceProfile
import org.apache.spark.rpc._
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala
index 8caf67ff4680b..e979671890eac 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala
@@ -28,7 +28,7 @@ import jakarta.servlet.{DispatcherType, Filter, FilterChain, ServletException, S
import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse}
import org.eclipse.jetty.servlet.FilterHolder
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.metrics.source.Source
import org.apache.spark.ui.SparkUI
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
index f3f7db6bb0aba..89f0d12935ce1 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
@@ -99,6 +99,15 @@ private[history] abstract class ApplicationHistoryProvider {
*/
def getListing(): Iterator[ApplicationInfo]
+ /**
+ * Returns a list of applications available for the history server to show.
+ *
+ * @param max The maximum number of applications to return
+ * @param predicate A function that filters the applications to be returned
+ * @return An iterator of matching applications up to the specified maximum
+ */
+ def getListing(max: Int)(predicate: ApplicationInfo => Boolean): Iterator[ApplicationInfo]
+
/**
* Returns the Spark UI for a specific application.
*
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala
index 20e60c679b06a..efa5b10c93f15 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventFilter.scala
@@ -23,7 +23,7 @@ import scala.util.control.NonFatal
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.deploy.history.EventFilter.FilterStatistics
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{LINE, LINE_NUM, PATH}
import org.apache.spark.scheduler._
import org.apache.spark.util.{JsonProtocol, Utils}
@@ -72,7 +72,6 @@ private[spark] object EventFilter extends Logging {
fs: FileSystem,
filters: Seq[EventFilter],
path: Path,
- jsonProtocol: JsonProtocol,
onAccepted: (String, SparkListenerEvent) => Unit,
onRejected: (String, SparkListenerEvent) => Unit,
onUnidentified: String => Unit): Unit = {
@@ -82,7 +81,7 @@ private[spark] object EventFilter extends Logging {
lines.zipWithIndex.foreach { case (line, lineNum) =>
try {
val event = try {
- Some(jsonProtocol.sparkEventFromJson(line))
+ Some(JsonProtocol.sparkEventFromJson(line))
} catch {
// ignore any exception occurred from unidentified json
case NonFatal(_) =>
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala
index 5cec3d2a9d28f..e0227641fc0eb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileCompactor.scala
@@ -28,10 +28,10 @@ import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
import org.apache.spark.SparkConf
import org.apache.spark.deploy.history.EventFilter.FilterStatistics
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.scheduler.ReplayListenerBus
-import org.apache.spark.util.{JsonProtocol, Utils}
+import org.apache.spark.util.Utils
/**
* This class compacts the old event log files into one compact file, via two phases reading:
@@ -55,8 +55,6 @@ class EventLogFileCompactor(
require(maxFilesToRetain > 0, "Max event log files to retain should be higher than 0.")
- private val jsonProtocol = new JsonProtocol(sparkConf)
-
/**
* Compacts the old event log files into one compact file, and clean old event log files being
* compacted away.
@@ -115,7 +113,7 @@ class EventLogFileCompactor(
* them via replaying events in given files.
*/
private def initializeBuilders(fs: FileSystem, files: Seq[Path]): Seq[EventFilterBuilder] = {
- val bus = new ReplayListenerBus(jsonProtocol)
+ val bus = new ReplayListenerBus()
val builders = ServiceLoader.load(classOf[EventFilterBuilder],
Utils.getContextOrSparkClassLoader).asScala.toSeq
@@ -155,7 +153,7 @@ class EventLogFileCompactor(
val startTime = System.currentTimeMillis()
logWriter.start()
eventLogFiles.foreach { file =>
- EventFilter.applyFilterToFile(fs, filters, file.getPath, jsonProtocol,
+ EventFilter.applyFilterToFile(fs, filters, file.getPath,
onAccepted = (line, _) => logWriter.writeEvent(line, flushLogger = true),
onRejected = (_, _) => {},
onUnidentified = line => logWriter.writeEvent(line, flushLogger = true)
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
index 8827fcde7b733..1721ef51807a5 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
@@ -21,7 +21,6 @@ import java.io.{BufferedInputStream, InputStream}
import java.util.concurrent.ConcurrentHashMap
import java.util.zip.{ZipEntry, ZipOutputStream}
-import com.google.common.io.ByteStreams
import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
import org.apache.hadoop.hdfs.DFSInputStream
@@ -52,7 +51,7 @@ abstract class EventLogFileReader(
entryName: String): Unit = {
Utils.tryWithResource(fileSystem.open(path, 1 * 1024 * 1024)) { inputStream =>
zipStream.putNextEntry(new ZipEntry(entryName))
- ByteStreams.copy(inputStream, zipStream)
+ inputStream.transferTo(zipStream)
zipStream.closeEntry()
}
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
index 990ab680f3aaf..4e3bee1015ff3 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.permission.FsPermission
import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
@@ -43,6 +43,7 @@ import org.apache.spark.util.Utils
* spark.eventLog.compression.codec - The codec to compress logged events
* spark.eventLog.overwrite - Whether to overwrite any existing files
* spark.eventLog.buffer.kb - Buffer size to use when writing to output streams
+ * spark.eventLog.excludedPatterns - Specifes a comma-separated event names to be excluded
*
* Note that descendant classes can maintain its own parameters: refer the javadoc of each class
* for more details.
@@ -58,6 +59,8 @@ abstract class EventLogFileWriter(
protected val shouldCompress = sparkConf.get(EVENT_LOG_COMPRESS) &&
!sparkConf.get(EVENT_LOG_COMPRESSION_CODEC).equalsIgnoreCase("none")
+ protected val excludedPatterns = sparkConf.get(EVENT_LOG_EXCLUDED_PATTERNS)
+ .map(name => s"""{"Event":"$name"""")
protected val shouldOverwrite = sparkConf.get(EVENT_LOG_OVERWRITE)
protected val outputBufferSize = sparkConf.get(EVENT_LOG_OUTPUT_BUFFER_SIZE).toInt * 1024
protected val fileSystem = Utils.getHadoopFileSystem(logBaseDir, hadoopConf)
@@ -117,6 +120,7 @@ abstract class EventLogFileWriter(
}
protected def writeLine(line: String, flushLogger: Boolean = false): Unit = {
+ if (excludedPatterns.exists(line.startsWith(_))) return
// scalastyle:off println
writer.foreach(_.println(line))
// scalastyle:on println
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index ad372a414c007..0c6d6acf66c89 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -37,7 +37,7 @@ import org.apache.hadoop.security.AccessControlException
import org.apache.spark.{SecurityManager, SparkConf, SparkException}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
@@ -52,8 +52,9 @@ import org.apache.spark.status._
import org.apache.spark.status.KVUtils._
import org.apache.spark.status.api.v1.{ApplicationAttemptInfo, ApplicationInfo}
import org.apache.spark.ui.SparkUI
-import org.apache.spark.util.{CallerContext, Clock, JsonProtocol, SystemClock, ThreadUtils, Utils}
+import org.apache.spark.util.{CallerContext, Clock, SystemClock, ThreadUtils, Utils}
import org.apache.spark.util.ArrayImplicits._
+import org.apache.spark.util.SparkStringUtils.stringToSeq
import org.apache.spark.util.kvstore._
/**
@@ -306,6 +307,14 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
.index("endTime").reverse())(_.toApplicationInfo()).iterator
}
+ override def getListing(max: Int)(
+ predicate: ApplicationInfo => Boolean): Iterator[ApplicationInfo] = {
+ // Return the filtered listing in end time descending order.
+ KVUtils.mapToSeqWithFilter(
+ listing.view(classOf[ApplicationInfoWrapper]).index("endTime").reverse(),
+ max)(_.toApplicationInfo())(predicate).iterator
+ }
+
override def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
try {
Some(load(appId).toApplicationInfo())
@@ -319,17 +328,14 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
override def getLastUpdatedTime(): Long = lastScanTime.get()
- /**
- * Split a comma separated String, filter out any empty items, and return a Sequence of strings
- */
- private def stringToSeq(list: String): Seq[String] = {
- list.split(',').map(_.trim).filter(_.nonEmpty).toImmutableArraySeq
- }
-
override def getAppUI(appId: String, attemptId: Option[String]): Option[LoadedAppUI] = {
+ val logPath = RollingEventLogFilesWriter.EVENT_LOG_DIR_NAME_PREFIX +
+ EventLogFileWriter.nameForAppAndAttempt(appId, attemptId)
val app = try {
load(appId)
} catch {
+ case _: NoSuchElementException if this.conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED) =>
+ loadFromFallbackLocation(appId, attemptId, logPath)
case _: NoSuchElementException =>
return None
}
@@ -351,6 +357,13 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
createInMemoryStore(attempt)
}
} catch {
+ case _: FileNotFoundException if this.conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED) =>
+ if (app.attempts.head.info.appSparkVersion == "unknown") {
+ listing.synchronized {
+ listing.delete(classOf[ApplicationInfoWrapper], appId)
+ }
+ }
+ return None
case _: FileNotFoundException =>
return None
}
@@ -370,6 +383,18 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
Some(loadedUI)
}
+ private def loadFromFallbackLocation(appId: String, attemptId: Option[String], logPath: String)
+ : ApplicationInfoWrapper = {
+ val date = new Date(0)
+ val lastUpdate = new Date()
+ val info = ApplicationAttemptInfo(
+ attemptId, date, date, lastUpdate, 0, "spark", false, "unknown")
+ addListing(new ApplicationInfoWrapper(
+ ApplicationInfo(appId, appId, None, None, None, None, List.empty),
+ List(new AttemptInfoWrapper(info, logPath, 0, Some(1), None, None, None, None))))
+ load(appId)
+ }
+
override def getEmptyListingHtml(): Seq[Node] = {
Did you specify the correct logging directory? Please verify your setting of
@@ -802,7 +827,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
val shouldHalt = enableOptimizations &&
((!appCompleted && fastInProgressParsing) || reparseChunkSize > 0)
- val bus = new ReplayListenerBus(new JsonProtocol(conf))
+ val bus = new ReplayListenerBus()
val listener = new AppListingListener(reader, clock, shouldHalt)
bus.addListener(listener)
@@ -1119,7 +1144,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
// to parse the event logs in the SHS.
val replayConf = conf.clone().set(ASYNC_TRACKING_ENABLED, false)
val trackingStore = new ElementTrackingStore(store, replayConf)
- val replayBus = new ReplayListenerBus(new JsonProtocol(conf))
+ val replayBus = new ReplayListenerBus()
val listener = new AppStatusListener(trackingStore, replayConf, false,
lastUpdateTime = Some(lastUpdated))
replayBus.addListener(listener)
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
index ff1629b698096..4eeddd7cc7098 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
@@ -109,7 +109,7 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
}
def shouldDisplayApplications(requestedIncomplete: Boolean): Boolean = {
- parent.getApplicationList().exists(isApplicationCompleted(_) != requestedIncomplete)
+ parent.getApplicationInfoList(1)(isApplicationCompleted(_) != requestedIncomplete).nonEmpty
}
private def makePageLink(request: HttpServletRequest, showIncomplete: Boolean): String = {
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index ce9f70c9a83ed..14daa5d88b1c4 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -28,7 +28,7 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.Utils.addRenderLogHandler
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.History
import org.apache.spark.internal.config.UI._
@@ -224,6 +224,11 @@ class HistoryServer(
getApplicationList()
}
+ override def getApplicationInfoList(max: Int)(
+ filter: ApplicationInfo => Boolean): Iterator[ApplicationInfo] = {
+ provider.getListing(max)(filter)
+ }
+
def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
provider.getApplicationInfo(appId)
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
index f1343a0551384..14d6e4a0381ae 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerArguments.scala
@@ -83,7 +83,7 @@ private[history] class HistoryServerArguments(conf: SparkConf, args: Array[Strin
configs.sortBy(_.key).foreach { conf =>
sb.append(" ").append(conf.key.padTo(maxConfigLength, ' '))
var currentDocLen = 0
- val intention = "\n" + " " * (maxConfigLength + 2)
+ val intention = "\n" + " ".repeat(maxConfigLength + 2)
conf.doc.split("\\s+").foreach { word =>
if (currentDocLen + word.length > 60) {
sb.append(intention).append(" ").append(word)
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
index 122ed299242f5..753fe59dc0894 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
@@ -22,10 +22,8 @@ import java.util.concurrent.atomic.AtomicLong
import scala.collection.mutable.{HashMap, ListBuffer}
-import org.apache.commons.io.FileUtils
-
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.History
import org.apache.spark.internal.config.History._
@@ -76,7 +74,7 @@ private class HistoryServerDiskManager(
// Clean up any temporary stores during start up. This assumes that they're leftover from other
// instances and are not useful.
- tmpStoreDir.listFiles().foreach(FileUtils.deleteQuietly)
+ tmpStoreDir.listFiles().foreach(Utils.deleteQuietly)
// Go through the recorded store directories and remove any that may have been removed by
// external code.
@@ -214,7 +212,7 @@ private class HistoryServerDiskManager(
def committed(): Long = committedUsage.get()
private def deleteStore(path: File): Unit = {
- FileUtils.deleteDirectory(path)
+ Utils.deleteRecursively(path)
listing.delete(classOf[ApplicationStoreInfo], path.getAbsolutePath())
}
@@ -286,7 +284,7 @@ private class HistoryServerDiskManager(
}
/** Visible for testing. Return the size of a directory. */
- private[history] def sizeOf(path: File): Long = FileUtils.sizeOf(path)
+ private[history] def sizeOf(path: File): Long = Utils.sizeOf(path)
private[history] class Lease(val tmpPath: File, private val leased: Long) {
@@ -334,7 +332,7 @@ private class HistoryServerDiskManager(
/** Deletes the temporary directory created for the lease. */
def rollback(): Unit = {
updateUsage(-leased)
- FileUtils.deleteDirectory(tmpPath)
+ Utils.deleteRecursively(tmpPath)
}
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala
index 6e3dbb1170998..f9a6c7f64896c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerMemoryManager.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicLong
import scala.collection.mutable.HashMap
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.History._
import org.apache.spark.io.CompressionCodec
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
index 4332544e4491c..0c9baf9e6f70c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
@@ -22,7 +22,7 @@ import java.nio.file.{FileAlreadyExistsException, Files, Paths}
import scala.reflect.ClassTag
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.io.CompressionCodec
import org.apache.spark.serializer.{DeserializationStream, SerializationStream, Serializer}
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 7d15744de6b45..95778771d4239 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -32,7 +32,7 @@ import org.apache.spark.deploy.master.DriverState.DriverState
import org.apache.spark.deploy.master.MasterMessages._
import org.apache.spark.deploy.master.ui.MasterWebUI
import org.apache.spark.deploy.rest.StandaloneRestServer
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.Deploy._
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala b/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala
index 964b115865aef..a751c7bfc67c4 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.master
import org.apache.spark.SparkConf
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.Deploy.{RECOVERY_COMPRESSION_CODEC, RECOVERY_DIRECTORY}
import org.apache.spark.io.CompressionCodec
import org.apache.spark.serializer.Serializer
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
index 6c7a8f582d915..66036e7a5e5ce 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
@@ -24,7 +24,7 @@ import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletRespons
import org.apache.spark.deploy.DeployMessages.{DecommissionWorkersOnHosts, MasterStateResponse, RequestMasterState}
import org.apache.spark.deploy.Utils.addRenderLogHandler
import org.apache.spark.deploy.master.Master
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{HOSTS, NUM_REMOVED_WORKERS}
import org.apache.spark.internal.config.DECOMMISSION_ENABLED
import org.apache.spark.internal.config.UI.MASTER_UI_DECOMMISSION_ALLOW_MODE
diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
index 3c0baacbf10be..6dc422515dbdb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
@@ -33,7 +33,7 @@ import jakarta.servlet.http.HttpServletResponse
import org.apache.spark.{SPARK_VERSION => sparkVersion, SparkConf, SparkException}
import org.apache.spark.deploy.SparkApplication
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
index f8afa86fd36fc..e172a06f0a32e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala
@@ -32,7 +32,7 @@ import org.json4s._
import org.json4s.jackson.JsonMethods._
import org.apache.spark.{SPARK_VERSION => sparkVersion, SparkConf}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{MASTER_REST_SERVER_FILTERS, MASTER_REST_SERVER_MAX_THREADS, MASTER_REST_SERVER_VIRTUAL_THREADS}
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
index 3a262a0d19fb5..83f36e1245844 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.security.Credentials
import org.apache.hadoop.security.token.{Token, TokenIdentifier}
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.security.HadoopDelegationTokenProvider
import org.apache.spark.util.Utils
@@ -54,8 +54,8 @@ private[security] class HBaseDelegationTokenProvider
creds.addToken(token.getService, token)
} catch {
case NonFatal(e) =>
- logWarning(Utils.createFailedToGetTokenMessage(serviceName, e) + log" Retrying to fetch " +
- log"HBase security token with ${MDC(SERVICE_NAME, serviceName)} connection parameter.")
+ logWarning(Utils.createFailedToGetTokenMessage(serviceName) + log" Retrying to fetch " +
+ log"HBase security token with ${MDC(SERVICE_NAME, serviceName)} connection parameter.", e)
// Seems to be spark is trying to get the token from HBase 2.x.x version or above where the
// obtainToken(Configuration conf) API has been removed. Lets try obtaining the token from
// another compatible API of HBase service.
@@ -98,7 +98,7 @@ private[security] class HBaseDelegationTokenProvider
creds.addToken(token.getService, token)
} catch {
case NonFatal(e) =>
- logWarning(Utils.createFailedToGetTokenMessage(serviceName, e))
+ logWarning(Utils.createFailedToGetTokenMessage(serviceName), e)
} finally {
if (null != hbaseConnection) {
hbaseConnection.close()
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
index b9d88266ed538..bc77159415473 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
@@ -31,7 +31,7 @@ import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.rpc.RpcEndpointRef
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
index fc750b54d0b8e..3a38eb4a08d7d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.security.{Credentials, UserGroupInformation}
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier
import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.security.HadoopDelegationTokenProvider
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
index a3e7276fc83e1..5bc17e2164738 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
@@ -24,7 +24,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.{SecurityManager, SSLOptions}
import org.apache.spark.deploy.Command
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.launcher.WorkerCommandBuilder
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
index ca0e024ad1aed..28de0f45eb27f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
@@ -19,19 +19,18 @@ package org.apache.spark.deploy.worker
import java.io._
import java.net.URI
-import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.StandardOpenOption
import scala.jdk.CollectionConverters._
-import com.google.common.io.{Files, FileWriteMode}
-
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.{DriverDescription, SparkHadoopUtil}
import org.apache.spark.deploy.DeployMessages.DriverStateChanged
import org.apache.spark.deploy.StandaloneResourceUtils.prepareResourcesFile
import org.apache.spark.deploy.master.DriverState
import org.apache.spark.deploy.master.DriverState.DriverState
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{DRIVER_RESOURCES_FILE, SPARK_DRIVER_PREFIX}
import org.apache.spark.internal.config.UI.UI_REVERSE_PROXY
@@ -145,7 +144,7 @@ private[deploy] class DriverRunner(
*/
private def createWorkingDirectory(): File = {
val driverDir = new File(workDir, driverId)
- if (!driverDir.exists() && !driverDir.mkdirs()) {
+ if (!driverDir.exists() && !Utils.createDirectory(driverDir)) {
throw new IOException("Failed to create directory " + driverDir)
}
driverDir
@@ -215,8 +214,8 @@ private[deploy] class DriverRunner(
val stderr = new File(baseDir, "stderr")
val redactedCommand = Utils.redactCommandLineArgs(conf, builder.command.asScala.toSeq)
.mkString("\"", "\" \"", "\"")
- val header = "Launch Command: %s\n%s\n\n".format(redactedCommand, "=" * 40)
- Files.asCharSink(stderr, StandardCharsets.UTF_8, FileWriteMode.APPEND).write(header)
+ val header = "Launch Command: %s\n%s\n\n".format(redactedCommand, "=".repeat(40))
+ Files.writeString(stderr.toPath, header, StandardOpenOption.APPEND)
CommandUtils.redirectStream(process.getErrorStream, stderr)
}
runCommandWithRetry(ProcessBuilderLike(builder), initialize, supervise)
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala
index 4f42088903464..1d64dbc224c49 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverWrapper.scala
@@ -21,7 +21,7 @@ import java.io.File
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.RPC_ADDRESS
import org.apache.spark.rpc.RpcEnv
import org.apache.spark.util._
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
index d21904dd16ea7..8f0b684a93e81 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
@@ -18,17 +18,15 @@
package org.apache.spark.deploy.worker
import java.io._
-import java.nio.charset.StandardCharsets
+import java.nio.file.Files
import scala.jdk.CollectionConverters._
-import com.google.common.io.Files
-
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.{ApplicationDescription, ExecutorState}
import org.apache.spark.deploy.DeployMessages.ExecutorStateChanged
import org.apache.spark.deploy.StandaloneResourceUtils.prepareResourcesFile
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.SPARK_EXECUTOR_PREFIX
import org.apache.spark.internal.config.UI._
@@ -184,14 +182,14 @@ private[deploy] class ExecutorRunner(
process = builder.start()
val header = "Spark Executor Command: %s\n%s\n\n".format(
- redactedCommand, "=" * 40)
+ redactedCommand, "=".repeat(40))
// Redirect its stdout and stderr to files
val stdout = new File(executorDir, "stdout")
stdoutAppender = FileAppender(process.getInputStream, stdout, conf, true)
val stderr = new File(executorDir, "stderr")
- Files.asCharSink(stderr, StandardCharsets.UTF_8).write(header)
+ Files.writeString(stderr.toPath, header)
stderrAppender = FileAppender(process.getErrorStream, stderr, conf, true)
state = ExecutorState.RUNNING
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index b2ec23887a400..98da33a429eca 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -37,7 +37,7 @@ import org.apache.spark.deploy.ExternalShuffleService
import org.apache.spark.deploy.StandaloneResourceUtils._
import org.apache.spark.deploy.master.{DriverState, Master}
import org.apache.spark.deploy.worker.ui.WorkerWebUI
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.Tests.IS_TESTING
import org.apache.spark.internal.config.UI._
@@ -600,7 +600,7 @@ private[deploy] class Worker(
// Create the executor's working directory
val executorDir = new File(workDir, appId + "/" + execId)
- if (!executorDir.mkdirs()) {
+ if (!Utils.createDirectory(executorDir)) {
throw new IOException("Failed to create directory " + executorDir)
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
index bd07a0ade523d..ef6d84631c83c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.worker
import java.util.concurrent.atomic.AtomicBoolean
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.LogKeys.WORKER_URL
import org.apache.spark.rpc._
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
index defce5acc6168..bdc143be91b94 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
@@ -23,7 +23,7 @@ import scala.xml.{Node, Unparsed}
import jakarta.servlet.http.HttpServletRequest
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{LOG_TYPE, PATH}
import org.apache.spark.ui.{UIUtils, WebUIPage}
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
index a30759e5d794e..206a6a0fe385c 100644
--- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
+++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@ -31,7 +31,7 @@ import org.apache.spark._
import org.apache.spark.TaskState.TaskState
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.worker.WorkerWatcher
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.network.netty.SparkTransportConf
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 3b066e15386b8..a14ba21a0c186 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -36,11 +36,11 @@ import scala.util.control.NonFatal
import com.google.common.cache.{Cache, CacheBuilder, RemovalListener, RemovalNotification}
import com.google.common.util.concurrent.ThreadFactoryBuilder
-import org.slf4j.MDC
+import org.slf4j.{MDC => SLF4JMDC}
import org.apache.spark._
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, LogKeys, MDC => LogMDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.{EXECUTOR_USER_CLASS_PATH_FIRST => EXECUTOR_USER_CLASS_PATH_FIRST_CONFIG}
@@ -82,12 +82,12 @@ private[spark] class Executor(
resources: immutable.Map[String, ResourceInformation])
extends Logging {
- logInfo(log"Starting executor ID ${LogMDC(LogKeys.EXECUTOR_ID, executorId)}" +
- log" on host ${LogMDC(HOST, executorHostname)}")
- logInfo(log"OS info ${LogMDC(OS_NAME, System.getProperty("os.name"))}," +
- log" ${LogMDC(OS_VERSION, System.getProperty("os.version"))}, " +
- log"${LogMDC(OS_ARCH, System.getProperty("os.arch"))}")
- logInfo(log"Java version ${LogMDC(JAVA_VERSION, System.getProperty("java.version"))}")
+ logInfo(log"Starting executor ID ${MDC(LogKeys.EXECUTOR_ID, executorId)}" +
+ log" on host ${MDC(HOST, executorHostname)}")
+ logInfo(log"OS info ${MDC(OS_NAME, Utils.osName)}," +
+ log" ${MDC(OS_VERSION, Utils.osVersion)}, " +
+ log"${MDC(OS_ARCH, Utils.osArch)}")
+ logInfo(log"Java version ${MDC(JAVA_VERSION, Utils.javaVersion)}")
private val executorShutdown = new AtomicBoolean(false)
val stopHookReference = ShutdownHookManager.addShutdownHook(
@@ -102,7 +102,7 @@ private[spark] class Executor(
private[executor] val taskNameMDCKey = if (conf.get(LEGACY_TASK_NAME_MDC_ENABLED)) {
"mdc.taskName"
} else {
- LogKeys.TASK_NAME.name
+ TASK_NAME.name.toLowerCase(Locale.ROOT)
}
// SPARK-40235: updateDependencies() uses a ReentrantLock instead of the `synchronized` keyword
@@ -223,7 +223,7 @@ private[spark] class Executor(
if (sessionBasedRoot.isDirectory && sessionBasedRoot.exists()) {
Utils.deleteRecursively(sessionBasedRoot)
}
- logInfo(log"Session evicted: ${LogMDC(SESSION_ID, state.sessionUUID)}")
+ logInfo(log"Session evicted: ${MDC(SESSION_ID, state.sessionUUID)}")
}
})
.build[String, IsolatedSessionState]
@@ -505,9 +505,9 @@ private[spark] class Executor(
@volatile var task: Task[Any] = _
def kill(interruptThread: Boolean, reason: String): Unit = {
- logInfo(log"Executor is trying to kill ${LogMDC(TASK_NAME, taskName)}, " +
- log"interruptThread: ${LogMDC(INTERRUPT_THREAD, interruptThread)}, " +
- log"reason: ${LogMDC(REASON, reason)}")
+ logInfo(log"Executor is trying to kill ${MDC(TASK_NAME, taskName)}, " +
+ log"interruptThread: ${MDC(INTERRUPT_THREAD, interruptThread)}, " +
+ log"reason: ${MDC(REASON, reason)}")
reasonIfKilled = Some(reason)
if (task != null) {
synchronized {
@@ -579,7 +579,7 @@ private[spark] class Executor(
} else 0L
Thread.currentThread.setContextClassLoader(isolatedSession.replClassLoader)
val ser = env.closureSerializer.newInstance()
- logInfo(log"Running ${LogMDC(TASK_NAME, taskName)}")
+ logInfo(log"Running ${MDC(TASK_NAME, taskName)}")
execBackend.statusUpdate(taskId, TaskState.RUNNING, EMPTY_BYTE_BUFFER)
var taskStartTimeNs: Long = 0
var taskStartCpu: Long = 0
@@ -653,7 +653,7 @@ private[spark] class Executor(
if (freedMemory > 0 && !threwException) {
val errMsg = log"Managed memory leak detected; size = " +
- log"${LogMDC(NUM_BYTES, freedMemory)} bytes, ${LogMDC(TASK_NAME, taskName)}"
+ log"${MDC(NUM_BYTES, freedMemory)} bytes, ${MDC(TASK_NAME, taskName)}"
if (conf.get(UNSAFE_EXCEPTION_ON_MEMORY_LEAK)) {
throw SparkException.internalError(errMsg.message, category = "EXECUTOR")
} else {
@@ -663,9 +663,9 @@ private[spark] class Executor(
if (releasedLocks.nonEmpty && !threwException) {
val errMsg =
- log"${LogMDC(NUM_RELEASED_LOCKS, releasedLocks.size)} block locks" +
- log" were not released by ${LogMDC(TASK_NAME, taskName)}\n" +
- log" ${LogMDC(RELEASED_LOCKS, releasedLocks.mkString("[", ", ", "]"))})"
+ log"${MDC(NUM_RELEASED_LOCKS, releasedLocks.size)} block locks" +
+ log" were not released by ${MDC(TASK_NAME, taskName)}\n" +
+ log" ${MDC(RELEASED_LOCKS, releasedLocks.mkString("[", ", ", "]"))})"
if (conf.get(STORAGE_EXCEPTION_PIN_LEAK)) {
throw SparkException.internalError(errMsg.message, category = "EXECUTOR")
} else {
@@ -677,10 +677,10 @@ private[spark] class Executor(
// uh-oh. it appears the user code has caught the fetch-failure without throwing any
// other exceptions. Its *possible* this is what the user meant to do (though highly
// unlikely). So we will log an error and keep going.
- logError(log"${LogMDC(TASK_NAME, taskName)} completed successfully though internally " +
+ logError(log"${MDC(TASK_NAME, taskName)} completed successfully though internally " +
log"it encountered unrecoverable fetch failures! Most likely this means user code " +
log"is incorrectly swallowing Spark's internal " +
- log"${LogMDC(CLASS_NAME, classOf[FetchFailedException])}", fetchFailure)
+ log"${MDC(CLASS_NAME, classOf[FetchFailedException])}", fetchFailure)
}
val taskFinishNs = System.nanoTime()
val taskFinishCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) {
@@ -746,10 +746,10 @@ private[spark] class Executor(
// directSend = sending directly back to the driver
val serializedResult: ByteBuffer = {
if (maxResultSize > 0 && resultSize > maxResultSize) {
- logWarning(log"Finished ${LogMDC(TASK_NAME, taskName)}. " +
+ logWarning(log"Finished ${MDC(TASK_NAME, taskName)}. " +
log"Result is larger than maxResultSize " +
- log"(${LogMDC(RESULT_SIZE_BYTES, Utils.bytesToString(resultSize))} > " +
- log"${LogMDC(RESULT_SIZE_BYTES_MAX, Utils.bytesToString(maxResultSize))}), " +
+ log"(${MDC(RESULT_SIZE_BYTES, Utils.bytesToString(resultSize))} > " +
+ log"${MDC(RESULT_SIZE_BYTES_MAX, Utils.bytesToString(maxResultSize))}), " +
log"dropping it.")
ser.serialize(new IndirectTaskResult[Any](TaskResultBlockId(taskId), resultSize))
} else if (resultSize > maxDirectResultSize) {
@@ -758,12 +758,12 @@ private[spark] class Executor(
blockId,
serializedDirectResult,
StorageLevel.MEMORY_AND_DISK_SER)
- logInfo(log"Finished ${LogMDC(TASK_NAME, taskName)}." +
- log" ${LogMDC(NUM_BYTES, resultSize)} bytes result sent via BlockManager)")
+ logInfo(log"Finished ${MDC(TASK_NAME, taskName)}." +
+ log" ${MDC(NUM_BYTES, resultSize)} bytes result sent via BlockManager)")
ser.serialize(new IndirectTaskResult[Any](blockId, resultSize))
} else {
- logInfo(log"Finished ${LogMDC(TASK_NAME, taskName)}." +
- log" ${LogMDC(NUM_BYTES, resultSize)} bytes result sent to driver")
+ logInfo(log"Finished ${MDC(TASK_NAME, taskName)}." +
+ log" ${MDC(NUM_BYTES, resultSize)} bytes result sent to driver")
// toByteBuffer is safe here, guarded by maxDirectResultSize
serializedDirectResult.toByteBuffer
}
@@ -775,8 +775,8 @@ private[spark] class Executor(
execBackend.statusUpdate(taskId, TaskState.FINISHED, serializedResult)
} catch {
case t: TaskKilledException =>
- logInfo(log"Executor killed ${LogMDC(TASK_NAME, taskName)}," +
- log" reason: ${LogMDC(REASON, t.reason)}")
+ logInfo(log"Executor killed ${MDC(TASK_NAME, taskName)}," +
+ log" reason: ${MDC(REASON, t.reason)}")
val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs)
// Here and below, put task metric peaks in an immutable.ArraySeq to expose them as an
@@ -789,8 +789,8 @@ private[spark] class Executor(
case _: InterruptedException | NonFatal(_) if
task != null && task.reasonIfKilled.isDefined =>
val killReason = task.reasonIfKilled.getOrElse("unknown reason")
- logInfo(log"Executor interrupted and killed ${LogMDC(TASK_NAME, taskName)}," +
- log" reason: ${LogMDC(REASON, killReason)}")
+ logInfo(log"Executor interrupted and killed ${MDC(TASK_NAME, taskName)}," +
+ log" reason: ${MDC(REASON, killReason)}")
val (accums, accUpdates) = collectAccumulatorsAndResetStatusOnFailure(taskStartTimeNs)
val metricPeaks = metricsPoller.getTaskMetricPeaks(taskId).toImmutableArraySeq
@@ -803,12 +803,12 @@ private[spark] class Executor(
if (!t.isInstanceOf[FetchFailedException]) {
// there was a fetch failure in the task, but some user code wrapped that exception
// and threw something else. Regardless, we treat it as a fetch failure.
- logWarning(log"${LogMDC(TASK_NAME, taskName)} encountered a " +
- log"${LogMDC(CLASS_NAME, classOf[FetchFailedException].getName)} " +
+ logWarning(log"${MDC(TASK_NAME, taskName)} encountered a " +
+ log"${MDC(CLASS_NAME, classOf[FetchFailedException].getName)} " +
log"and failed, but the " +
- log"${LogMDC(CLASS_NAME, classOf[FetchFailedException].getName)} " +
+ log"${MDC(CLASS_NAME, classOf[FetchFailedException].getName)} " +
log"was hidden by another exception. Spark is handling this like a fetch failure " +
- log"and ignoring the other exception: ${LogMDC(ERROR, t)}")
+ log"and ignoring the other exception: ${MDC(ERROR, t)}")
}
setTaskFinishedAndClearInterruptStatus()
plugins.foreach(_.onTaskFailed(reason))
@@ -823,13 +823,13 @@ private[spark] class Executor(
case t: Throwable if env.isStopped =>
// Log the expected exception after executor.stop without stack traces
// see: SPARK-19147
- logError(log"Exception in ${LogMDC(TASK_NAME, taskName)}: ${LogMDC(ERROR, t.getMessage)}")
+ logError(log"Exception in ${MDC(TASK_NAME, taskName)}: ${MDC(ERROR, t.getMessage)}")
case t: Throwable =>
// Attempt to exit cleanly by informing the driver of our failure.
// If anything goes wrong (or this was a fatal exception), we will delegate to
// the default uncaught exception handler, which will terminate the Executor.
- logError(log"Exception in ${LogMDC(TASK_NAME, taskName)}", t)
+ logError(log"Exception in ${MDC(TASK_NAME, taskName)}", t)
// SPARK-20904: Do not report failure to driver if if happened during shut down. Because
// libraries may set up shutdown hooks that race with running tasks during shutdown,
@@ -934,16 +934,16 @@ private[spark] class Executor(
private def setMDCForTask(taskName: String, mdc: Seq[(String, String)]): Unit = {
if (Executor.mdcIsSupported) {
- mdc.foreach { case (key, value) => MDC.put(key, value) }
+ mdc.foreach { case (key, value) => SLF4JMDC.put(key, value) }
// avoid overriding the takName by the user
- MDC.put(taskNameMDCKey, taskName)
+ SLF4JMDC.put(taskNameMDCKey, taskName)
}
}
private def cleanMDCForTask(taskName: String, mdc: Seq[(String, String)]): Unit = {
if (Executor.mdcIsSupported) {
- mdc.foreach { case (key, _) => MDC.remove(key) }
- MDC.remove(taskNameMDCKey)
+ mdc.foreach { case (key, _) => SLF4JMDC.remove(key) }
+ SLF4JMDC.remove(taskNameMDCKey)
}
}
@@ -1016,14 +1016,14 @@ private[spark] class Executor(
finished = true
} else {
val elapsedTimeMs = TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs)
- logWarning(log"Killed task ${LogMDC(TASK_ID, taskId)} " +
- log"is still running after ${LogMDC(TIME_UNITS, elapsedTimeMs)} ms")
+ logWarning(log"Killed task ${MDC(TASK_ID, taskId)} " +
+ log"is still running after ${MDC(TIME_UNITS, elapsedTimeMs)} ms")
if (takeThreadDump) {
try {
taskRunner.theadDump().foreach { thread =>
if (thread.threadName == taskRunner.threadName) {
- logWarning(log"Thread dump from task ${LogMDC(TASK_ID, taskId)}:\n" +
- log"${LogMDC(THREAD, thread.toString)}")
+ logWarning(log"Thread dump from task ${MDC(TASK_ID, taskId)}:\n" +
+ log"${MDC(THREAD, thread.toString)}")
}
}
} catch {
@@ -1037,8 +1037,8 @@ private[spark] class Executor(
if (!taskRunner.isFinished && timeoutExceeded()) {
val killTimeoutMs = TimeUnit.NANOSECONDS.toMillis(killTimeoutNs)
if (isLocal) {
- logError(log"Killed task ${LogMDC(TASK_ID, taskId)} could not be stopped within " +
- log"${LogMDC(TIMEOUT, killTimeoutMs)} ms; " +
+ logError(log"Killed task ${MDC(TASK_ID, taskId)} could not be stopped within " +
+ log"${MDC(TIMEOUT, killTimeoutMs)} ms; " +
log"not killing JVM because we are running in local mode.")
} else {
// In non-local-mode, the exception thrown here will bubble up to the uncaught exception
@@ -1093,8 +1093,8 @@ private[spark] class Executor(
logInfo(
log"Starting executor with user classpath" +
log" (userClassPathFirst =" +
- log" ${LogMDC(LogKeys.EXECUTOR_USER_CLASS_PATH_FIRST, userClassPathFirst)}): " +
- log"${LogMDC(URLS, urls.mkString("'", ",", "'"))}"
+ log" ${MDC(LogKeys.EXECUTOR_USER_CLASS_PATH_FIRST, userClassPathFirst)}): " +
+ log"${MDC(URLS, urls.mkString("'", ",", "'"))}"
)
if (useStub) {
@@ -1150,13 +1150,13 @@ private[spark] class Executor(
sessionUUID: String): ClassLoader = {
val classUri = sessionClassUri.getOrElse(conf.get("spark.repl.class.uri", null))
val classLoader = if (classUri != null) {
- logInfo(log"Using REPL class URI: ${LogMDC(LogKeys.URI, classUri)}")
+ logInfo(log"Using REPL class URI: ${MDC(LogKeys.URI, classUri)}")
new ExecutorClassLoader(conf, env, classUri, parent, userClassPathFirst)
} else {
parent
}
- logInfo(log"Created or updated repl class loader ${LogMDC(CLASS_LOADER, classLoader)}" +
- log" for ${LogMDC(SESSION_ID, sessionUUID)}.")
+ logInfo(log"Created or updated repl class loader ${MDC(CLASS_LOADER, classLoader)}" +
+ log" for ${MDC(SESSION_ID, sessionUUID)}.")
classLoader
}
@@ -1191,16 +1191,16 @@ private[spark] class Executor(
// Fetch missing dependencies
for ((name, timestamp) <- newFiles if state.currentFiles.getOrElse(name, -1L) < timestamp) {
- logInfo(log"Fetching ${LogMDC(FILE_NAME, name)} with" +
- log" timestamp ${LogMDC(TIMESTAMP, timestamp)}")
+ logInfo(log"Fetching ${MDC(FILE_NAME, name)} with" +
+ log" timestamp ${MDC(TIMESTAMP, timestamp)}")
// Fetch file with useCache mode, close cache for local mode.
Utils.fetchFile(name, root, conf, hadoopConf, timestamp, useCache = !isLocal)
state.currentFiles(name) = timestamp
}
for ((name, timestamp) <- newArchives if
state.currentArchives.getOrElse(name, -1L) < timestamp) {
- logInfo(log"Fetching ${LogMDC(ARCHIVE_NAME, name)} with" +
- log" timestamp ${LogMDC(TIMESTAMP, timestamp)}")
+ logInfo(log"Fetching ${MDC(ARCHIVE_NAME, name)} with" +
+ log" timestamp ${MDC(TIMESTAMP, timestamp)}")
val sourceURI = new URI(name)
val uriToDownload = Utils.getUriBuilder(sourceURI).fragment(null).build()
val source = Utils.fetchFile(uriToDownload.toString, Utils.createTempDir(), conf,
@@ -1209,10 +1209,10 @@ private[spark] class Executor(
root,
if (sourceURI.getFragment != null) sourceURI.getFragment else source.getName)
logInfo(
- log"Unpacking an archive ${LogMDC(ARCHIVE_NAME, name)}" +
- log" (${LogMDC(BYTE_SIZE, source.length)} bytes)" +
- log" from ${LogMDC(SOURCE_PATH, source.getAbsolutePath)}" +
- log" to ${LogMDC(DESTINATION_PATH, dest.getAbsolutePath)}")
+ log"Unpacking an archive ${MDC(ARCHIVE_NAME, name)}" +
+ log" (${MDC(BYTE_SIZE, source.length)} bytes)" +
+ log" from ${MDC(SOURCE_PATH, source.getAbsolutePath)}" +
+ log" to ${MDC(DESTINATION_PATH, dest.getAbsolutePath)}")
Utils.deleteRecursively(dest)
Utils.unpack(source, dest)
state.currentArchives(name) = timestamp
@@ -1223,8 +1223,8 @@ private[spark] class Executor(
.orElse(state.currentJars.get(localName))
.getOrElse(-1L)
if (currentTimeStamp < timestamp) {
- logInfo(log"Fetching ${LogMDC(JAR_URL, name)} with" +
- log" timestamp ${LogMDC(TIMESTAMP, timestamp)}")
+ logInfo(log"Fetching ${MDC(JAR_URL, name)} with" +
+ log" timestamp ${MDC(TIMESTAMP, timestamp)}")
// Fetch file with useCache mode, close cache for local mode.
Utils.fetchFile(name, root, conf,
hadoopConf, timestamp, useCache = !isLocal)
@@ -1232,8 +1232,8 @@ private[spark] class Executor(
// Add it to our class loader
val url = new File(root, localName).toURI.toURL
if (!state.urlClassLoader.getURLs().contains(url)) {
- logInfo(log"Adding ${LogMDC(LogKeys.URL, url)} to" +
- log" class loader ${LogMDC(UUID, state.sessionUUID)}")
+ logInfo(log"Adding ${MDC(LogKeys.URL, url)} to" +
+ log" class loader ${MDC(UUID, state.sessionUUID)}")
state.urlClassLoader.addURL(url)
if (isStubbingEnabledForState(state.sessionUUID)) {
renewClassLoader = true
@@ -1298,7 +1298,7 @@ private[spark] class Executor(
heartbeatFailures += 1
if (heartbeatFailures >= HEARTBEAT_MAX_FAILURES) {
logError(log"Exit as unable to send heartbeats to driver " +
- log"more than ${LogMDC(MAX_ATTEMPTS, HEARTBEAT_MAX_FAILURES)} times")
+ log"more than ${MDC(MAX_ATTEMPTS, HEARTBEAT_MAX_FAILURES)} times")
System.exit(ExecutorExitCode.HEARTBEAT_FAILURE)
}
}
@@ -1309,7 +1309,7 @@ private[spark] class Executor(
if (runner != null) {
runner.theadDump()
} else {
- logWarning(log"Failed to dump thread for task ${LogMDC(TASK_ID, taskId)}")
+ logWarning(log"Failed to dump thread for task ${MDC(TASK_ID, taskId)}")
None
}
}
@@ -1328,8 +1328,8 @@ private[spark] object Executor extends Logging {
try {
// This tests if any class initialization error is thrown
val testKey = System.nanoTime().toString
- MDC.put(testKey, "testValue")
- MDC.remove(testKey)
+ SLF4JMDC.put(testKey, "testValue")
+ SLF4JMDC.remove(testKey)
true
} catch {
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorClassLoader.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorClassLoader.scala
index 6f8138da6f4fb..faea70933071a 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorClassLoader.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorClassLoader.scala
@@ -30,7 +30,7 @@ import org.apache.xbean.asm9.Opcodes._
import org.apache.spark.{SparkConf, SparkEnv}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.util.ParentClassLoader
/**
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorLogUrlHandler.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorLogUrlHandler.scala
index 2202489509fc4..9807cb2c81e45 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorLogUrlHandler.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorLogUrlHandler.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicBoolean
import scala.util.matching.Regex
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
private[spark] class ExecutorLogUrlHandler(logUrlPattern: Option[String]) extends Logging {
diff --git a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
index b9a462d62e413..00334e9cc55f1 100644
--- a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
@@ -96,7 +96,14 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String = "/proc/") extends L
}
Utils.tryWithResource(openReader()) { in =>
val procInfo = in.readLine
- val procInfoSplit = procInfo.split(" ")
+ // The comm field, which is inside parentheses, could contain spaces. We should not split
+ // by those spaces as doing so could cause the numbers after it to be shifted.
+ val commStartIndex = procInfo.indexOf('(')
+ val commEndIndex = procInfo.lastIndexOf(')') + 1
+ val pidArray = Array(procInfo.substring(0, commStartIndex).trim)
+ val commArray = Array(procInfo.substring(commStartIndex, commEndIndex))
+ val splitAfterComm = procInfo.substring(commEndIndex).trim.split(" ")
+ val procInfoSplit = pidArray ++ commArray ++ splitAfterComm
val vmem = procInfoSplit(22).toLong
val rssMem = procInfoSplit(23).toLong * pageSize
if (procInfoSplit(1).toLowerCase(Locale.US).contains("java")) {
diff --git a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
index 3c3017a9a64c1..9211bfec5cbd5 100644
--- a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
+++ b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
@@ -21,7 +21,7 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, Da
import scala.jdk.CollectionConverters._
-import com.google.common.io.{ByteStreams, Closeables}
+import com.google.common.io.Closeables
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapreduce.{InputSplit, JobContext, RecordReader, TaskAttemptContext}
@@ -202,7 +202,7 @@ class PortableDataStream(
def toArray(): Array[Byte] = {
val stream = open()
try {
- ByteStreams.toByteArray(stream)
+ stream.readAllBytes()
} finally {
Closeables.close(stream, true)
}
diff --git a/core/src/main/scala/org/apache/spark/input/WholeTextFileRecordReader.scala b/core/src/main/scala/org/apache/spark/input/WholeTextFileRecordReader.scala
index 0bd2d551cc912..ba975237cb934 100644
--- a/core/src/main/scala/org/apache/spark/input/WholeTextFileRecordReader.scala
+++ b/core/src/main/scala/org/apache/spark/input/WholeTextFileRecordReader.scala
@@ -17,15 +17,16 @@
package org.apache.spark.input
-import com.google.common.io.{ByteStreams, Closeables}
+import com.google.common.io.Closeables
import org.apache.hadoop.conf.{Configurable => HConfigurable, Configuration}
import org.apache.hadoop.io.Text
-import org.apache.hadoop.io.compress.CompressionCodecFactory
import org.apache.hadoop.mapreduce.InputSplit
import org.apache.hadoop.mapreduce.RecordReader
import org.apache.hadoop.mapreduce.TaskAttemptContext
import org.apache.hadoop.mapreduce.lib.input.{CombineFileRecordReader, CombineFileSplit}
+import org.apache.spark.io.HadoopCodecStreams
+
/**
* A trait to implement [[org.apache.hadoop.conf.Configurable Configurable]] interface.
*/
@@ -69,15 +70,8 @@ private[spark] class WholeTextFileRecordReader(
override def nextKeyValue(): Boolean = {
if (!processed) {
- val conf = getConf
- val factory = new CompressionCodecFactory(conf)
- val codec = factory.getCodec(path) // infers from file ext.
- val fileIn = fs.open(path)
- val innerBuffer = if (codec != null) {
- ByteStreams.toByteArray(codec.createInputStream(fileIn))
- } else {
- ByteStreams.toByteArray(fileIn)
- }
+ val fileIn = HadoopCodecStreams.createInputStream(getConf, path)
+ val innerBuffer = fileIn.readAllBytes()
value = new Text(innerBuffer)
Closeables.close(fileIn, false)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index bbd4afcaebab4..8eaa37cceee97 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -159,6 +159,13 @@ private[spark] object History {
.doubleConf
.createWithDefault(0.7d)
+ val EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED =
+ ConfigBuilder("spark.history.fs.eventLog.rolling.onDemandLoadEnabled")
+ .doc("Whether to look up rolling event log locations on demand manner before listing files.")
+ .version("4.1.0")
+ .booleanConf
+ .createWithDefault(true)
+
val DRIVER_LOG_CLEANER_ENABLED = ConfigBuilder("spark.history.fs.driverlog.cleaner.enabled")
.version("3.0.0")
.doc("Specifies whether the History Server should periodically clean up driver logs from " +
@@ -227,6 +234,7 @@ private[spark] object History {
"exceeded, then the oldest applications will be removed from the cache. If an application " +
"is not in the cache, it will have to be loaded from disk if it is accessed from the UI.")
.intConf
+ .checkValue(v => v > 0, "The number of applications to retain should be a positive integer.")
.createWithDefault(50)
val PROVIDER = ConfigBuilder("spark.history.provider")
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Python.scala b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
index 46d54be92f3d6..de95e2fa1f7a2 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Python.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
@@ -117,4 +117,25 @@ private[spark] object Python {
.version("4.1.0")
.booleanConf
.createWithDefault(false)
+
+ val PYTHON_WORKER_TRACEBACK_DUMP_INTERVAL_SECONDS =
+ ConfigBuilder("spark.python.worker.tracebackDumpIntervalSeconds")
+ .doc("The interval (in seconds) for Python workers to dump their tracebacks. " +
+ "If it's positive, the Python worker will periodically dump the traceback into " +
+ "its `stderr`. The default is `0` that means it is disabled.")
+ .version("4.1.0")
+ .timeConf(TimeUnit.SECONDS)
+ .checkValue(_ >= 0, "The interval should be 0 or positive.")
+ .createWithDefault(0)
+
+ val PYTHON_FACTORY_IDLE_WORKER_MAX_POOL_SIZE =
+ ConfigBuilder("spark.python.factory.idleWorkerMaxPoolSize")
+ .doc("Maximum number of idle Python workers to keep. " +
+ "If unset, the number is unbounded. " +
+ "If set to a positive integer N, at most N idle workers are retained; " +
+ "least-recently used workers are evicted first.")
+ .version("4.1.0")
+ .intConf
+ .checkValue(_ > 0, "If set, the idle worker max size must be > 0.")
+ .createOptional
}
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 7cb3d068b676f..c0c605dd1643f 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -224,6 +224,14 @@ package object config {
.booleanConf
.createWithDefault(false)
+ private[spark] val EVENT_LOG_EXCLUDED_PATTERNS =
+ ConfigBuilder("spark.eventLog.excludedPatterns")
+ .doc("Specifies comma-separated event names to be excluded from the event logs.")
+ .version("4.1.0")
+ .stringConf
+ .toSequence
+ .createWithDefault(Nil)
+
private[spark] val EVENT_LOG_ALLOW_EC =
ConfigBuilder("spark.eventLog.erasureCoding.enabled")
.version("3.0.0")
@@ -283,14 +291,6 @@ package object config {
.booleanConf
.createWithDefault(true)
- private[spark] val EVENT_LOG_READER_MAX_STRING_LENGTH =
- ConfigBuilder("spark.eventLog.readerMaxStringLength")
- .doc("Limit the maximum string size an eventlog item can have when deserializing it.")
- .version("4.1.0")
- .intConf
- .checkValue(_ > 0, "Maximum string size of an eventLog item should be positive.")
- .createWithDefault(Int.MaxValue)
-
private[spark] val EVENT_LOG_OVERWRITE =
ConfigBuilder("spark.eventLog.overwrite")
.version("1.0.0")
@@ -317,8 +317,8 @@ package object config {
" to be rolled over.")
.version("3.0.0")
.bytesConf(ByteUnit.BYTE)
- .checkValue(_ >= ByteUnit.MiB.toBytes(10), "Max file size of event log should be " +
- "configured to be at least 10 MiB.")
+ .checkValue(_ >= ByteUnit.MiB.toBytes(2), "Max file size of event log should be " +
+ "configured to be at least 2 MiB.")
.createWithDefaultString("128m")
private[spark] val EXECUTOR_ID =
@@ -492,6 +492,16 @@ package object config {
.doubleConf
.createWithDefault(0.6)
+ private[spark] val UNMANAGED_MEMORY_POLLING_INTERVAL =
+ ConfigBuilder("spark.memory.unmanagedMemoryPollingInterval")
+ .doc("Interval for polling unmanaged memory users to track their memory usage. " +
+ "Unmanaged memory users are components that manage their own memory outside of " +
+ "Spark's core memory management, such as RocksDB for Streaming State Store. " +
+ "Setting this to 0 disables unmanaged memory polling.")
+ .version("4.1.0")
+ .timeConf(TimeUnit.MILLISECONDS)
+ .createWithDefaultString("0s")
+
private[spark] val STORAGE_UNROLL_MEMORY_THRESHOLD =
ConfigBuilder("spark.storage.unrollMemoryThreshold")
.doc("Initial memory to request before unrolling any block")
@@ -604,6 +614,26 @@ package object config {
.checkValue(_.endsWith(java.io.File.separator), "Path should end with separator.")
.createOptional
+ private[spark] val STORAGE_DECOMMISSION_FALLBACK_STORAGE_REPLICATION_DELAY =
+ ConfigBuilder("spark.storage.decommission.fallbackStorage.replicationDelay")
+ .doc("The maximum expected delay for files written by one executor to become " +
+ "available to other executors.")
+ .version("4.1.0")
+ .timeConf(TimeUnit.MILLISECONDS)
+ .checkValue(_ > 0, "Value must be positive.")
+ .createOptional
+
+ private[spark] val STORAGE_DECOMMISSION_FALLBACK_STORAGE_REPLICATION_WAIT =
+ ConfigBuilder("spark.storage.decommission.fallbackStorage.replicationWait")
+ .doc(
+ "When an executor cannot find a file in the fallback storage it waits " +
+ "this amount of time before attempting to open the file again, " +
+ f"while not exceeding ${STORAGE_DECOMMISSION_FALLBACK_STORAGE_REPLICATION_DELAY.key}.")
+ .version("4.1.0")
+ .timeConf(TimeUnit.MILLISECONDS)
+ .checkValue(_ > 0, "Value must be positive.")
+ .createWithDefaultString("1s")
+
private[spark] val STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP =
ConfigBuilder("spark.storage.decommission.fallbackStorage.cleanUp")
.doc("If true, Spark cleans up its fallback storage data during shutting down.")
@@ -1588,6 +1618,18 @@ package object config {
.intConf
.createWithDefault(Integer.MAX_VALUE)
+ private[spark] val SHUFFLE_SPILL_MAX_SIZE_FORCE_SPILL_THRESHOLD =
+ ConfigBuilder("spark.shuffle.spill.maxRecordsSizeForSpillThreshold")
+ .internal()
+ .doc("The maximum size in memory before forcing the shuffle sorter to spill. " +
+ "By default it is Long.MAX_VALUE, which means we never force the sorter to spill, " +
+ "until we reach some limitations, like the max page size limitation for the pointer " +
+ "array in the sorter.")
+ .version("4.1.0")
+ .bytesConf(ByteUnit.BYTE)
+ .checkValue(v => v > 0, "The threshold should be positive.")
+ .createWithDefault(Long.MaxValue)
+
private[spark] val SHUFFLE_MAP_OUTPUT_PARALLEL_AGGREGATION_THRESHOLD =
ConfigBuilder("spark.shuffle.mapOutput.parallelAggregationThreshold")
.internal()
@@ -2105,6 +2147,15 @@ package object config {
.intConf
.createWithDefault(1)
+ private[spark] val IO_COMPRESSION_ZSTD_STRATEGY =
+ ConfigBuilder("spark.io.compression.zstd.strategy")
+ .doc("Compression strategy for Zstd compression codec. The higher the value is, the more " +
+ "complex it becomes, usually resulting stronger but slower compression or higher CPU " +
+ "cost.")
+ .version("4.1.0")
+ .intConf
+ .createOptional
+
private[spark] val IO_COMPRESSION_LZF_PARALLEL =
ConfigBuilder("spark.io.compression.lzf.parallel.enabled")
.doc("When true, LZF compression will use multiple threads to compress data in parallel.")
@@ -2280,6 +2331,15 @@ package object config {
.toSequence
.createWithDefault(Nil)
+ private[spark] val SUBMIT_CALL_SYSTEM_EXIT_ON_MAIN_EXIT =
+ ConfigBuilder("spark.submit.callSystemExitOnMainExit")
+ .doc("If true, SparkSubmit will call System.exit() to initiate JVM shutdown once the " +
+ "user's main method has exited. This can be useful in cases where non-daemon JVM " +
+ "threads might otherwise prevent the JVM from shutting down on its own.")
+ .version("4.1.0")
+ .booleanConf
+ .createWithDefault(false)
+
private[spark] val SCHEDULER_ALLOCATION_FILE =
ConfigBuilder("spark.scheduler.allocation.file")
.version("0.8.1")
@@ -2838,4 +2898,30 @@ package object config {
.checkValues(Set("connect", "classic"))
.createWithDefault(
if (sys.env.get("SPARK_CONNECT_MODE").contains("1")) "connect" else "classic")
+
+ private[spark] val DRIVER_REDIRECT_CONSOLE_OUTPUTS =
+ ConfigBuilder("spark.driver.log.redirectConsoleOutputs")
+ .doc("Comma-separated list of the console output kind for driver that needs to redirect " +
+ "to logging system. Supported values are `stdout`, `stderr`. It only takes affect when " +
+ s"`${PLUGINS.key}` is configured with `org.apache.spark.deploy.RedirectConsolePlugin`.")
+ .version("4.1.0")
+ .stringConf
+ .transform(_.toLowerCase(Locale.ROOT))
+ .toSequence
+ .checkValue(v => v.forall(Set("stdout", "stderr").contains),
+ "The value only can be one or more of 'stdout, stderr'.")
+ .createWithDefault(Seq("stdout", "stderr"))
+
+ private[spark] val EXEC_REDIRECT_CONSOLE_OUTPUTS =
+ ConfigBuilder("spark.executor.log.redirectConsoleOutputs")
+ .doc("Comma-separated list of the console output kind for executor that needs to redirect " +
+ "to logging system. Supported values are `stdout`, `stderr`. It only takes affect when " +
+ s"`${PLUGINS.key}` is configured with `org.apache.spark.deploy.RedirectConsolePlugin`.")
+ .version("4.1.0")
+ .stringConf
+ .transform(_.toLowerCase(Locale.ROOT))
+ .toSequence
+ .checkValue(v => v.forall(Set("stdout", "stderr").contains),
+ "The value only can be one or more of 'stdout, stderr'.")
+ .createWithDefault(Seq("stdout", "stderr"))
}
diff --git a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
index e2a96267082b8..651895bf1f7a2 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala
@@ -20,6 +20,7 @@ package org.apache.spark.internal.io
import org.apache.hadoop.fs._
import org.apache.hadoop.mapreduce._
+import org.apache.spark.SparkException
import org.apache.spark.annotation.Unstable
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
@@ -96,7 +97,9 @@ abstract class FileCommitProtocol extends Logging {
* guarantees that files written by different tasks will not conflict.
*/
@deprecated("use newTaskTempFile(..., spec: FileNameSpec) instead", "3.3.0")
- def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], ext: String): String
+ def newTaskTempFile(taskContext: TaskAttemptContext, dir: Option[String], ext: String): String = {
+ throw SparkException.mustOverrideOneMethodError("newTaskTempFile")
+ }
/**
* Notifies the commit protocol to add a new file, and gets back the full path that should be
@@ -135,7 +138,9 @@ abstract class FileCommitProtocol extends Logging {
*/
@deprecated("use newTaskTempFileAbsPath(..., spec: FileNameSpec) instead", "3.3.0")
def newTaskTempFileAbsPath(
- taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String
+ taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String = {
+ throw SparkException.mustOverrideOneMethodError("newTaskTempFileAbsPath")
+ }
/**
* Similar to newTaskTempFile(), but allows files to committed to an absolute output location.
diff --git a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapRedCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapRedCommitProtocol.scala
index 44f8d7cd63635..358424abefba5 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapRedCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapRedCommitProtocol.scala
@@ -21,7 +21,6 @@ import org.apache.hadoop.mapred._
import org.apache.hadoop.mapreduce.{TaskAttemptContext => NewTaskAttemptContext}
import org.apache.spark.internal.LogKeys
-import org.apache.spark.internal.MDC
/**
* An [[FileCommitProtocol]] implementation backed by an underlying Hadoop OutputCommitter
diff --git a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
index 476cddc643954..d7bda5bbe721a 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala
@@ -29,7 +29,7 @@ import org.apache.hadoop.mapreduce._
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.mapred.SparkHadoopMapRedUtil
@@ -117,11 +117,6 @@ class HadoopMapReduceCommitProtocol(
format.getOutputCommitter(context)
}
- override def newTaskTempFile(
- taskContext: TaskAttemptContext, dir: Option[String], ext: String): String = {
- newTaskTempFile(taskContext, dir, FileNameSpec("", ext))
- }
-
override def newTaskTempFile(
taskContext: TaskAttemptContext, dir: Option[String], spec: FileNameSpec): String = {
val filename = getFilename(taskContext, spec)
@@ -145,11 +140,6 @@ class HadoopMapReduceCommitProtocol(
}
}
- override def newTaskTempFileAbsPath(
- taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String = {
- newTaskTempFileAbsPath(taskContext, absoluteDir, FileNameSpec("", ext))
- }
-
override def newTaskTempFileAbsPath(
taskContext: TaskAttemptContext, absoluteDir: String, spec: FileNameSpec): String = {
val filename = getFilename(taskContext, spec)
diff --git a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
index db961b3c42f4c..de136a86a2cc3 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
@@ -32,7 +32,7 @@ import org.apache.hadoop.mapreduce.task.{TaskAttemptContextImpl => NewTaskAttemp
import org.apache.spark.{SerializableWritable, SparkConf, SparkException, TaskContext}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{DURATION, JOB_ID, TASK_ATTEMPT_ID}
import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage
import org.apache.spark.rdd.{HadoopRDD, RDD}
diff --git a/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala b/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala
index a0c07bd75f885..51b29e1a0ec68 100644
--- a/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala
+++ b/core/src/main/scala/org/apache/spark/internal/plugin/PluginContainer.scala
@@ -22,7 +22,7 @@ import scala.util.{Either, Left, Right}
import org.apache.spark.{SparkContext, SparkEnv, TaskFailedReason}
import org.apache.spark.api.plugin._
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config._
import org.apache.spark.resource.ResourceInformation
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/internal/plugin/PluginEndpoint.scala b/core/src/main/scala/org/apache/spark/internal/plugin/PluginEndpoint.scala
index 6ff918979c9ed..2c2f2f022a7a3 100644
--- a/core/src/main/scala/org/apache/spark/internal/plugin/PluginEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/internal/plugin/PluginEndpoint.scala
@@ -18,7 +18,7 @@
package org.apache.spark.internal.plugin
import org.apache.spark.api.plugin.DriverPlugin
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rpc.{IsolatedThreadSafeRpcEndpoint, RpcCallContext, RpcEnv}
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
index 233228a9c6d4c..b81e46667323e 100644
--- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
+++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
@@ -228,6 +228,7 @@ class ZStdCompressionCodec(conf: SparkConf) extends CompressionCodec {
// Default compression level for zstd compression to 1 because it is
// fastest of all with reasonably high compression ratio.
private val level = conf.get(IO_COMPRESSION_ZSTD_LEVEL)
+ private val strategy = conf.get(IO_COMPRESSION_ZSTD_STRATEGY)
private val bufferPool = if (conf.get(IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED)) {
RecyclingBufferPool.INSTANCE
@@ -241,6 +242,7 @@ class ZStdCompressionCodec(conf: SparkConf) extends CompressionCodec {
// Wrap the zstd output stream in a buffered output stream, so that we can
// avoid overhead excessive of JNI call while trying to compress small amount of data.
val os = new ZstdOutputStreamNoFinalizer(s, bufferPool).setLevel(level).setWorkers(workers)
+ strategy.foreach(os.setStrategy)
new BufferedOutputStream(os, bufferSize)
}
diff --git a/core/src/main/scala/org/apache/spark/io/HadoopCodecStreams.scala b/core/src/main/scala/org/apache/spark/io/HadoopCodecStreams.scala
new file mode 100644
index 0000000000000..cba0e7ffd73bf
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/io/HadoopCodecStreams.scala
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.io
+
+import java.io.InputStream
+import java.util.Locale
+
+import scala.collection.Seq
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.io.compress._
+
+import org.apache.spark.{SparkConf, SparkEnv}
+import org.apache.spark.io.{CompressionCodec => SparkCompressionCodec}
+
+/**
+ * An utility object to look up Hadoop compression codecs and create input streams.
+ * In addition to standard Hadoop codecs, it also supports Spark's Zstandard codec
+ * if Hadopp is not compiled with Zstandard support. Additionally, it supports
+ * non-standard file extensions like `.zstd` and `.gzip` for Zstandard and Gzip codecs.
+ */
+object HadoopCodecStreams {
+ private val ZSTD_EXTENSIONS = Seq(".zstd", ".zst")
+
+ // get codec based on file name extension
+ def getDecompressionCodec(
+ config: Configuration,
+ file: Path): Option[CompressionCodec] = {
+ val factory = new CompressionCodecFactory(config)
+ Option(factory.getCodec(file)).orElse {
+ // Try some non-standards extensions for Zstandard and Gzip
+ file.getName.toLowerCase() match {
+ case name if name.endsWith(".zstd") =>
+ Option(factory.getCodecByName(classOf[ZStandardCodec].getName))
+ case name if name.endsWith(".gzip") =>
+ Option(factory.getCodecByName(classOf[GzipCodec].getName))
+ case _ => None
+ }
+ }
+ }
+
+ def createZstdInputStream(
+ file: Path,
+ inputStream: InputStream): Option[InputStream] = {
+ val sparkConf = Option(SparkEnv.get).map(_.conf).getOrElse(new SparkConf)
+ val fileName = file.getName.toLowerCase(Locale.ROOT)
+
+ val isOpt = if (ZSTD_EXTENSIONS.exists(fileName.endsWith)) {
+ Some(
+ SparkCompressionCodec
+ .createCodec(sparkConf, SparkCompressionCodec.ZSTD)
+ .compressedInputStream(inputStream)
+ )
+ } else {
+ None
+ }
+ isOpt
+ }
+
+ def createInputStream(
+ config: Configuration,
+ file: Path): InputStream = {
+ val fs = file.getFileSystem(config)
+ val inputStream: InputStream = fs.open(file)
+
+ getDecompressionCodec(config, file)
+ .map { codec =>
+ try {
+ codec.createInputStream(inputStream)
+ } catch {
+ case e: RuntimeException =>
+ // createInputStream may fail for ZSTD if hadoop is not already compiled with ZSTD
+ // support. In that case, we try to use Spark's Zstandard codec.
+ createZstdInputStream(file, inputStream).getOrElse(throw e)
+ }
+ }.getOrElse(inputStream)
+ }
+}
diff --git a/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala b/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala
index 0aaa222e6195e..e9745a7c2ba67 100644
--- a/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala
+++ b/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala
@@ -24,7 +24,7 @@ import org.apache.hadoop.mapreduce.{OutputCommitter => MapReduceOutputCommitter}
import org.apache.spark.{SparkEnv, TaskContext}
import org.apache.spark.executor.CommitDeniedException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{TASK_ATTEMPT_ID, TOTAL_TIME}
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala b/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala
index 7098961d1649a..8b31d02cf97e3 100644
--- a/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala
+++ b/core/src/main/scala/org/apache/spark/memory/ExecutionMemoryPool.scala
@@ -21,7 +21,7 @@ import javax.annotation.concurrent.GuardedBy
import scala.collection.mutable
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
/**
diff --git a/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala b/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala
index 24fcb5b17f388..7cde44b553321 100644
--- a/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala
+++ b/core/src/main/scala/org/apache/spark/memory/StorageMemoryPool.scala
@@ -20,7 +20,7 @@ package org.apache.spark.memory
import javax.annotation.concurrent.GuardedBy
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.storage.BlockId
import org.apache.spark.storage.memory.MemoryStore
diff --git a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala
index d4ec6ed8495af..db51f14415e1a 100644
--- a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala
+++ b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala
@@ -17,11 +17,19 @@
package org.apache.spark.memory
+import java.util.concurrent.{ConcurrentHashMap, ScheduledExecutorService, TimeUnit}
+import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong}
+
+import scala.jdk.CollectionConverters._
+import scala.util.control.NonFatal
+
import org.apache.spark.{SparkConf, SparkIllegalArgumentException}
-import org.apache.spark.internal.{config, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.Tests._
+import org.apache.spark.internal.config.UNMANAGED_MEMORY_POLLING_INTERVAL
import org.apache.spark.storage.BlockId
+import org.apache.spark.util.{ThreadUtils, Utils}
/**
* A [[MemoryManager]] that enforces a soft boundary between execution and storage such that
@@ -56,7 +64,47 @@ private[spark] class UnifiedMemoryManager(
conf,
numCores,
onHeapStorageRegionSize,
- maxHeapMemory - onHeapStorageRegionSize) {
+ maxHeapMemory - onHeapStorageRegionSize) with Logging {
+
+ /**
+ * Unmanaged memory tracking infrastructure.
+ *
+ * Unmanaged memory refers to memory consumed by components that manage their own memory
+ * outside of Spark's unified memory management system. Examples include:
+ * - RocksDB state stores used in structured streaming
+ * - Native libraries with their own memory management
+ * - Off-heap caches managed by unmanaged systems
+ *
+ * We track this memory to:
+ * 1. Provide visibility into total memory usage on executors
+ * 2. Prevent OOM errors by accounting for it in memory allocation decisions
+ * 3. Enable better debugging and monitoring of memory-intensive applications
+ *
+ * The polling mechanism periodically queries registered unmanaged memory consumers
+ * to detect inactive consumers and handle cleanup.
+ */
+ // Configuration for polling interval (in milliseconds)
+ private val unmanagedMemoryPollingIntervalMs = conf.get(UNMANAGED_MEMORY_POLLING_INTERVAL)
+ // Initialize background polling if enabled
+ if (unmanagedMemoryPollingIntervalMs > 0) {
+ UnifiedMemoryManager.startPollingIfNeeded(unmanagedMemoryPollingIntervalMs)
+ }
+
+ /**
+ * Get the current unmanaged memory usage in bytes for a specific memory mode.
+ * @param memoryMode The memory mode (ON_HEAP or OFF_HEAP) to get usage for
+ * @return The current unmanaged memory usage in bytes
+ */
+ private def getUnmanagedMemoryUsed(memoryMode: MemoryMode): Long = {
+ // Only consider unmanaged memory if polling is enabled
+ if (unmanagedMemoryPollingIntervalMs <= 0) {
+ return 0L
+ }
+ memoryMode match {
+ case MemoryMode.ON_HEAP => UnifiedMemoryManager.unmanagedOnHeapUsed.get()
+ case MemoryMode.OFF_HEAP => UnifiedMemoryManager.unmanagedOffHeapUsed.get()
+ }
+ }
private def assertInvariants(): Unit = {
assert(onHeapExecutionMemoryPool.poolSize + onHeapStorageMemoryPool.poolSize == maxHeapMemory)
@@ -140,9 +188,15 @@ private[spark] class UnifiedMemoryManager(
* in execution memory allocation across tasks, Otherwise, a task may occupy more than
* its fair share of execution memory, mistakenly thinking that other tasks can acquire
* the portion of storage memory that cannot be evicted.
+ *
+ * This also factors in unmanaged memory usage to ensure we don't over-allocate memory
+ * when unmanaged components are consuming significant memory.
*/
def computeMaxExecutionPoolSize(): Long = {
- maxMemory - math.min(storagePool.memoryUsed, storageRegionSize)
+ val unmanagedMemory = getUnmanagedMemoryUsed(memoryMode)
+ val availableMemory = maxMemory - math.min(storagePool.memoryUsed, storageRegionSize)
+ // Reduce available memory by unmanaged memory usage to prevent over-allocation
+ math.max(0L, availableMemory - unmanagedMemory)
}
executionPool.acquireMemory(
@@ -165,11 +219,21 @@ private[spark] class UnifiedMemoryManager(
offHeapStorageMemoryPool,
maxOffHeapStorageMemory)
}
- if (numBytes > maxMemory) {
+
+ // Factor in unmanaged memory usage for the specific memory mode
+ val unmanagedMemory = getUnmanagedMemoryUsed(memoryMode)
+ val effectiveMaxMemory = math.max(0L, maxMemory - unmanagedMemory)
+
+ if (numBytes > effectiveMaxMemory) {
// Fail fast if the block simply won't fit
logInfo(log"Will not store ${MDC(BLOCK_ID, blockId)} as the required space" +
log" (${MDC(NUM_BYTES, numBytes)} bytes) exceeds our" +
- log" memory limit (${MDC(NUM_BYTES_MAX, maxMemory)} bytes)")
+ log" memory limit (${MDC(NUM_BYTES_MAX, effectiveMaxMemory)} bytes)" +
+ (if (unmanagedMemory > 0) {
+ log" (unmanaged memory usage: ${MDC(NUM_BYTES, unmanagedMemory)} bytes)"
+ } else {
+ log""
+ }))
return false
}
if (numBytes > storagePool.memoryFree) {
@@ -191,7 +255,7 @@ private[spark] class UnifiedMemoryManager(
}
}
-object UnifiedMemoryManager {
+object UnifiedMemoryManager extends Logging {
// Set aside a fixed amount of memory for non-storage, non-execution purposes.
// This serves a function similar to `spark.memory.fraction`, but guarantees that we reserve
@@ -199,6 +263,181 @@ object UnifiedMemoryManager {
// the memory used for execution and storage will be (1024 - 300) * 0.6 = 434MB by default.
private val RESERVED_SYSTEM_MEMORY_BYTES = 300 * 1024 * 1024
+ private val unmanagedMemoryConsumers =
+ new ConcurrentHashMap[UnmanagedMemoryConsumerId, UnmanagedMemoryConsumer]
+
+ // Cached unmanaged memory usage values updated by polling
+ private val unmanagedOnHeapUsed = new AtomicLong(0L)
+ private val unmanagedOffHeapUsed = new AtomicLong(0L)
+
+ // Atomic flag to ensure polling is only started once per JVM
+ private val pollingStarted = new AtomicBoolean(false)
+
+ /**
+ * Register an unmanaged memory consumer to track its memory usage.
+ *
+ * Unmanaged memory consumers are components that manage their own memory outside
+ * of Spark's unified memory management system. By registering, their memory usage
+ * will be periodically polled and factored into Spark's memory allocation decisions.
+ *
+ * @param unmanagedMemoryConsumer The consumer to register for memory tracking
+ */
+ def registerUnmanagedMemoryConsumer(
+ unmanagedMemoryConsumer: UnmanagedMemoryConsumer): Unit = {
+ val id = unmanagedMemoryConsumer.unmanagedMemoryConsumerId
+ unmanagedMemoryConsumers.put(id, unmanagedMemoryConsumer)
+ }
+
+ /**
+ * Unregister an unmanaged memory consumer.
+ * This should be called when a component is shutting down to prevent memory leaks
+ * and ensure accurate memory tracking.
+ *
+ * @param unmanagedMemoryConsumer The consumer to unregister. Only used in tests
+ */
+ private[spark] def unregisterUnmanagedMemoryConsumer(
+ unmanagedMemoryConsumer: UnmanagedMemoryConsumer): Unit = {
+ val id = unmanagedMemoryConsumer.unmanagedMemoryConsumerId
+ unmanagedMemoryConsumers.remove(id)
+ }
+
+
+ /**
+ * Get the current memory usage in bytes for a specific component type.
+ * @param componentType The type of component to filter by (e.g., "RocksDB")
+ * @return Total memory usage in bytes for the specified component type
+ */
+ def getMemoryByComponentType(componentType: String): Long = {
+ unmanagedMemoryConsumers.asScala.values.toSeq
+ .filter(_.unmanagedMemoryConsumerId.componentType == componentType)
+ .map { memoryUser =>
+ try {
+ memoryUser.getMemBytesUsed
+ } catch {
+ case e: Exception =>
+ 0L
+ }
+ }
+ .sum
+ }
+
+ /**
+ * Clear all unmanaged memory users.
+ * This is useful during executor shutdown or cleanup.
+ * Since each executor runs in its own JVM, this clears all users for this executor.
+ */
+ def clearUnmanagedMemoryUsers(): Unit = {
+ unmanagedMemoryConsumers.clear()
+ // Reset cached values when clearing consumers
+ unmanagedOnHeapUsed.set(0L)
+ unmanagedOffHeapUsed.set(0L)
+ }
+
+ // Shared polling infrastructure - only one polling thread per JVM
+ @volatile private var unmanagedMemoryPoller: ScheduledExecutorService = _
+
+ /**
+ * Start unmanaged memory polling if not already started.
+ * This ensures only one polling thread is created per JVM, regardless of how many
+ * UnifiedMemoryManager instances are created.
+ */
+ private[memory] def startPollingIfNeeded(pollingIntervalMs: Long): Unit = {
+ if (pollingStarted.compareAndSet(false, true)) {
+ unmanagedMemoryPoller = ThreadUtils.newDaemonSingleThreadScheduledExecutor(
+ "unmanaged-memory-poller")
+
+ val pollingTask = new Runnable {
+ override def run(): Unit = Utils.tryLogNonFatalError {
+ pollUnmanagedMemoryUsers()
+ }
+ }
+
+ unmanagedMemoryPoller.scheduleAtFixedRate(
+ pollingTask,
+ 0L, // initial delay
+ pollingIntervalMs,
+ TimeUnit.MILLISECONDS)
+
+ logInfo(log"Unmanaged memory polling started with interval " +
+ log"${MDC(LogKeys.TIME, pollingIntervalMs)}ms")
+ }
+ }
+
+ private def pollUnmanagedMemoryUsers(): Unit = {
+ val consumers = unmanagedMemoryConsumers.asScala.toMap
+
+ // Get memory usage for each consumer, handling failures gracefully
+ val memoryUsages = consumers.map { case (userId, memoryUser) =>
+ try {
+ val memoryUsed = memoryUser.getMemBytesUsed
+ if (memoryUsed == -1L) {
+ logDebug(log"Unmanaged memory consumer ${MDC(LogKeys.OBJECT_ID, userId.toString)} " +
+ log"is no longer active, marking for removal")
+ (userId, memoryUser, None) // Mark for removal
+ } else if (memoryUsed < 0L) {
+ logWarning(log"Invalid memory usage value ${MDC(LogKeys.NUM_BYTES, memoryUsed)} " +
+ log"from unmanaged memory user ${MDC(LogKeys.OBJECT_ID, userId.toString)}")
+ (userId, memoryUser, Some(0L)) // Treat as 0
+ } else {
+ (userId, memoryUser, Some(memoryUsed))
+ }
+ } catch {
+ case NonFatal(e) =>
+ logWarning(log"Failed to get memory usage for unmanaged memory user " +
+ log"${MDC(LogKeys.OBJECT_ID, userId.toString)} ${MDC(LogKeys.EXCEPTION, e)}")
+ (userId, memoryUser, Some(0L)) // Treat as 0 on error
+ }
+ }
+
+ // Remove inactive consumers
+ memoryUsages.filter(_._3.isEmpty).foreach { case (userId, _, _) =>
+ unmanagedMemoryConsumers.remove(userId)
+ logInfo(log"Removed inactive unmanaged memory consumer " +
+ log"${MDC(LogKeys.OBJECT_ID, userId.toString)}")
+ }
+ // Calculate total memory usage by mode
+ val activeUsages = memoryUsages.filter(_._3.isDefined)
+ val onHeapTotal = activeUsages
+ .filter(_._2.memoryMode == MemoryMode.ON_HEAP)
+ .map(_._3.get)
+ .sum
+ val offHeapTotal = activeUsages
+ .filter(_._2.memoryMode == MemoryMode.OFF_HEAP)
+ .map(_._3.get)
+ .sum
+ // Update cached values atomically
+ unmanagedOnHeapUsed.set(onHeapTotal)
+ unmanagedOffHeapUsed.set(offHeapTotal)
+ // Log polling results for monitoring
+ val totalMemoryUsed = onHeapTotal + offHeapTotal
+ val numConsumers = activeUsages.size
+ logDebug(s"Unmanaged memory polling completed: $numConsumers consumers, " +
+ s"total memory used: ${totalMemoryUsed} bytes " +
+ s"(on-heap: ${onHeapTotal}, off-heap: ${offHeapTotal})")
+ }
+
+ /**
+ * Shutdown the unmanaged memory polling thread. Only used in tests
+ */
+ private[spark] def shutdownUnmanagedMemoryPoller(): Unit = {
+ synchronized {
+ if (unmanagedMemoryPoller != null) {
+ unmanagedMemoryPoller.shutdown()
+ try {
+ if (!unmanagedMemoryPoller.awaitTermination(5, TimeUnit.SECONDS)) {
+ unmanagedMemoryPoller.shutdownNow()
+ }
+ } catch {
+ case _: InterruptedException =>
+ Thread.currentThread().interrupt()
+ }
+ unmanagedMemoryPoller = null
+ pollingStarted.set(false)
+ logInfo(log"Unmanaged memory poller shutdown complete")
+ }
+ }
+ }
+
def apply(conf: SparkConf, numCores: Int): UnifiedMemoryManager = {
val maxMemory = getMaxMemory(conf)
new UnifiedMemoryManager(
diff --git a/core/src/main/scala/org/apache/spark/memory/UnmanagedMemoryConsumer.scala b/core/src/main/scala/org/apache/spark/memory/UnmanagedMemoryConsumer.scala
new file mode 100644
index 0000000000000..8351918282156
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/memory/UnmanagedMemoryConsumer.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.memory
+
+/**
+ * Identifier for an unmanaged memory consumer.
+ *
+ * @param componentType The type of component (e.g., "RocksDB", "NativeLibrary")
+ * @param instanceKey A unique key to identify this specific instance of the component.
+ * For shared memory consumers, this should be a common key across
+ * all instances to avoid double counting.
+ */
+case class UnmanagedMemoryConsumerId(
+ componentType: String,
+ instanceKey: String
+)
+
+/**
+ * Interface for components that consume memory outside of Spark's unified memory management.
+ *
+ * Components implementing this trait can register themselves with the memory manager
+ * to have their memory usage tracked and factored into memory allocation decisions.
+ * This helps prevent OOM errors when unmanaged components use significant memory.
+ *
+ * Examples of unmanaged memory consumers:
+ * - RocksDB state stores in structured streaming
+ * - Native libraries with custom memory allocation
+ * - Off-heap caches managed outside of Spark
+ */
+trait UnmanagedMemoryConsumer {
+ /**
+ * Returns the unique identifier for this memory consumer.
+ * The identifier is used to track and manage the consumer in the memory tracking system.
+ */
+ def unmanagedMemoryConsumerId: UnmanagedMemoryConsumerId
+
+ /**
+ * Returns the memory mode (ON_HEAP or OFF_HEAP) that this consumer uses.
+ * This is used to ensure unmanaged memory usage only affects the correct memory pool.
+ */
+ def memoryMode: MemoryMode
+
+ /**
+ * Returns the current memory usage in bytes.
+ *
+ * This method is called periodically by the memory polling mechanism to track
+ * memory usage over time. Implementations should return the current total memory
+ * consumed by this component.
+ *
+ * @return Current memory usage in bytes. Should return 0 if no memory is currently used.
+ * Return -1L to indicate this consumer is no longer active and should be
+ * automatically removed from tracking.
+ * @throws Exception if memory usage cannot be determined. The polling mechanism
+ * will handle exceptions gracefully and log warnings.
+ */
+ def getMemBytesUsed: Long
+}
diff --git a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
index 965468ac2418f..b192a9f9babd1 100644
--- a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala
@@ -24,7 +24,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.SparkEnv
import org.apache.spark.executor.ProcfsMetricsGetter
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.memory.MemoryManager
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
index a845feeb67ff0..b975cefb336c2 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import scala.util.matching.Regex
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PATH
import org.apache.spark.internal.config.METRICS_CONF
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
index 709ce0060e150..af1f54007248d 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
@@ -26,7 +26,7 @@ import com.codahale.metrics.{Metric, MetricRegistry}
import org.eclipse.jetty.servlet.ServletContextHandler
import org.apache.spark.{SecurityManager, SparkConf}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config._
import org.apache.spark.metrics.sink.{MetricsServlet, PrometheusServlet, Sink}
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala
index 30b10d64882ac..4b74354aa15e3 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/StatsdSink.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit
import com.codahale.metrics.{Metric, MetricFilter, MetricRegistry}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PREFIX
import org.apache.spark.metrics.MetricsSystem
diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
index a922eb336c28f..03810292bd05c 100644
--- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
+++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
@@ -23,7 +23,7 @@ import scala.jdk.CollectionConverters._
import scala.reflect.ClassTag
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.BlockDataManager
import org.apache.spark.network.buffer.NioManagedBuffer
diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
index 7ceb50db5966a..fe55518d80004 100644
--- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
+++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
@@ -30,7 +30,7 @@ import com.codahale.metrics.{Metric, MetricSet}
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.ExecutorDeadException
-import org.apache.spark.internal.{config, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.network._
import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer}
import org.apache.spark.network.client.{RpcResponseCallback, TransportClientBootstrap}
@@ -57,7 +57,7 @@ private[spark] class NettyBlockTransferService(
_port: Int,
numCores: Int,
driverEndPointRef: RpcEndpointRef = null)
- extends BlockTransferService {
+ extends BlockTransferService with Logging {
// TODO: Don't use Java serialization, use a more cross-version compatible serialization format.
private val serializer = serializerManager.getSerializer(scala.reflect.classTag[Any], false)
@@ -196,9 +196,9 @@ private[spark] class NettyBlockTransferService(
override def onFailure(e: Throwable): Unit = {
if (asStream) {
- logger.error(s"Error while uploading {} as stream", e, MDC.of(LogKeys.BLOCK_ID, blockId))
+ logger.error(s"Error while uploading {} as stream", e, MDC(LogKeys.BLOCK_ID, blockId))
} else {
- logger.error(s"Error while uploading {}", e, MDC.of(LogKeys.BLOCK_ID, blockId))
+ logger.error(s"Error while uploading {}", e, MDC(LogKeys.BLOCK_ID, blockId))
}
result.failure(e)
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
index 44ce1bfa8c5c8..a16bd98a74669 100644
--- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
@@ -39,7 +39,7 @@ import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.rdd.HadoopRDD.HadoopMapPartitionsWithSplitRDD
diff --git a/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala b/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
index 8c10bcbc25a86..e71cf4d002a74 100644
--- a/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
@@ -25,7 +25,7 @@ import org.apache.spark.{Partition, SparkContext, TaskContext}
import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
import org.apache.spark.api.java.function.{Function => JFunction}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.NextIterator
@@ -54,7 +54,9 @@ private[spark] class JdbcPartition(idx: Int, val lower: Long, val upper: Long) e
* @param mapRow a function from a ResultSet to a single row of the desired result type(s).
* This should only call getInt, getString, etc; the RDD takes care of calling next.
* The default maps a ResultSet to an array of Object.
+ * @deprecated Jdbc RDD is deprecated, consider using JDBC data source instead.
*/
+@deprecated("Jdbc RDD is deprecated, consider using JDBC data source instead.", "4.1.0")
class JdbcRDD[T: ClassTag](
sc: SparkContext,
getConnection: () => Connection,
diff --git a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
index 27e4948ce1297..cdf90a03a554a 100644
--- a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
@@ -39,7 +39,7 @@ import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.rdd.NewHadoopRDD.NewHadoopMapPartitionsWithSplitRDD
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
index c0966dd5ede14..956620e0b2c23 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -35,7 +35,7 @@ import org.apache.hadoop.mapreduce.{Job => NewAPIHadoopJob, OutputFormat => NewO
import org.apache.spark._
import org.apache.spark.Partitioner.defaultPartitioner
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.SPECULATION_ENABLED
import org.apache.spark.internal.io._
diff --git a/core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala b/core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala
index 965618ee827d1..99b1546f858c1 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala
@@ -47,23 +47,12 @@ class PartitionerAwareUnionRDDPartition(
}
}
-/**
- * Class representing an RDD that can take multiple RDDs partitioned by the same partitioner and
- * unify them into a single RDD while preserving the partitioner. So m RDDs with p partitions each
- * will be unified to a single RDD with p partitions and the same partitioner. The preferred
- * location for each partition of the unified RDD will be the most common preferred location
- * of the corresponding partitions of the parent RDDs. For example, location of partition 0
- * of the unified RDD will be where most of partition 0 of the parent RDDs are located.
- */
private[spark]
-class PartitionerAwareUnionRDD[T: ClassTag](
+abstract class PartitionerAwareUnionRDDBase[T: ClassTag](
sc: SparkContext,
var rdds: Seq[RDD[T]]
) extends RDD[T](sc, rdds.map(x => new OneToOneDependency(x))) {
- require(rdds.nonEmpty)
- require(rdds.forall(_.partitioner.isDefined))
- require(rdds.flatMap(_.partitioner).toSet.size == 1,
- "Parent RDDs have different partitioners: " + rdds.flatMap(_.partitioner))
+ require(rdds.nonEmpty, "RDDs cannot be empty")
override val partitioner = rdds.head.partitioner
@@ -111,3 +100,49 @@ class PartitionerAwareUnionRDD[T: ClassTag](
rdd.context.getPreferredLocs(rdd, part.index).map(tl => tl.host)
}
}
+
+/**
+ * Class representing an RDD that can take multiple RDDs partitioned by the same partitioner and
+ * unify them into a single RDD while preserving the partitioner. So m RDDs with p partitions each
+ * will be unified to a single RDD with p partitions and the same partitioner. The preferred
+ * location for each partition of the unified RDD will be the most common preferred location
+ * of the corresponding partitions of the parent RDDs. For example, location of partition 0
+ * of the unified RDD will be where most of partition 0 of the parent RDDs are located.
+ */
+private[spark]
+class PartitionerAwareUnionRDD[T: ClassTag](
+ sc: SparkContext,
+ var _rdds: Seq[RDD[T]]
+ ) extends PartitionerAwareUnionRDDBase(sc, _rdds) {
+ require(_rdds.forall(_.partitioner.isDefined))
+ require(_rdds.flatMap(_.partitioner).toSet.size == 1,
+ "Parent RDDs have different partitioners: " + _rdds.flatMap(_.partitioner))
+}
+
+/**
+ * This is similar to [[PartitionerAwareUnionRDD]], but it doesn't require the parent RDDs
+ * to have defined partitioner and have the same partitioner if defined.
+ * It is because SQL's shuffle RDD's partitioner is not defined in `ShuffledRowRDD`.
+ * The actual partitioning is implemented in `ShuffleExchangeExec.prepareShuffleDependency`.
+ *
+ * Thus, this RDD doesn't check the partitioner of parent RDDs. Its correctness relies on the
+ * fact that the given RDDs are partitioned in the same way. So before using this RDD, you must
+ * ensure that all parent RDDs are partitioned correctly by checking their SQL output partitioning.
+ */
+private[spark]
+class SQLPartitioningAwareUnionRDD[T: ClassTag](
+ sc: SparkContext,
+ var _rdds: Seq[RDD[T]],
+ val numPartitions: Int
+ ) extends PartitionerAwareUnionRDDBase(sc, _rdds) {
+ require(partitioner.isEmpty || partitioner.get.numPartitions == numPartitions,
+ "Partitioner of parent RDDs does not match the number of partitions: " +
+ s"expected $numPartitions, but got ${partitioner.map(_.numPartitions).getOrElse("none")}")
+
+ override def getPartitions: Array[Partition] = {
+ (0 until numPartitions).map { index =>
+ new PartitionerAwareUnionRDDPartition(_rdds, index)
+ }.toArray
+ }
+}
+
diff --git a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
index 126c92e4cb656..5a9e51026ba98 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
@@ -35,7 +35,6 @@ import scala.reflect.ClassTag
import org.apache.spark.{Partition, TaskContext}
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.internal.LogKeys.{COMMAND, ERROR, PATH}
-import org.apache.spark.internal.MDC
import org.apache.spark.util.Utils
@@ -90,7 +89,7 @@ private[spark] class PipedRDD[T: ClassTag](
val currentDir = new File(".")
logDebug("currentDir = " + currentDir.getAbsolutePath())
val taskDirFile = new File(taskDirectory)
- taskDirFile.mkdirs()
+ Utils.createDirectory(taskDirFile)
try {
val tasksDirFilter = new NotEqualsFileNameFilter("tasks")
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 80db818b77e42..117b2925710d3 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -36,7 +36,7 @@ import org.apache.spark.Partitioner._
import org.apache.spark.annotation.{DeveloperApi, Experimental, Since}
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.RDD_LIMIT_SCALE_UP_FACTOR
@@ -2016,7 +2016,7 @@ abstract class RDD[T: ClassTag](
def firstDebugString(rdd: RDD[_]): Seq[String] = {
val partitionStr = "(" + rdd.partitions.length + ")"
val leftOffset = (partitionStr.length - 1) / 2
- val nextPrefix = (" " * leftOffset) + "|" + (" " * (partitionStr.length - leftOffset))
+ val nextPrefix = " ".repeat(leftOffset) + "|" + " ".repeat(partitionStr.length - leftOffset)
debugSelf(rdd).zipWithIndex.map{
case (desc: String, 0) => s"$partitionStr $desc"
@@ -2030,7 +2030,7 @@ abstract class RDD[T: ClassTag](
val nextPrefix = (
thisPrefix
+ (if (isLastChild) " " else "| ")
- + (" " * leftOffset) + "|" + (" " * (partitionStr.length - leftOffset)))
+ + (" ".repeat(leftOffset)) + "|" + (" ".repeat(partitionStr.length - leftOffset)))
debugSelf(rdd).zipWithIndex.map{
case (desc: String, 0) => s"$thisPrefix+-$partitionStr $desc"
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
index 3abb2d8a11f35..49c259999a471 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
@@ -17,13 +17,13 @@
package org.apache.spark.rdd
+import java.util.Objects
import java.util.concurrent.atomic.AtomicInteger
import com.fasterxml.jackson.annotation.{JsonIgnore, JsonInclude, JsonPropertyOrder}
import com.fasterxml.jackson.annotation.JsonInclude.Include
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
-import com.google.common.base.Objects
import org.apache.spark.SparkContext
import org.apache.spark.internal.Logging
@@ -69,7 +69,7 @@ private[spark] class RDDOperationScope(
}
}
- override def hashCode(): Int = Objects.hashCode(id, name, parent)
+ override def hashCode(): Int = Objects.hash(id, name, parent)
override def toString: String = toJson
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala b/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
index ee13acc650337..fd42cea795d60 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path
import org.apache.spark._
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{BUFFER_SIZE, CACHE_CHECKPOINT_PREFERRED_LOCS_EXPIRE_TIME, CHECKPOINT_COMPRESS}
import org.apache.spark.io.CompressionCodec
diff --git a/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala b/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala
index b468a38fcf229..9f80d53d2c0bf 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ReliableRDDCheckpointData.scala
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NEW_RDD_ID, RDD_CHECKPOINT_DIR, RDD_ID}
import org.apache.spark.internal.config.CLEANER_REFERENCE_TRACKING_CLEAN_CHECKPOINTS
diff --git a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
index 118660ef69476..14e8bade2200a 100644
--- a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
@@ -23,7 +23,7 @@ import org.apache.hadoop.io.compress.CompressionCodec
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapred.SequenceFileOutputFormat
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
/**
* Extra functions available on RDDs of (key, value) pairs to create a Hadoop SequenceFile,
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
index 51de7e2b9ac70..b11c88828e065 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
@@ -23,7 +23,7 @@ import java.util.Optional
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.api.resource.ResourceDiscoveryPlugin
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.util.Utils.executeAndGetOutput
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
index 0baa578764d08..1b4b4f61016a4 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
@@ -26,7 +26,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.{SparkConf, SparkContext, SparkEnv, SparkException}
import org.apache.spark.annotation.{Evolving, Since}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.internal.config.Python.PYSPARK_EXECUTOR_MEMORY
@@ -106,6 +106,14 @@ class ResourceProfile(
executorResources.get(ResourceProfile.PYSPARK_MEM).map(_.amount)
}
+ private[spark] def getOverheadMemory: Option[Long] = {
+ executorResources.get(ResourceProfile.OVERHEAD_MEM).map(_.amount)
+ }
+
+ private[spark] def getExecutorOffHeap: Option[Long] = {
+ executorResources.get(ResourceProfile.OFFHEAP_MEM).map(_.amount)
+ }
+
private[spark] def getExecutorMemory: Option[Long] = {
executorResources.get(ResourceProfile.MEMORY).map(_.amount)
}
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala
index 6a6b5067f70f2..10121f6ef2667 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfileManager.scala
@@ -23,7 +23,7 @@ import scala.collection.mutable.HashMap
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.annotation.Evolving
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.config.Tests._
import org.apache.spark.scheduler.{LiveListenerBus, SparkListenerResourceProfileAdded}
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
index 78c45cdc75418..8e11923a9e8f0 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
@@ -28,7 +28,7 @@ import org.json4s.jackson.JsonMethods._
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.api.resource.ResourceDiscoveryPlugin
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{EXECUTOR_CORES, RESOURCES_DISCOVERY_PLUGIN, SPARK_TASK_PREFIX}
import org.apache.spark.internal.config.Tests.RESOURCES_WARNING_TESTING
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala b/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala
index 8acfef38659c0..faa263c7db6a7 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/Dispatcher.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import scala.util.control.NonFatal
import org.apache.spark.{SparkEnv, SparkException}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.client.RpcResponseCallback
import org.apache.spark.rpc._
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
index 0de67a65593b1..dfaa9ea436458 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
@@ -22,7 +22,7 @@ import javax.annotation.concurrent.GuardedBy
import scala.util.control.NonFatal
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rpc.{RpcAddress, RpcEndpoint, ThreadSafeRpcEndpoint}
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/MessageLoop.scala b/core/src/main/scala/org/apache/spark/rpc/netty/MessageLoop.scala
index 2fd1c6d7fe71e..cce455270df43 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/MessageLoop.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/MessageLoop.scala
@@ -22,7 +22,7 @@ import java.util.concurrent._
import scala.util.control.NonFatal
import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.ERROR
import org.apache.spark.internal.config.EXECUTOR_ID
import org.apache.spark.internal.config.Network._
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
index c2688610fe8b1..a4f6d5438bf3d 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
@@ -30,7 +30,7 @@ import scala.util.{DynamicVariable, Failure, Success, Try}
import scala.util.control.NonFatal
import org.apache.spark.{SecurityManager, SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.EXECUTOR_ID
import org.apache.spark.internal.config.Network._
diff --git a/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala b/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
index 16e9211b54851..3931b8ff5190e 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
@@ -23,7 +23,7 @@ import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong}
import com.codahale.metrics.{Gauge, Timer}
import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index baf0ed4df5309..30eb49b0c0798 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -37,7 +37,7 @@ import org.apache.spark._
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{LEGACY_ABORT_STAGE_AFTER_KILL_TASKS, RDD_CACHE_VISIBILITY_TRACKING_ENABLED}
import org.apache.spark.internal.config.Tests.TEST_NO_STAGE_RETRY
diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 1a47d5ed17158..1e46142fab255 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -31,7 +31,7 @@ import org.apache.spark.deploy.history.EventLogFileWriter
import org.apache.spark.executor.ExecutorMetrics
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config._
-import org.apache.spark.util.{JsonProtocol, Utils}
+import org.apache.spark.util.{JsonProtocol, JsonProtocolOptions, Utils}
/**
* A SparkListener that logs events to persistent storage.
@@ -74,7 +74,7 @@ private[spark] class EventLoggingListener(
private val liveStageExecutorMetrics =
mutable.HashMap.empty[(Int, Int), mutable.HashMap[String, ExecutorMetrics]]
- private[this] val jsonProtocol = new JsonProtocol(sparkConf)
+ private[this] val jsonProtocolOptions = new JsonProtocolOptions(sparkConf)
/**
* Creates the log file in the configured log directory.
@@ -86,7 +86,7 @@ private[spark] class EventLoggingListener(
private def initEventLog(): Unit = {
val metadata = SparkListenerLogStart(SPARK_VERSION)
- val eventJson = jsonProtocol.sparkEventToJsonString(metadata)
+ val eventJson = JsonProtocol.sparkEventToJsonString(metadata, jsonProtocolOptions)
logWriter.writeEvent(eventJson, flushLogger = true)
if (testing && loggedEvents != null) {
loggedEvents += eventJson
@@ -95,7 +95,7 @@ private[spark] class EventLoggingListener(
/** Log the event as JSON. */
private def logEvent(event: SparkListenerEvent, flushLogger: Boolean = false): Unit = {
- val eventJson = jsonProtocol.sparkEventToJsonString(event)
+ val eventJson = JsonProtocol.sparkEventToJsonString(event, jsonProtocolOptions)
logWriter.writeEvent(eventJson, flushLogger)
if (testing) {
loggedEvents += eventJson
diff --git a/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala b/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala
index 82ec0ef91f4fc..fe62e39432250 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/HealthTracker.scala
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicReference
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
import org.apache.spark.{ExecutorAllocationClient, SparkConf, SparkContext}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.{Clock, SystemClock, Utils}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
index 7251eb2c86ea1..656f9875de2e2 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/LiveListenerBus.scala
@@ -30,7 +30,7 @@ import com.codahale.metrics.{Counter, MetricRegistry, Timer}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLASS_NAME, MAX_SIZE}
import org.apache.spark.internal.config._
import org.apache.spark.metrics.MetricsSystem
diff --git a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
index a769c3fa14b62..3e6f0fe3bfc44 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
@@ -20,7 +20,7 @@ package org.apache.spark.scheduler
import scala.collection.mutable
import org.apache.spark._
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.rpc.{RpcCallContext, RpcEndpoint, RpcEndpointRef, RpcEnv}
import org.apache.spark.util.{RpcUtils, ThreadUtils}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ReplayListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/ReplayListenerBus.scala
index 0aa74190d9f3d..2e6cfa98ff373 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ReplayListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ReplayListenerBus.scala
@@ -24,7 +24,7 @@ import scala.io.{Codec, Source}
import com.fasterxml.jackson.core.JsonParseException
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.scheduler.ReplayListenerBus._
import org.apache.spark.util.JsonProtocol
@@ -32,8 +32,7 @@ import org.apache.spark.util.JsonProtocol
/**
* A SparkListenerBus that can be used to replay events from serialized event data.
*/
-private[spark] class ReplayListenerBus(
- jsonProtocol: JsonProtocol) extends SparkListenerBus with Logging {
+private[spark] class ReplayListenerBus extends SparkListenerBus with Logging {
/**
* Replay each event in the order maintained in the given stream. The stream is expected to
@@ -87,7 +86,7 @@ private[spark] class ReplayListenerBus(
currentLine = entry._1
lineNumber = entry._2 + 1
- postToAll(jsonProtocol.sparkEventFromJson(currentLine))
+ postToAll(JsonProtocol.sparkEventFromJson(currentLine))
} catch {
case e: ClassNotFoundException =>
// Ignore unknown events, parse through the event log file.
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala
index bea49fb279ee3..06b6045cccd99 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala
@@ -26,7 +26,7 @@ import scala.xml.{Node, XML}
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkContext
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{SCHEDULER_ALLOCATION_FILE, SCHEDULER_MODE}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala b/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala
index e46dde5561a26..c8ffaefba3d64 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/StatsReportListener.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.util.{Distribution, Utils}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala
index e21ec77ce69ec..0e5e2c9faa05a 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala
@@ -22,7 +22,7 @@ import java.util.Properties
import org.apache.spark._
import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.APP_CALLER_CONTEXT
import org.apache.spark.internal.plugin.PluginContainer
import org.apache.spark.memory.{MemoryMode, TaskMemoryManager}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala
index 97c539bb05a58..70f58747506cc 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala
@@ -25,7 +25,7 @@ import scala.util.control.NonFatal
import org.apache.spark._
import org.apache.spark.TaskState.TaskState
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.CLASS_LOADER
import org.apache.spark.serializer.{SerializerHelper, SerializerInstance}
import org.apache.spark.util.{LongAccumulator, ThreadUtils, Utils}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 13018da5bc274..1351d8c778b58 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -33,7 +33,7 @@ import org.apache.spark.InternalAccumulator.{input, shuffleRead}
import org.apache.spark.TaskState.TaskState
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.ExecutorMetrics
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.resource.ResourceProfile
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala
index 3637305293107..c1b09567013f0 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetExcludeList.scala
@@ -19,7 +19,7 @@ package org.apache.spark.scheduler
import scala.collection.mutable.{HashMap, HashSet}
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.util.Clock
/**
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index 0eaf138d3eb8d..69e0a10a34b28 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -30,7 +30,7 @@ import org.apache.spark.InternalAccumulator
import org.apache.spark.InternalAccumulator.{input, shuffleRead}
import org.apache.spark.TaskState.TaskState
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.scheduler.SchedulingMode._
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 4b3a16b4d3f60..49590f0940f76 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -32,7 +32,7 @@ import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.security.HadoopDelegationTokenManager
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.ExecutorLogUrlHandler
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
index eb408a95589f7..061b54914c839 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
@@ -27,7 +27,7 @@ import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.deploy.{ApplicationDescription, Command}
import org.apache.spark.deploy.client.{StandaloneAppClient, StandaloneAppClientListener}
import org.apache.spark.executor.ExecutorExitCode
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.config.EXECUTOR_REMOVE_DELAY
import org.apache.spark.internal.config.Tests.IS_TESTING
import org.apache.spark.launcher.{LauncherBackend, SparkAppHandle}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
index 57505c87f879e..a98672fd7db4a 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
@@ -25,7 +25,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config._
import org.apache.spark.resource.ResourceProfile.UNKNOWN_RESOURCE_PROFILE_ID
import org.apache.spark.scheduler._
diff --git a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
index 1ee46d51ce70b..b230df93e1fac 100644
--- a/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
+++ b/core/src/main/scala/org/apache/spark/security/CryptoStreamUtils.scala
@@ -26,12 +26,11 @@ import javax.crypto.spec.{IvParameterSpec, SecretKeySpec}
import scala.jdk.CollectionConverters._
-import com.google.common.io.ByteStreams
import org.apache.commons.crypto.random._
import org.apache.commons.crypto.stream._
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.network.util.{CryptoUtils, JavaUtils}
@@ -84,7 +83,7 @@ private[spark] object CryptoStreamUtils extends Logging {
sparkConf: SparkConf,
key: Array[Byte]): InputStream = {
val iv = new Array[Byte](IV_LENGTH_IN_BYTES)
- ByteStreams.readFully(is, iv)
+ JavaUtils.readFully(is, iv, 0, IV_LENGTH_IN_BYTES)
val params = new CryptoParams(key, sparkConf)
new CryptoInputStream(params.transformation, params.conf, is, params.keySpec,
new IvParameterSpec(iv))
diff --git a/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala
index d09abff2773b8..23a5965d6aead 100644
--- a/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala
@@ -28,7 +28,6 @@ import com.esotericsoftware.kryo.io.{Input => KryoInput, Output => KryoOutput}
import org.apache.avro.{Schema, SchemaNormalization}
import org.apache.avro.generic.{GenericContainer, GenericData}
import org.apache.avro.io._
-import org.apache.commons.io.IOUtils
import org.apache.spark.{SparkEnv, SparkException}
import org.apache.spark.io.CompressionCodec
@@ -93,7 +92,7 @@ private[serializer] class GenericAvroSerializer[D <: GenericContainer]
schemaBytes.remaining())
val in = codec.compressedInputStream(bis)
val bytes = Utils.tryWithSafeFinally {
- IOUtils.toByteArray(in)
+ in.readAllBytes()
} {
in.close()
}
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 000ba8d79bc02..28261a4520a28 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -40,7 +40,7 @@ import org.roaringbitmap.RoaringBitmap
import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.api.python.PythonBroadcast
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.CLASS_NAME
import org.apache.spark.internal.config.Kryo._
import org.apache.spark.internal.io.FileCommitProtocol._
diff --git a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
index bf3117a9a9b12..9b27f648134ee 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
@@ -29,7 +29,7 @@ import com.google.common.cache.CacheBuilder
import org.apache.spark.{SecurityManager, SparkConf, SparkEnv, SparkException}
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{config, Logging, LogKeys, MDC}
+import org.apache.spark.internal.{config, Logging, LogKeys}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.io.NioBufferedFileInputStream
import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer}
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala
index 4e3191e44fbdf..548ecb399d5ff 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleBlockPusher.scala
@@ -27,7 +27,7 @@ import scala.util.control.NonFatal
import org.apache.spark.{SecurityManager, ShuffleDependency, SparkConf, SparkContext, SparkEnv}
import org.apache.spark.annotation.Since
import org.apache.spark.executor.{CoarseGrainedExecutorBackend, ExecutorBackend}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.launcher.SparkLauncher
diff --git a/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala b/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala
index be42af092f24a..47d54ae4f10b0 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/ShuffleWriteProcessor.scala
@@ -18,7 +18,7 @@
package org.apache.spark.shuffle
import org.apache.spark.{ShuffleDependency, SparkEnv, TaskContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NUM_MERGER_LOCATIONS, SHUFFLE_ID, STAGE_ID}
import org.apache.spark.scheduler.MapStatus
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index 5c93bf4bf77a0..52856427cb37a 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -26,7 +26,7 @@ import scala.jdk.CollectionConverters._
import org.apache.spark._
import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.CPUS_PER_TASK
import org.apache.spark.internal.config.Status._
import org.apache.spark.resource.ResourceProfile.CPUS
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 87f876467c30e..a7f3fde9e6f6f 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -25,7 +25,7 @@ import scala.collection.mutable.HashMap
import scala.jdk.CollectionConverters._
import org.apache.spark.{JobExecutionStatus, SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PATH
import org.apache.spark.internal.config.Status.LIVE_UI_LOCAL_STORE_DIR
import org.apache.spark.status.AppStatusUtils.getQuantilesValue
diff --git a/core/src/main/scala/org/apache/spark/status/KVUtils.scala b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
index e334626413dc0..76fb654f8da2d 100644
--- a/core/src/main/scala/org/apache/spark/status/KVUtils.scala
+++ b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
@@ -31,7 +31,7 @@ import org.rocksdb.RocksDBException
import org.apache.spark.SparkConf
import org.apache.spark.deploy.history.{FsHistoryProvider, FsHistoryProviderMetadata}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.History
import org.apache.spark.internal.config.History.HYBRID_STORE_DISK_BACKEND
@@ -213,6 +213,20 @@ private[spark] object KVUtils extends Logging {
}
}
+ /**
+ * Maps all values of KVStoreView to new values using a transformation function
+ * and filtered by a filter function.
+ */
+ def mapToSeqWithFilter[T, B](
+ view: KVStoreView[T],
+ max: Int)
+ (mapFunc: T => B)
+ (filterFunc: B => Boolean): Seq[B] = {
+ Utils.tryWithResource(view.closeableIterator()) { iter =>
+ iter.asScala.map(mapFunc).filter(filterFunc).take(max).toList
+ }
+ }
+
def size[T](view: KVStoreView[T]): Int = {
Utils.tryWithResource(view.closeableIterator()) { iter =>
iter.asScala.size
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
index 66fac8a9d105a..61e800844db09 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
@@ -82,6 +82,10 @@ private[spark] trait UIRoot {
def withSparkUI[T](appId: String, attemptId: Option[String])(fn: SparkUI => T): T
def getApplicationInfoList: Iterator[ApplicationInfo]
+
+ def getApplicationInfoList(max: Int)(
+ filter: ApplicationInfo => Boolean): Iterator[ApplicationInfo]
+
def getApplicationInfo(appId: String): Option[ApplicationInfo]
/**
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
index 2d5dd97b501f1..aaaa08b3340b9 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
@@ -38,7 +38,7 @@ private[v1] class ApplicationListResource extends ApiRequestContext {
val includeCompleted = status.isEmpty || status.contains(ApplicationStatus.COMPLETED)
val includeRunning = status.isEmpty || status.contains(ApplicationStatus.RUNNING)
- uiRoot.getApplicationInfoList.filter { app =>
+ uiRoot.getApplicationInfoList(numApps) { app =>
val anyRunning = app.attempts.isEmpty || !app.attempts.head.completed
// if any attempt is still running, we consider the app to also still be running;
// keep the app if *any* attempts fall in the right time window
@@ -46,7 +46,7 @@ private[v1] class ApplicationListResource extends ApiRequestContext {
app.attempts.exists { attempt =>
isAttemptInRange(attempt, minDate, maxDate, minEndDate, maxEndDate, anyRunning)
}
- }.take(numApps)
+ }
}
private def isAttemptInRange(
diff --git a/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala b/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
index d83cff5f23a27..26e5068d15325 100644
--- a/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/status/protobuf/StageDataWrapperSerializer.scala
@@ -21,12 +21,11 @@ import java.util.Date
import scala.jdk.CollectionConverters._
-import org.apache.commons.collections4.MapUtils
-
import org.apache.spark.status.StageDataWrapper
import org.apache.spark.status.api.v1.{ExecutorMetricsDistributions, ExecutorPeakMetricsDistributions, InputMetricDistributions, InputMetrics, OutputMetricDistributions, OutputMetrics, ShufflePushReadMetricDistributions, ShufflePushReadMetrics, ShuffleReadMetricDistributions, ShuffleReadMetrics, ShuffleWriteMetricDistributions, ShuffleWriteMetrics, SpeculationStageSummary, StageData, TaskData, TaskMetricDistributions, TaskMetrics}
import org.apache.spark.status.protobuf.Utils._
import org.apache.spark.util.Utils.weakIntern
+import org.apache.spark.util.collection.Utils.isNotEmpty
private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDataWrapper] {
@@ -397,11 +396,11 @@ private[protobuf] class StageDataWrapperSerializer extends ProtobufSerDe[StageDa
val failureReason = getOptional(binary.hasFailureReason, binary.getFailureReason)
val description = getOptional(binary.hasDescription, binary.getDescription)
val accumulatorUpdates = AccumulableInfoSerializer.deserialize(binary.getAccumulatorUpdatesList)
- val tasks = if (MapUtils.isNotEmpty(binary.getTasksMap)) {
+ val tasks = if (isNotEmpty(binary.getTasksMap)) {
Some(binary.getTasksMap.asScala.map(
entry => (entry._1.toLong, deserializeTaskData(entry._2))).toMap)
} else None
- val executorSummary = if (MapUtils.isNotEmpty(binary.getExecutorSummaryMap)) {
+ val executorSummary = if (isNotEmpty(binary.getExecutorSummaryMap)) {
Some(binary.getExecutorSummaryMap.asScala.toMap
.transform((_, v) => ExecutorStageSummarySerializer.deserialize(v)))
} else None
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
index d99bc5bf30546..1e1cb8bf9fd53 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
@@ -36,12 +36,11 @@ import scala.util.control.NonFatal
import com.codahale.metrics.{MetricRegistry, MetricSet}
import com.esotericsoftware.kryo.KryoException
import com.google.common.cache.CacheBuilder
-import org.apache.commons.io.IOUtils
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.{DataReadMethod, ExecutorExitCode}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{Network, RDD_CACHE_VISIBILITY_TRACKING_ENABLED, Tests}
import org.apache.spark.memory.{MemoryManager, MemoryMode}
@@ -373,7 +372,7 @@ private[spark] class BlockManager(
logInfo(extendMessageWithBlockDetails(ex.getMessage, blockId))
throw ex
} finally {
- IOUtils.closeQuietly(inputStream)
+ Utils.closeQuietly(inputStream)
}
}
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala
index 19807453ee28c..1a067fab04c1d 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerDecommissioner.scala
@@ -26,7 +26,7 @@ import scala.util.control.NonFatal
import org.apache.spark._
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.shuffle.ShuffleBlockInfo
import org.apache.spark.storage.BlockManagerMessages.ReplicateBlock
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
index 276bd63e14237..16c2cbbe7fdc4 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
@@ -23,7 +23,7 @@ import scala.concurrent.Future
import org.apache.spark.SparkConf
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.rpc.RpcEndpointRef
import org.apache.spark.storage.BlockManagerMessages._
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
index 858db498e83ad..f63d4a55669aa 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
@@ -31,7 +31,7 @@ import com.google.common.cache.CacheBuilder
import org.apache.spark.{MapOutputTrackerMaster, SparkConf, SparkContext, SparkEnv}
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.RDD_CACHE_VISIBILITY_TRACKING_ENABLED
import org.apache.spark.network.shuffle.{ExternalBlockStoreClient, RemoteBlockPushResolver}
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala
index f29e8778da037..54329c5b1e514 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerStorageEndpoint.scala
@@ -20,7 +20,7 @@ package org.apache.spark.storage
import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService, Future}
import org.apache.spark.{MapOutputTracker, SparkEnv}
-import org.apache.spark.internal.{Logging, MDC, MessageWithContext}
+import org.apache.spark.internal.{Logging, MessageWithContext}
import org.apache.spark.internal.LogKeys.{BLOCK_ID, BROADCAST_ID, RDD_ID, SHUFFLE_ID}
import org.apache.spark.rpc.{IsolatedThreadSafeRpcEndpoint, RpcCallContext, RpcEnv}
import org.apache.spark.storage.BlockManagerMessages._
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala b/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
index 5186cbfa217cc..9126dc8a83561 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockReplicationPolicy.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
import scala.util.Random
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
/**
diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
index 72d8dc0b19d21..df29ddf6999f5 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
@@ -30,7 +30,7 @@ import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.errors.SparkCoreErrors
import org.apache.spark.executor.ExecutorExitCode
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys.{MERGE_DIR_NAME, PATH}
import org.apache.spark.network.shuffle.ExecutorDiskUtils
import org.apache.spark.storage.DiskBlockManager.ATTEMPT_ID_KEY
@@ -316,7 +316,7 @@ private[spark] class DiskBlockManager(
throw SparkCoreErrors.failToCreateDirectoryError(dirToCreate.getAbsolutePath, maxAttempts)
}
try {
- dirToCreate.mkdirs()
+ Utils.createDirectory(dirToCreate)
Files.setPosixFilePermissions(
dirToCreate.toPath, PosixFilePermissions.fromString("rwxrwx---"))
if (dirToCreate.exists()) {
diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
index efcdb7fa8c69e..9964e64724f6b 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockObjectWriter.scala
@@ -24,7 +24,7 @@ import java.util.zip.Checksum
import org.apache.spark.SparkException
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.io.MutableCheckedOutputStream
import org.apache.spark.serializer.{SerializationStream, SerializerInstance, SerializerManager}
diff --git a/core/src/main/scala/org/apache/spark/storage/DiskStore.scala b/core/src/main/scala/org/apache/spark/storage/DiskStore.scala
index 3e57094b36a7e..b304d7bc32a84 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskStore.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskStore.scala
@@ -27,10 +27,9 @@ import scala.collection.mutable.ListBuffer
import com.google.common.io.Closeables
import io.netty.channel.DefaultFileRegion
-import org.apache.commons.io.FileUtils
import org.apache.spark.{SecurityManager, SparkConf, SparkException}
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.buffer.ManagedBuffer
import org.apache.spark.network.util.{AbstractFileRegion, JavaUtils}
@@ -151,7 +150,7 @@ private[spark] class DiskStore(
blockSizes.put(targetBlockId, blockSize)
val targetFile = diskManager.getFile(targetBlockId.name)
logDebug(s"${sourceFile.getPath()} -> ${targetFile.getPath()}")
- FileUtils.moveFile(sourceFile, targetFile)
+ Utils.moveFile(sourceFile, targetFile)
}
def contains(blockId: BlockId): Boolean = diskManager.containsBlock(blockId)
diff --git a/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala b/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala
index 0f2bfaede4454..cf6f3ea1f6dd5 100644
--- a/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala
+++ b/core/src/main/scala/org/apache/spark/storage/FallbackStorage.scala
@@ -17,26 +17,27 @@
package org.apache.spark.storage
-import java.io.DataInputStream
+import java.io.{DataInputStream, FileNotFoundException}
import java.nio.ByteBuffer
+import scala.annotation.tailrec
import scala.concurrent.Future
import scala.reflect.ClassTag
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.fs.{FileSystem, FSDataInputStream, Path}
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
-import org.apache.spark.internal.config.{STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP, STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH}
+import org.apache.spark.internal.config.{STORAGE_DECOMMISSION_FALLBACK_STORAGE_CLEANUP, STORAGE_DECOMMISSION_FALLBACK_STORAGE_PATH, STORAGE_DECOMMISSION_FALLBACK_STORAGE_REPLICATION_DELAY, STORAGE_DECOMMISSION_FALLBACK_STORAGE_REPLICATION_WAIT}
import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer}
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.rpc.{RpcAddress, RpcEndpointRef, RpcTimeout}
import org.apache.spark.shuffle.{IndexShuffleBlockResolver, ShuffleBlockInfo}
import org.apache.spark.shuffle.IndexShuffleBlockResolver.NOOP_REDUCE_ID
-import org.apache.spark.util.Utils
+import org.apache.spark.util.{Clock, SystemClock, Utils}
/**
* A fallback storage used by storage decommissioners.
@@ -155,6 +156,61 @@ private[spark] object FallbackStorage extends Logging {
FALLBACK_BLOCK_MANAGER_ID, blockId, StorageLevel.DISK_ONLY, memSize = 0, dataLength)
}
+ /**
+ * Open the file, retry a FileNotFoundException for waitMs milliseconds, unless this would
+ * exceed the deadline. In the latter case, rethrow the exception.
+ */
+ @tailrec
+ private def open(
+ filesystem: FileSystem,
+ path: Path,
+ deadlineTs: Long,
+ waitMs: Long,
+ clock: Clock): FSDataInputStream = {
+ try {
+ filesystem.open(path)
+ } catch {
+ case fnf: FileNotFoundException =>
+ val waitTillTs = clock.getTimeMillis() + waitMs
+ if (waitTillTs <= deadlineTs) {
+ logInfo(f"File not found, waiting ${waitMs / 1000}s: $path")
+ clock.waitTillTime(waitTillTs)
+ open(filesystem, path, deadlineTs, waitMs, clock)
+ } else {
+ throw fnf
+ }
+ }
+ }
+
+ /**
+ * Open the file and retry FileNotFoundExceptions according to
+ * STORAGE_DECOMMISSION_FALLBACK_STORAGE_REPLICATION_DELAY and
+ * STORAGE_DECOMMISSION_FALLBACK_STORAGE_REPLICATION_WAIT
+ */
+ // Visible for testing
+ private[spark] def open(
+ conf: SparkConf,
+ filesystem: FileSystem,
+ path: Path,
+ clock: Clock = new SystemClock()): FSDataInputStream = {
+ val replicationDelay = conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_REPLICATION_DELAY)
+ if (replicationDelay.isDefined) {
+ val replicationDeadline = clock.getTimeMillis() + replicationDelay.get
+ val replicationWaitMs = conf.get(STORAGE_DECOMMISSION_FALLBACK_STORAGE_REPLICATION_WAIT)
+ try {
+ open(filesystem, path, replicationDeadline, replicationWaitMs, clock)
+ } catch {
+ case fnf: FileNotFoundException =>
+ logInfo(
+ "File not found, exceeded expected replication delay " +
+ f"of ${replicationDelay.get}s: $path")
+ throw fnf
+ }
+ } else {
+ filesystem.open(path)
+ }
+ }
+
/**
* Read a ManagedBuffer.
*/
@@ -180,7 +236,7 @@ private[spark] object FallbackStorage extends Logging {
val indexFile = new Path(fallbackPath, s"$appId/$shuffleId/$hash/$name")
val start = startReduceId * 8L
val end = endReduceId * 8L
- Utils.tryWithResource(fallbackFileSystem.open(indexFile)) { inputStream =>
+ Utils.tryWithResource(open(conf, fallbackFileSystem, indexFile)) { inputStream =>
Utils.tryWithResource(new DataInputStream(inputStream)) { index =>
index.skip(start)
val offset = index.readLong()
@@ -193,7 +249,7 @@ private[spark] object FallbackStorage extends Logging {
logDebug(s"To byte array $size")
val array = new Array[Byte](size.toInt)
val startTimeNs = System.nanoTime()
- Utils.tryWithResource(fallbackFileSystem.open(dataFile)) { f =>
+ Utils.tryWithResource(open(conf, fallbackFileSystem, dataFile)) { f =>
f.seek(offset)
f.readFully(array)
logDebug(s"Took ${(System.nanoTime() - startTimeNs) / (1000 * 1000)}ms")
diff --git a/core/src/main/scala/org/apache/spark/storage/PushBasedFetchHelper.scala b/core/src/main/scala/org/apache/spark/storage/PushBasedFetchHelper.scala
index 8a3ca3066961c..16a68bbf50e25 100644
--- a/core/src/main/scala/org/apache/spark/storage/PushBasedFetchHelper.scala
+++ b/core/src/main/scala/org/apache/spark/storage/PushBasedFetchHelper.scala
@@ -28,7 +28,7 @@ import org.roaringbitmap.RoaringBitmap
import org.apache.spark.MapOutputTracker
import org.apache.spark.MapOutputTracker.SHUFFLE_PUSH_MAP_ID
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.shuffle.{BlockStoreClient, MergedBlockMeta, MergedBlocksMetaListener}
import org.apache.spark.shuffle.ShuffleReadMetricsReporter
diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
index 57f6901a7a735..b2f185bc590fd 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -19,7 +19,7 @@ package org.apache.spark.storage
import java.io.{InputStream, IOException}
import java.nio.channels.ClosedByInterruptException
-import java.util.concurrent.{LinkedBlockingQueue, TimeUnit}
+import java.util.concurrent.{LinkedBlockingDeque, TimeUnit}
import java.util.concurrent.atomic.AtomicBoolean
import java.util.zip.CheckedInputStream
import javax.annotation.concurrent.GuardedBy
@@ -30,13 +30,12 @@ import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Queue}
import scala.util.{Failure, Success}
import io.netty.util.internal.OutOfDirectMemoryError
-import org.apache.commons.io.IOUtils
import org.roaringbitmap.RoaringBitmap
import org.apache.spark.{MapOutputTracker, SparkException, TaskContext}
import org.apache.spark.MapOutputTracker.SHUFFLE_PUSH_MAP_ID
import org.apache.spark.errors.SparkCoreErrors
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer}
import org.apache.spark.network.shuffle._
@@ -132,7 +131,7 @@ final class ShuffleBlockFetcherIterator(
* A queue to hold our results. This turns the asynchronous model provided by
* [[org.apache.spark.network.BlockTransferService]] into a synchronous model (iterator).
*/
- private[this] val results = new LinkedBlockingQueue[FetchResult]
+ private[this] val results = new LinkedBlockingDeque[FetchResult]()
/**
* Current [[FetchResult]] being processed. We track this so we can release the current buffer
@@ -358,7 +357,7 @@ final class ShuffleBlockFetcherIterator(
results.put(FallbackOnPushMergedFailureResult(
block, address, infoMap(blockId)._1, remainingBlocks.isEmpty))
} else {
- results.put(FailureFetchResult(block, infoMap(blockId)._2, address, e))
+ results.putFirst(FailureFetchResult(block, infoMap(blockId)._2, address, e))
}
}
}
@@ -594,7 +593,8 @@ final class ShuffleBlockFetcherIterator(
log"Error occurred while fetching local blocks, ${MDC(ERROR, ce.getMessage)}")
case ex: Exception => logError("Error occurred while fetching local blocks", ex)
}
- results.put(FailureFetchResult(blockId, mapIndex, blockManager.blockManagerId, e))
+ results.putFirst(
+ FailureFetchResult(blockId, mapIndex, blockManager.blockManagerId, e))
return
}
}
@@ -615,7 +615,7 @@ final class ShuffleBlockFetcherIterator(
case e: Exception =>
// If we see an exception, stop immediately.
logError(s"Error occurred while fetching local blocks", e)
- results.put(FailureFetchResult(blockId, mapIndex, blockManagerId, e))
+ results.putFirst(FailureFetchResult(blockId, mapIndex, blockManagerId, e))
false
}
}
@@ -668,7 +668,7 @@ final class ShuffleBlockFetcherIterator(
val bmId = bmIds.head
val blockInfoSeq = hostLocalBlocksWithMissingDirs(bmId)
val (blockId, _, mapIndex) = blockInfoSeq.head
- results.put(FailureFetchResult(blockId, mapIndex, bmId, throwable))
+ results.putFirst(FailureFetchResult(blockId, mapIndex, bmId, throwable))
}
}
}
@@ -1407,7 +1407,7 @@ private class BufferReleasingInputStream(
val diagnosisResponse = checkedInOpt.map { checkedIn =>
iterator.diagnoseCorruption(checkedIn, address, blockId)
}
- IOUtils.closeQuietly(this)
+ Utils.closeQuietly(this)
// We'd never retry the block whatever the cause is since the block has been
// partially consumed by downstream RDDs.
iterator.throwFetchFailedException(blockId, mapIndex, address, e, diagnosisResponse)
diff --git a/core/src/main/scala/org/apache/spark/storage/TopologyMapper.scala b/core/src/main/scala/org/apache/spark/storage/TopologyMapper.scala
index f1dca53c7e3b1..45d3a4acb9ca9 100644
--- a/core/src/main/scala/org/apache/spark/storage/TopologyMapper.scala
+++ b/core/src/main/scala/org/apache/spark/storage/TopologyMapper.scala
@@ -19,7 +19,7 @@ package org.apache.spark.storage
import org.apache.spark.SparkConf
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.{config, Logging, MDC}
+import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.LogKeys._
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
index 6746bbd490c42..3981006c7ffe6 100644
--- a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
+++ b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
@@ -27,10 +27,8 @@ import scala.jdk.CollectionConverters._
import scala.reflect.ClassTag
import scala.util.control.NonFatal
-import com.google.common.io.ByteStreams
-
import org.apache.spark.{SparkConf, SparkException, TaskContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.{STORAGE_UNROLL_MEMORY_THRESHOLD, UNROLL_MEMORY_CHECK_PERIOD, UNROLL_MEMORY_GROWTH_FACTOR}
import org.apache.spark.memory.{MemoryManager, MemoryMode}
@@ -905,7 +903,7 @@ private[storage] class PartiallySerializedBlock[T](
// We want to close the output stream in order to free any resources associated with the
// serializer itself (such as Kryo's internal buffers). close() might cause data to be
// written, so redirect the output stream to discard that data.
- redirectableOutputStream.setOutputStream(ByteStreams.nullOutputStream())
+ redirectableOutputStream.setOutputStream(OutputStream.nullOutputStream())
serializationStream.close()
} finally {
discarded = true
@@ -923,7 +921,7 @@ private[storage] class PartiallySerializedBlock[T](
verifyNotConsumedAndNotDiscarded()
consumed = true
// `unrolled`'s underlying buffers will be freed once this input stream is fully read:
- ByteStreams.copy(unrolledBuffer.toInputStream(dispose = true), os)
+ unrolledBuffer.toInputStream(dispose = true).transferTo(os)
memoryStore.releaseUnrollMemoryForThisTask(memoryMode, unrollMemory)
redirectableOutputStream.setOutputStream(os)
while (rest.hasNext) {
diff --git a/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala b/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
index e95eeddbdace3..0ce0f69b20704 100644
--- a/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
+++ b/core/src/main/scala/org/apache/spark/ui/ConsoleProgressBar.scala
@@ -38,6 +38,8 @@ private[spark] class ConsoleProgressBar(sc: SparkContext) extends Logging {
private val updatePeriodMSec = sc.conf.get(UI_CONSOLE_PROGRESS_UPDATE_INTERVAL)
// Delay to show up a progress bar, in milliseconds
private val firstDelayMSec = 500L
+ // Get the stderr (which is console for spark-shell) before installing RedirectConsolePlugin
+ private val console = System.err
// The width of terminal
private val TerminalWidth = sys.env.getOrElse("COLUMNS", "80").toInt
@@ -92,7 +94,7 @@ private[spark] class ConsoleProgressBar(sc: SparkContext) extends Logging {
// only refresh if it's changed OR after 1 minute (or the ssh connection will be closed
// after idle some time)
if (bar != lastProgressBar || now - lastUpdateTime > 60 * 1000L) {
- System.err.print(s"$CR$bar$CR")
+ console.print(s"$CR$bar$CR")
lastUpdateTime = now
}
lastProgressBar = bar
@@ -103,7 +105,7 @@ private[spark] class ConsoleProgressBar(sc: SparkContext) extends Logging {
*/
private def clear(): Unit = {
if (!lastProgressBar.isEmpty) {
- System.err.printf(s"$CR${" " * TerminalWidth}$CR")
+ console.printf(s"$CR${" ".repeat(TerminalWidth)}$CR")
lastProgressBar = ""
}
}
diff --git a/core/src/main/scala/org/apache/spark/ui/DriverLogPage.scala b/core/src/main/scala/org/apache/spark/ui/DriverLogPage.scala
index 8b4eebc26b3ba..dca85b53178ad 100644
--- a/core/src/main/scala/org/apache/spark/ui/DriverLogPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/DriverLogPage.scala
@@ -21,7 +21,7 @@ import scala.xml.{Node, Unparsed}
import jakarta.servlet.http.HttpServletRequest
import org.apache.spark.SparkConf
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{LOG_TYPE, PATH}
import org.apache.spark.internal.config.DRIVER_LOG_LOCAL_DIR
import org.apache.spark.util.Utils
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 068f9b0b2cbcb..bd7848c764375 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -40,7 +40,7 @@ import org.json4s.JValue
import org.json4s.jackson.JsonMethods.{pretty, render}
import org.apache.spark.{SecurityManager, SparkConf, SSLOptions}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config.UI._
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index b8d422c9d9fbb..a23764f98d49d 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -23,7 +23,7 @@ import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletRespons
import org.eclipse.jetty.servlet.ServletContextHandler
import org.apache.spark.{SecurityManager, SparkConf, SparkContext}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLASS_NAME, WEB_URL}
import org.apache.spark.internal.config.DRIVER_LOG_LOCAL_DIR
import org.apache.spark.internal.config.UI._
@@ -201,6 +201,11 @@ private[spark] class SparkUI private (
))
}
+ override def getApplicationInfoList(max: Int)(
+ filter: ApplicationInfo => Boolean): Iterator[ApplicationInfo] = {
+ getApplicationInfoList.filter(filter).take(max)
+ }
+
def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
getApplicationInfoList.find(_.id == appId)
}
diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
index 60d4e5db99d7e..c0d0fb50ae36b 100644
--- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
@@ -29,7 +29,7 @@ import org.eclipse.jetty.servlet.{FilterHolder, FilterMapping, ServletContextHan
import org.json4s.JsonAST.{JNothing, JValue}
import org.apache.spark.{SecurityManager, SparkConf, SSLOptions}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys._
import org.apache.spark.internal.config._
import org.apache.spark.ui.JettyUtils._
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
index ef26a02a406a8..fa10c8937144c 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorThreadDumpPage.scala
@@ -73,6 +73,7 @@ private[ui] class ExecutorThreadDumpPage(
Updated at {UIUtils.formatDate(time)}
+ {threadDumpSummary(threadDump)}
{ if (flamegraphEnabled) {
drawExecutorFlamegraph(request, threadDump) }
else {
@@ -123,10 +124,8 @@ private[ui] class ExecutorThreadDumpPage(
}.getOrElse(Text("Error fetching thread dump"))
UIUtils.headerSparkPage(request, s"Thread dump for executor $executorId", content, parent)
- // scalastyle:on
}
- // scalastyle:off
private def drawExecutorFlamegraph(request: HttpServletRequest, thread: Array[ThreadStackTrace]): Seq[Node] = {
val js =
s"""
@@ -154,5 +153,32 @@ private[ui] class ExecutorThreadDumpPage(
}
- // scalastyle:off
+
+
+ private def threadDumpSummary(threadDump: Array[ThreadStackTrace]): Seq[Node] = {
+ val totalCount = threadDump.length
+
+
+
+
+
+ | Thread State | Count | Percentage |
+
+ {
+ threadDump.groupBy(_.threadState).map { case (state, threads) =>
+
+ | {state} |
+ {threads.length} |
+ {"%.2f%%".format(threads.length * 100.0 / totalCount)} |
+
+ }.toSeq
+ }
+
+
+
" in line and not inserted:
+ f.write(preview_newline + "\n")
+ inserted = True
+ f.write(line)
+ else:
+ for line in lines:
+ match = re.search(r'docs/(\d+\.\d+\.\d+)/', line)
+ if not inserted and match:
+ existing_version = match.group(1)
+ if vercmp(release_version, existing_version) >= 0:
+ f.write(stable_newline + "\n")
+ inserted = True
+ f.write(line)
+ if not inserted:
+ f.write(stable_newline + "\n")
+EOF
+
+ echo "Edited documentation.md"
+
+ # 2. Add download link to js/downloads.js
+ if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
+ echo "Skipping js/downloads.js for preview release: $RELEASE_VERSION"
+ else
+ RELEASE_DATE=$(TZ=America/Los_Angeles date +"%m/%d/%Y")
+ IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
+ NEW_PACKAGES="packagesV14"
+ if [[ "$rel_maj" -ge 4 ]]; then
+ NEW_PACKAGES="packagesV15"
+ fi
+
+ python3 <
b) - (a < b)
+
+inserted = replaced = False
+
+with open("js/downloads.js") as f:
+ lines = f.readlines()
+
+with open("js/downloads.js", "w") as f:
+ for line in lines:
+ m = re.search(r'addRelease\("(\d+\.\d+\.\d+)"', line)
+ if m:
+ existing_version = m.group(1)
+ cmp_result = vercmp(release_version, existing_version)
+ ex_major, ex_minor, ex_patch = parse_version(existing_version)
+
+ if cmp_result == 0:
+ f.write(newline + "\n")
+ replaced = True
+ elif not replaced and ex_major == new_major and ex_minor == new_minor:
+ f.write(newline + "\n")
+ replaced = True
+ elif not replaced and not inserted and cmp_result > 0:
+ f.write(newline + "\n")
+ f.write(line)
+ inserted = True
+ else:
+ f.write(line)
+ else:
+ f.write(line)
+ if not replaced and not inserted:
+ f.write(newline + "\n")
+EOF
+
+ echo "Edited js/downloads.js"
+ fi
+
+ # 3. Add news post
+ RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
+ FILENAME="news/_posts/${RELEASE_DATE}-spark-${RELEASE_VERSION//./-}-released.md"
+ mkdir -p news/_posts
+
+ if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
+ BASE_VERSION="${RELEASE_VERSION%%-preview*}"
+ cat > "$FILENAME" <Spark ${RELEASE_VERSION} release.
+This preview is not a stable release in terms of either API or functionality, but it is meant to give the community early
+access to try the code that will become Spark ${BASE_VERSION}. If you would like to test the release,
+please download it, and send feedback using either
+mailing lists or
+JIRA.
+The documentation is available at the link.
+
+We'd like to thank our contributors and users for their contributions and early feedback to this release. This release would not have been possible without you.
+EOF
+
+ else
+ cat > "$FILENAME" <Apache Spark ${RELEASE_VERSION}! Visit the release notes to read about the new features, or download the release today.
+EOF
+ fi
+
+ echo "Created $FILENAME"
+
+ # 4. Add release notes with Python to extract JIRA version ID
+ if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
+ echo "Skipping JIRA release notes for preview release: $RELEASE_VERSION"
+ else
+ RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
+ JIRA_PROJECT_ID=12315420
+ JIRA_URL="https://issues.apache.org/jira/rest/api/2/project/SPARK/versions"
+ JSON=$(curl -s "$JIRA_URL")
+
+ VERSION_ID=$(python3 - <&2
+ fi
+
+ JIRA_LINK="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=${JIRA_PROJECT_ID}&version=${VERSION_ID}"
+
+ IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
+ if [[ "$rel_patch" -eq 0 ]]; then
+ ACKNOWLEDGE="patches and features to this release."
+ BODY="Apache Spark ${RELEASE_VERSION} is a new feature release. It introduces new functionality and improvements. We encourage users to try it and provide feedback."
+ else
+ ACKNOWLEDGE="patches to this release."
+ BODY="Apache Spark ${RELEASE_VERSION} is a maintenance release containing security and correctness fixes. This release is based on the branch-${rel_maj}.${rel_min} maintenance branch of Spark. We strongly recommend all ${rel_maj}.${rel_min} users to upgrade to this stable release."
+ fi
+
+ BODY+="
+
+You can find the list of resolved issues and detailed changes in the [JIRA release notes](${JIRA_LINK}).
+
+We would like to acknowledge all community members for contributing ${ACKNOWLEDGE}"
+
+ FILENAME="releases/_posts/${RELEASE_DATE}-spark-release-${RELEASE_VERSION}.md"
+ mkdir -p releases/_posts
+ cat > "$FILENAME" < $RELEASE_VERSION (preview release)"
+
+ else
+ LINK_PATH="site/docs/latest"
+
+ if [[ "$rel_patch" -eq 0 ]]; then
+ if [[ -L "$LINK_PATH" ]]; then
+ CURRENT_TARGET=$(readlink "$LINK_PATH")
+ else
+ CURRENT_TARGET=""
+ fi
+
+ if [[ "$CURRENT_TARGET" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+ IFS='.' read -r cur_maj cur_min cur_patch <<< "$CURRENT_TARGET"
+
+ if [[ "$rel_maj" -gt "$cur_maj" ]]; then
+ ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
+ echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (major version increased)"
+ elif [[ "$rel_maj" -eq "$cur_maj" && "$rel_min" -gt "$cur_min" ]]; then
+ ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
+ echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (minor version increased)"
+ else
+ echo "Symlink $LINK_PATH points to $CURRENT_TARGET with equal or newer major.minor, no change"
+ fi
+ else
+ echo "No valid existing version target."
+ fi
+ else
+ echo "Patch release detected ($RELEASE_VERSION), not updating symlink"
+ fi
+ fi
+
+ git add .
+ git commit -m "Add release docs for Apache Spark $RELEASE_VERSION"
+ git push origin HEAD:asf-site
+ cd ..
+ echo "release docs uploaded"
+ rm -rf spark-website
+
# Moves the docs from dev directory to release directory.
echo "Moving Spark docs to the release directory"
svn mv --username "$ASF_USERNAME" --password "$ASF_PASSWORD" -m"Apache Spark $RELEASE_VERSION" \
@@ -180,6 +494,76 @@ if [[ "$1" == "finalize" ]]; then
echo "KEYS sync'ed"
rm -rf svn-spark
+ # TODO: Test it in the actual release
+ # Release artifacts in the Nexus repository
+ # Find latest orgapachespark-* repo for this release version
+ REPO_ID=$(curl --retry 10 --retry-all-errors -s -u "$ASF_USERNAME:$ASF_PASSWORD" \
+ https://repository.apache.org/service/local/staging/profile_repositories | \
+ grep -A 5 "orgapachespark-" | \
+ awk '// { id = $0 } // && $0 ~ /Apache Spark '"$RELEASE_VERSION"'/ { print id }' | \
+ grep -oP '(?<=)orgapachespark-[0-9]+(?=)' | \
+ sort -V | tail -n 1)
+
+ if [[ -z "$REPO_ID" ]]; then
+ echo "No matching staging repository found for Apache Spark $RELEASE_VERSION"
+ exit 1
+ fi
+
+ echo "Using repository ID: $REPO_ID"
+
+ # Release the repository
+ curl --retry 10 --retry-all-errors -s -u "$APACHE_USERNAME:$APACHE_PASSWORD" \
+ -H "Content-Type: application/json" \
+ -X POST https://repository.apache.org/service/local/staging/bulk/promote \
+ -d "{\"data\": {\"stagedRepositoryIds\": [\"$REPO_ID\"], \"description\": \"Apache Spark $RELEASE_VERSION\"}}"
+
+ # Wait for release to complete
+ echo "Waiting for release to complete..."
+ while true; do
+ STATUS=$(curl --retry 10 --retry-all-errors -s -u "$APACHE_USERNAME:$APACHE_PASSWORD" \
+ https://repository.apache.org/service/local/staging/repository/$REPO_ID | \
+ grep -oPm1 "(?<=)[^<]+")
+ echo "Current state: $STATUS"
+ if [[ "$STATUS" == "released" ]]; then
+ echo "Release complete."
+ break
+ elif [[ "$STATUS" == "release_failed" || "$STATUS" == "error" ]]; then
+ echo "Release failed."
+ exit 1
+ elif [[ "$STATUS" == "open" ]]; then
+ echo "Repository is still open. Cannot release. Please close it first."
+ exit 1
+ fi
+ sleep 10
+ done
+
+ # Drop the repository after release
+ curl --retry 10 --retry-all-errors -s -u "$APACHE_USERNAME:$APACHE_PASSWORD" \
+ -H "Content-Type: application/json" \
+ -X POST https://repository.apache.org/service/local/staging/bulk/drop \
+ -d "{\"data\": {\"stagedRepositoryIds\": [\"$REPO_ID\"], \"description\": \"Dropped after release\"}}"
+
+ echo "Done."
+
+ # TODO: Test it in the actual official release
+ # Remove old releases from the mirror
+ # Extract major.minor prefix
+ RELEASE_SERIES=$(echo "$RELEASE_VERSION" | cut -d. -f1-2)
+
+ # Fetch existing dist URLs
+ OLD_VERSION=$(svn ls https://dist.apache.org/repos/dist/release/spark/ | \
+ grep "^spark-$RELEASE_SERIES" | \
+ grep -v "^spark-$RELEASE_VERSION/" | \
+ sed 's#/##' | sed 's/^spark-//' | \
+ sort -V | tail -n 1)
+
+ if [[ -n "$OLD_VERSION" ]]; then
+ echo "Removing old version: spark-$OLD_VERSION"
+ svn rm "https://dist.apache.org/repos/dist/release/spark/spark-$OLD_VERSION" -m "Remove older $RELEASE_SERIES release after $RELEASE_VERSION"
+ else
+ echo "No previous $RELEASE_SERIES version found to remove. Manually remove it if there is."
+ fi
+
exit 0
fi
@@ -609,13 +993,15 @@ if [[ "$1" == "publish-release" ]]; then
# Calculate deadline in Pacific Time (PST/PDT)
DEADLINE=$(TZ=America/Los_Angeles date -d "+4 days" "+%a, %d %b %Y %H:%M:%S %Z")
+ PYSPARK_VERSION=`echo "$RELEASE_VERSION" | sed -e "s/-/./" -e "s/preview/dev/"`
JIRA_API_URL="https://issues.apache.org/jira/rest/api/2/project/SPARK/versions"
+ SPARK_VERSION_BASE=$(echo "$SPARK_VERSION" | sed 's/-preview[0-9]*//')
JIRA_VERSION_ID=$(curl -s "$JIRA_API_URL" | \
# Split JSON objects by replacing '},{' with a newline-separated pattern
tr '}' '\n' | \
# Find the block containing the exact version name
- grep -F "\"name\":\"$SPARK_VERSION\"" -A 5 | \
+ grep -F "\"name\":\"$SPARK_VERSION_BASE\"" -A 5 | \
# Extract the line with "id"
grep '"id"' | \
# Extract the numeric id value (assuming "id":"123456")
@@ -665,7 +1051,7 @@ EOF
echo "https://dist.apache.org/repos/dist/dev/spark/${GIT_REF}-bin/"
echo
echo "Signatures used for Spark RCs can be found in this file:"
- echo "https://dist.apache.org/repos/dist/dev/spark/KEYS"
+ echo "https://downloads.apache.org/spark/KEYS"
echo
echo "The staging repository for this release can be found at:"
echo "https://repository.apache.org/content/repositories/${staged_repo_id}/"
@@ -687,7 +1073,7 @@ EOF
echo "reporting any regressions."
echo
echo "If you're working in PySpark you can set up a virtual env and install"
- echo "the current RC via \"pip install https://dist.apache.org/repos/dist/dev/spark/${GIT_REF}-bin/pyspark-${SPARK_VERSION}.tar.gz\""
+ echo "the current RC via \"pip install https://dist.apache.org/repos/dist/dev/spark/${GIT_REF}-bin/pyspark-${PYSPARK_VERSION}.tar.gz\""
echo "and see if anything important breaks."
echo "In the Java/Scala, you can add the staging repository to your project's resolvers and test"
echo "with the RC (make sure to clean up the artifact cache before/after so"
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index b070aad1c52f0..79b4cfaeb8b0a 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -98,10 +98,10 @@ RUN mkdir -p /usr/local/pypy/pypy3.10 && \
ln -sf /usr/local/pypy/pypy3.10/bin/pypy /usr/local/bin/pypy3.10 && \
ln -sf /usr/local/pypy/pypy3.10/bin/pypy /usr/local/bin/pypy3
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | pypy3
-RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas==2.2.3' scipy coverage matplotlib lxml
+RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas==2.3.2' scipy coverage matplotlib lxml
-ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.2.3 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2 twine==3.4.1"
+ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.3.2 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2 twine==3.4.1"
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 protobuf==5.29.1 googleapis-common-protos==1.65.0 graphviz==0.20.3"
@@ -114,31 +114,19 @@ RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PI
python3.10 -m pip install deepspeed torcheval && \
python3.10 -m pip cache purge
-# Install Python 3.9
-RUN add-apt-repository ppa:deadsnakes/ppa
-RUN apt-get update && apt-get install -y \
- python3.9 python3.9-distutils \
- && rm -rf /var/lib/apt/lists/*
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9
-RUN python3.9 -m pip install --ignore-installed blinker>=1.6.2 # mlflow needs this
-RUN python3.9 -m pip install --force $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \
- python3.9 -m pip install 'torch<2.6.0' torchvision --index-url https://download.pytorch.org/whl/cpu && \
- python3.9 -m pip install torcheval && \
- python3.9 -m pip cache purge
-
# Should unpin 'sphinxcontrib-*' after upgrading sphinx>5
# See 'ipython_genutils' in SPARK-38517
# See 'docutils<0.18.0' in SPARK-39421
-RUN python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
-ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' 'docutils<0.18.0' \
+RUN python3.10 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
+ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.22' pyarrow pandas 'plotly>=4.8' 'docutils<0.18.0' \
'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' \
'pandas-stubs==1.2.0.53' 'grpcio==1.67.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5'
-RUN python3.9 -m pip list
+RUN python3.10 -m pip list
RUN gem install --no-document "bundler:2.4.22"
-RUN ln -s "$(which python3.9)" "/usr/local/bin/python"
-RUN ln -s "$(which python3.9)" "/usr/local/bin/python3"
+RUN ln -s "$(which python3.10)" "/usr/local/bin/python"
+RUN ln -s "$(which python3.10)" "/usr/local/bin/python3"
WORKDIR /opt/spark-rm/output
diff --git a/dev/create-release/vote.tmpl b/dev/create-release/vote.tmpl
index b5e53345f27b1..2dc56e3386b22 100644
--- a/dev/create-release/vote.tmpl
+++ b/dev/create-release/vote.tmpl
@@ -15,7 +15,7 @@ The release files, including signatures, digests, etc. can be found at:
https://dist.apache.org/repos/dist/dev/spark/{tag}-bin/
Signatures used for Spark RCs can be found in this file:
-https://dist.apache.org/repos/dist/dev/spark/KEYS
+https://downloads.apache.org/spark/KEYS
The staging repository for this release can be found at:
https://repository.apache.org/content/repositories/orgapachespark-{repo_id}/
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index f3f7ad4bcd1f9..a2ec364961155 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -9,11 +9,11 @@ aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar
aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar
aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar
aliyun-sdk-oss/3.13.2//aliyun-sdk-oss-3.13.2.jar
-annotations/17.0.0//annotations-17.0.0.jar
+analyticsaccelerator-s3/1.2.1//analyticsaccelerator-s3-1.2.1.jar
antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
antlr4-runtime/4.13.1//antlr4-runtime-4.13.1.jar
aopalliance-repackaged/3.0.6//aopalliance-repackaged-3.0.6.jar
-arpack/3.0.3//arpack-3.0.3.jar
+arpack/3.0.4//arpack-3.0.4.jar
arpack_combined_all/0.1//arpack_combined_all-0.1.jar
arrow-format/18.3.0//arrow-format-18.3.0.jar
arrow-memory-core/18.3.0//arrow-memory-core-18.3.0.jar
@@ -27,8 +27,8 @@ avro/1.12.0//avro-1.12.0.jar
azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
azure-storage/7.0.1//azure-storage-7.0.1.jar
-bcprov-jdk18on/1.80//bcprov-jdk18on-1.80.jar
-blas/3.0.3//blas-3.0.3.jar
+bcprov-jdk18on/1.81//bcprov-jdk18on-1.81.jar
+blas/3.0.4//blas-3.0.4.jar
breeze-macros_2.13/2.1.0//breeze-macros_2.13-2.1.0.jar
breeze_2.13/2.1.0//breeze_2.13-2.1.0.jar
bundle/2.29.52//bundle-2.29.52.jar
@@ -36,24 +36,24 @@ cats-kernel_2.13/2.8.0//cats-kernel_2.13-2.8.0.jar
checker-qual/3.43.0//checker-qual-3.43.0.jar
chill-java/0.10.0//chill-java-0.10.0.jar
chill_2.13/0.10.0//chill_2.13-0.10.0.jar
-commons-cli/1.9.0//commons-cli-1.9.0.jar
-commons-codec/1.18.0//commons-codec-1.18.0.jar
+commons-cli/1.10.0//commons-cli-1.10.0.jar
+commons-codec/1.19.0//commons-codec-1.19.0.jar
commons-collections/3.2.2//commons-collections-3.2.2.jar
commons-collections4/4.5.0//commons-collections4-4.5.0.jar
commons-compiler/3.1.9//commons-compiler-3.1.9.jar
-commons-compress/1.27.1//commons-compress-1.27.1.jar
+commons-compress/1.28.0//commons-compress-1.28.0.jar
commons-crypto/1.1.0//commons-crypto-1.1.0.jar
commons-dbcp/1.4//commons-dbcp-1.4.jar
-commons-io/2.19.0//commons-io-2.19.0.jar
+commons-io/2.20.0//commons-io-2.20.0.jar
commons-lang/2.6//commons-lang-2.6.jar
-commons-lang3/3.17.0//commons-lang3-3.17.0.jar
+commons-lang3/3.18.0//commons-lang3-3.18.0.jar
commons-math3/3.6.1//commons-math3-3.6.1.jar
commons-pool/1.5.4//commons-pool-1.5.4.jar
-commons-text/1.13.1//commons-text-1.13.1.jar
+commons-text/1.14.0//commons-text-1.14.0.jar
compress-lzf/1.1.2//compress-lzf-1.1.2.jar
-curator-client/5.7.1//curator-client-5.7.1.jar
-curator-framework/5.7.1//curator-framework-5.7.1.jar
-curator-recipes/5.7.1//curator-recipes-5.7.1.jar
+curator-client/5.9.0//curator-client-5.9.0.jar
+curator-framework/5.9.0//curator-framework-5.9.0.jar
+curator-recipes/5.9.0//curator-recipes-5.9.0.jar
datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar
datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar
datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar
@@ -67,20 +67,20 @@ error_prone_annotations/2.36.0//error_prone_annotations-2.36.0.jar
esdk-obs-java/3.20.4.2//esdk-obs-java-3.20.4.2.jar
failureaccess/1.0.2//failureaccess-1.0.2.jar
flatbuffers-java/25.2.10//flatbuffers-java-25.2.10.jar
-gcs-connector/hadoop3-2.2.26/shaded/gcs-connector-hadoop3-2.2.26-shaded.jar
+gcs-connector/hadoop3-2.2.28/shaded/gcs-connector-hadoop3-2.2.28-shaded.jar
gmetric4j/1.0.10//gmetric4j-1.0.10.jar
gson/2.11.0//gson-2.11.0.jar
guava/33.4.0-jre//guava-33.4.0-jre.jar
-hadoop-aliyun/3.4.1//hadoop-aliyun-3.4.1.jar
-hadoop-annotations/3.4.1//hadoop-annotations-3.4.1.jar
-hadoop-aws/3.4.1//hadoop-aws-3.4.1.jar
-hadoop-azure-datalake/3.4.1//hadoop-azure-datalake-3.4.1.jar
-hadoop-azure/3.4.1//hadoop-azure-3.4.1.jar
-hadoop-client-api/3.4.1//hadoop-client-api-3.4.1.jar
-hadoop-client-runtime/3.4.1//hadoop-client-runtime-3.4.1.jar
-hadoop-cloud-storage/3.4.1//hadoop-cloud-storage-3.4.1.jar
-hadoop-huaweicloud/3.4.1//hadoop-huaweicloud-3.4.1.jar
-hadoop-shaded-guava/1.3.0//hadoop-shaded-guava-1.3.0.jar
+hadoop-aliyun/3.4.2//hadoop-aliyun-3.4.2.jar
+hadoop-annotations/3.4.2//hadoop-annotations-3.4.2.jar
+hadoop-aws/3.4.2//hadoop-aws-3.4.2.jar
+hadoop-azure-datalake/3.4.2//hadoop-azure-datalake-3.4.2.jar
+hadoop-azure/3.4.2//hadoop-azure-3.4.2.jar
+hadoop-client-api/3.4.2//hadoop-client-api-3.4.2.jar
+hadoop-client-runtime/3.4.2//hadoop-client-runtime-3.4.2.jar
+hadoop-cloud-storage/3.4.2//hadoop-cloud-storage-3.4.2.jar
+hadoop-huaweicloud/3.4.2//hadoop-huaweicloud-3.4.2.jar
+hadoop-shaded-guava/1.4.0//hadoop-shaded-guava-1.4.0.jar
hive-beeline/2.3.10//hive-beeline-2.3.10.jar
hive-cli/2.3.10//hive-cli-2.3.10.jar
hive-common/2.3.10//hive-common-2.3.10.jar
@@ -104,15 +104,13 @@ ini4j/0.5.4//ini4j-0.5.4.jar
istack-commons-runtime/4.1.2//istack-commons-runtime-4.1.2.jar
ivy/2.5.3//ivy-2.5.3.jar
j2objc-annotations/3.0.0//j2objc-annotations-3.0.0.jar
-jackson-annotations/2.19.0//jackson-annotations-2.19.0.jar
-jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
-jackson-core/2.19.0//jackson-core-2.19.0.jar
-jackson-databind/2.19.0//jackson-databind-2.19.0.jar
-jackson-dataformat-cbor/2.19.0//jackson-dataformat-cbor-2.19.0.jar
-jackson-dataformat-yaml/2.19.0//jackson-dataformat-yaml-2.19.0.jar
-jackson-datatype-jsr310/2.19.0//jackson-datatype-jsr310-2.19.0.jar
-jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.13/2.19.0//jackson-module-scala_2.13-2.19.0.jar
+jackson-annotations/2.19.2//jackson-annotations-2.19.2.jar
+jackson-core/2.19.2//jackson-core-2.19.2.jar
+jackson-databind/2.19.2//jackson-databind-2.19.2.jar
+jackson-dataformat-cbor/2.19.2//jackson-dataformat-cbor-2.19.2.jar
+jackson-dataformat-yaml/2.19.2//jackson-dataformat-yaml-2.19.2.jar
+jackson-datatype-jsr310/2.19.2//jackson-datatype-jsr310-2.19.2.jar
+jackson-module-scala_2.13/2.19.2//jackson-module-scala_2.13-2.19.2.jar
jakarta.activation-api/2.1.3//jakarta.activation-api-2.1.3.jar
jakarta.annotation-api/2.1.1//jakarta.annotation-api-2.1.1.jar
jakarta.inject-api/2.0.1//jakarta.inject-api-2.0.1.jar
@@ -132,12 +130,12 @@ jaxb-runtime/4.0.5//jaxb-runtime-4.0.5.jar
jcl-over-slf4j/2.0.17//jcl-over-slf4j-2.0.17.jar
jdo-api/3.0.1//jdo-api-3.0.1.jar
jdom2/2.0.6//jdom2-2.0.6.jar
-jersey-client/3.0.17//jersey-client-3.0.17.jar
-jersey-common/3.0.17//jersey-common-3.0.17.jar
-jersey-container-servlet-core/3.0.17//jersey-container-servlet-core-3.0.17.jar
-jersey-container-servlet/3.0.17//jersey-container-servlet-3.0.17.jar
-jersey-hk2/3.0.17//jersey-hk2-3.0.17.jar
-jersey-server/3.0.17//jersey-server-3.0.17.jar
+jersey-client/3.0.18//jersey-client-3.0.18.jar
+jersey-common/3.0.18//jersey-common-3.0.18.jar
+jersey-container-servlet-core/3.0.18//jersey-container-servlet-core-3.0.18.jar
+jersey-container-servlet/3.0.18//jersey-container-servlet-3.0.18.jar
+jersey-hk2/3.0.18//jersey-hk2-3.0.18.jar
+jersey-server/3.0.18//jersey-server-3.0.18.jar
jettison/1.5.4//jettison-1.5.4.jar
jetty-util-ajax/11.0.25//jetty-util-ajax-11.0.25.jar
jetty-util/11.0.25//jetty-util-11.0.25.jar
@@ -146,8 +144,7 @@ jjwt-impl/0.12.6//jjwt-impl-0.12.6.jar
jjwt-jackson/0.12.6//jjwt-jackson-0.12.6.jar
jline/2.14.6//jline-2.14.6.jar
jline/3.27.1/jdk8/jline-3.27.1-jdk8.jar
-joda-time/2.13.0//joda-time-2.13.0.jar
-jodd-core/3.5.2//jodd-core-3.5.2.jar
+joda-time/2.14.0//joda-time-2.14.0.jar
jpam/1.1//jpam-1.1.jar
json/1.8//json-1.8.jar
json4s-ast_2.13/4.0.7//json4s-ast_2.13-4.0.7.jar
@@ -157,6 +154,7 @@ json4s-jackson_2.13/4.0.7//json4s-jackson_2.13-4.0.7.jar
json4s-scalap_2.13/4.0.7//json4s-scalap_2.13-4.0.7.jar
jsr305/3.0.0//jsr305-3.0.0.jar
jta/1.1//jta-1.1.jar
+jts-core/1.20.0//jts-core-1.20.0.jar
jul-to-slf4j/2.0.17//jul-to-slf4j-2.0.17.jar
kryo-shaded/4.0.3//kryo-shaded-4.0.3.jar
kubernetes-client-api/7.3.1//kubernetes-client-api-7.3.1.jar
@@ -184,61 +182,61 @@ kubernetes-model-rbac/7.3.1//kubernetes-model-rbac-7.3.1.jar
kubernetes-model-resource/7.3.1//kubernetes-model-resource-7.3.1.jar
kubernetes-model-scheduling/7.3.1//kubernetes-model-scheduling-7.3.1.jar
kubernetes-model-storageclass/7.3.1//kubernetes-model-storageclass-7.3.1.jar
-lapack/3.0.3//lapack-3.0.3.jar
+lapack/3.0.4//lapack-3.0.4.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
libfb303/0.9.3//libfb303-0.9.3.jar
libthrift/0.16.0//libthrift-0.16.0.jar
listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
-log4j-1.2-api/2.24.3//log4j-1.2-api-2.24.3.jar
-log4j-api/2.24.3//log4j-api-2.24.3.jar
-log4j-core/2.24.3//log4j-core-2.24.3.jar
-log4j-layout-template-json/2.24.3//log4j-layout-template-json-2.24.3.jar
-log4j-slf4j2-impl/2.24.3//log4j-slf4j2-impl-2.24.3.jar
+log4j-1.2-api/2.25.1//log4j-1.2-api-2.25.1.jar
+log4j-api/2.25.1//log4j-api-2.25.1.jar
+log4j-core/2.25.1//log4j-core-2.25.1.jar
+log4j-layout-template-json/2.25.1//log4j-layout-template-json-2.25.1.jar
+log4j-slf4j2-impl/2.25.1//log4j-slf4j2-impl-2.25.1.jar
lz4-java/1.8.0//lz4-java-1.8.0.jar
-metrics-core/4.2.30//metrics-core-4.2.30.jar
-metrics-graphite/4.2.30//metrics-graphite-4.2.30.jar
-metrics-jmx/4.2.30//metrics-jmx-4.2.30.jar
-metrics-json/4.2.30//metrics-json-4.2.30.jar
-metrics-jvm/4.2.30//metrics-jvm-4.2.30.jar
+metrics-core/4.2.33//metrics-core-4.2.33.jar
+metrics-graphite/4.2.33//metrics-graphite-4.2.33.jar
+metrics-jmx/4.2.33//metrics-jmx-4.2.33.jar
+metrics-json/4.2.33//metrics-json-4.2.33.jar
+metrics-jvm/4.2.33//metrics-jvm-4.2.33.jar
minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.1.121.Final//netty-all-4.1.121.Final.jar
-netty-buffer/4.1.121.Final//netty-buffer-4.1.121.Final.jar
-netty-codec-dns/4.1.121.Final//netty-codec-dns-4.1.121.Final.jar
-netty-codec-http/4.1.121.Final//netty-codec-http-4.1.121.Final.jar
-netty-codec-http2/4.1.121.Final//netty-codec-http2-4.1.121.Final.jar
-netty-codec-socks/4.1.121.Final//netty-codec-socks-4.1.121.Final.jar
-netty-codec/4.1.121.Final//netty-codec-4.1.121.Final.jar
-netty-common/4.1.121.Final//netty-common-4.1.121.Final.jar
-netty-handler-proxy/4.1.121.Final//netty-handler-proxy-4.1.121.Final.jar
-netty-handler/4.1.121.Final//netty-handler-4.1.121.Final.jar
-netty-resolver-dns/4.1.121.Final//netty-resolver-dns-4.1.121.Final.jar
-netty-resolver/4.1.121.Final//netty-resolver-4.1.121.Final.jar
-netty-tcnative-boringssl-static/2.0.70.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar
-netty-tcnative-boringssl-static/2.0.70.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar
-netty-tcnative-boringssl-static/2.0.70.Final/osx-aarch_64/netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar
-netty-tcnative-boringssl-static/2.0.70.Final/osx-x86_64/netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar
-netty-tcnative-boringssl-static/2.0.70.Final/windows-x86_64/netty-tcnative-boringssl-static-2.0.70.Final-windows-x86_64.jar
-netty-tcnative-classes/2.0.70.Final//netty-tcnative-classes-2.0.70.Final.jar
-netty-transport-classes-epoll/4.1.121.Final//netty-transport-classes-epoll-4.1.121.Final.jar
-netty-transport-classes-kqueue/4.1.121.Final//netty-transport-classes-kqueue-4.1.121.Final.jar
-netty-transport-native-epoll/4.1.121.Final/linux-aarch_64/netty-transport-native-epoll-4.1.121.Final-linux-aarch_64.jar
-netty-transport-native-epoll/4.1.121.Final/linux-riscv64/netty-transport-native-epoll-4.1.121.Final-linux-riscv64.jar
-netty-transport-native-epoll/4.1.121.Final/linux-x86_64/netty-transport-native-epoll-4.1.121.Final-linux-x86_64.jar
-netty-transport-native-kqueue/4.1.121.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.121.Final-osx-aarch_64.jar
-netty-transport-native-kqueue/4.1.121.Final/osx-x86_64/netty-transport-native-kqueue-4.1.121.Final-osx-x86_64.jar
-netty-transport-native-unix-common/4.1.121.Final//netty-transport-native-unix-common-4.1.121.Final.jar
-netty-transport/4.1.121.Final//netty-transport-4.1.121.Final.jar
-objenesis/3.3//objenesis-3.3.jar
+netty-all/4.1.124.Final//netty-all-4.1.124.Final.jar
+netty-buffer/4.1.124.Final//netty-buffer-4.1.124.Final.jar
+netty-codec-dns/4.1.124.Final//netty-codec-dns-4.1.124.Final.jar
+netty-codec-http/4.1.124.Final//netty-codec-http-4.1.124.Final.jar
+netty-codec-http2/4.1.124.Final//netty-codec-http2-4.1.124.Final.jar
+netty-codec-socks/4.1.124.Final//netty-codec-socks-4.1.124.Final.jar
+netty-codec/4.1.124.Final//netty-codec-4.1.124.Final.jar
+netty-common/4.1.124.Final//netty-common-4.1.124.Final.jar
+netty-handler-proxy/4.1.124.Final//netty-handler-proxy-4.1.124.Final.jar
+netty-handler/4.1.124.Final//netty-handler-4.1.124.Final.jar
+netty-resolver-dns/4.1.124.Final//netty-resolver-dns-4.1.124.Final.jar
+netty-resolver/4.1.124.Final//netty-resolver-4.1.124.Final.jar
+netty-tcnative-boringssl-static/2.0.72.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.72.Final-linux-aarch_64.jar
+netty-tcnative-boringssl-static/2.0.72.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.72.Final-linux-x86_64.jar
+netty-tcnative-boringssl-static/2.0.72.Final/osx-aarch_64/netty-tcnative-boringssl-static-2.0.72.Final-osx-aarch_64.jar
+netty-tcnative-boringssl-static/2.0.72.Final/osx-x86_64/netty-tcnative-boringssl-static-2.0.72.Final-osx-x86_64.jar
+netty-tcnative-boringssl-static/2.0.72.Final/windows-x86_64/netty-tcnative-boringssl-static-2.0.72.Final-windows-x86_64.jar
+netty-tcnative-classes/2.0.72.Final//netty-tcnative-classes-2.0.72.Final.jar
+netty-transport-classes-epoll/4.1.124.Final//netty-transport-classes-epoll-4.1.124.Final.jar
+netty-transport-classes-kqueue/4.1.124.Final//netty-transport-classes-kqueue-4.1.124.Final.jar
+netty-transport-native-epoll/4.1.124.Final/linux-aarch_64/netty-transport-native-epoll-4.1.124.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.1.124.Final/linux-riscv64/netty-transport-native-epoll-4.1.124.Final-linux-riscv64.jar
+netty-transport-native-epoll/4.1.124.Final/linux-x86_64/netty-transport-native-epoll-4.1.124.Final-linux-x86_64.jar
+netty-transport-native-kqueue/4.1.124.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.124.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.1.124.Final/osx-x86_64/netty-transport-native-kqueue-4.1.124.Final-osx-x86_64.jar
+netty-transport-native-unix-common/4.1.124.Final//netty-transport-native-unix-common-4.1.124.Final.jar
+netty-transport/4.1.124.Final//netty-transport-4.1.124.Final.jar
+objenesis/3.4//objenesis-3.4.jar
okhttp/3.12.12//okhttp-3.12.12.jar
okio/1.17.6//okio-1.17.6.jar
opencsv/2.3//opencsv-2.3.jar
opentracing-api/0.33.0//opentracing-api-0.33.0.jar
opentracing-noop/0.33.0//opentracing-noop-0.33.0.jar
opentracing-util/0.33.0//opentracing-util-0.33.0.jar
-orc-core/2.1.2/shaded-protobuf/orc-core-2.1.2-shaded-protobuf.jar
-orc-format/1.1.0/shaded-protobuf/orc-format-1.1.0-shaded-protobuf.jar
-orc-mapreduce/2.1.2/shaded-protobuf/orc-mapreduce-2.1.2-shaded-protobuf.jar
-orc-shims/2.1.2//orc-shims-2.1.2.jar
+orc-core/2.2.0/shaded-protobuf/orc-core-2.2.0-shaded-protobuf.jar
+orc-format/1.1.1/shaded-protobuf/orc-format-1.1.1-shaded-protobuf.jar
+orc-mapreduce/2.2.0/shaded-protobuf/orc-mapreduce-2.2.0-shaded-protobuf.jar
+orc-shims/2.2.0//orc-shims-2.2.0.jar
oro/2.0.8//oro-2.0.8.jar
osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
paranamer/2.8.3//paranamer-2.8.3.jar
@@ -252,7 +250,6 @@ pickle/1.5//pickle-1.5.jar
py4j/0.10.9.9//py4j-0.10.9.9.jar
remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar
rocksdbjni/9.8.4//rocksdbjni-9.8.4.jar
-scala-collection-compat_2.13/2.7.0//scala-collection-compat_2.13-2.7.0.jar
scala-compiler/2.13.16//scala-compiler-2.13.16.jar
scala-library/2.13.16//scala-library-2.13.16.jar
scala-parallel-collections_2.13/1.2.0//scala-parallel-collections_2.13-1.2.0.jar
@@ -262,7 +259,7 @@ scala-xml_2.13/2.3.0//scala-xml_2.13-2.3.0.jar
slf4j-api/2.0.17//slf4j-api-2.0.17.jar
snakeyaml-engine/2.9//snakeyaml-engine-2.9.jar
snakeyaml/2.4//snakeyaml-2.4.jar
-snappy-java/1.1.10.7//snappy-java-1.1.10.7.jar
+snappy-java/1.1.10.8//snappy-java-1.1.10.8.jar
spire-macros_2.13/0.18.0//spire-macros_2.13-0.18.0.jar
spire-platform_2.13/0.18.0//spire-platform_2.13-0.18.0.jar
spire-util_2.13/0.18.0//spire-util_2.13-0.18.0.jar
@@ -285,4 +282,4 @@ xz/1.10//xz-1.10.jar
zjsonpatch/7.3.1//zjsonpatch-7.3.1.jar
zookeeper-jute/3.9.3//zookeeper-jute-3.9.3.jar
zookeeper/3.9.3//zookeeper-3.9.3.jar
-zstd-jni/1.5.7-3//zstd-jni-1.5.7-3.jar
+zstd-jni/1.5.7-4//zstd-jni-1.5.7-4.jar
diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile
index 9cd6031023ca5..8813112127d11 100644
--- a/dev/infra/Dockerfile
+++ b/dev/infra/Dockerfile
@@ -91,10 +91,10 @@ RUN mkdir -p /usr/local/pypy/pypy3.10 && \
ln -sf /usr/local/pypy/pypy3.10/bin/pypy /usr/local/bin/pypy3.10 && \
ln -sf /usr/local/pypy/pypy3.10/bin/pypy /usr/local/bin/pypy3
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | pypy3
-RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas==2.2.3' scipy coverage matplotlib lxml
+RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas==2.3.0' scipy coverage matplotlib lxml
-ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.2.3 scipy plotly>=4.8 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
+ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.3.0 scipy plotly>=4.8 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 protobuf==5.29.1 googleapis-common-protos==1.65.0 graphviz==0.20.3"
@@ -148,7 +148,7 @@ RUN apt-get update && apt-get install -y \
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13
# TODO(SPARK-49862) Add BASIC_PIP_PKGS and CONNECT_PIP_PKGS to Python 3.13 image when it supports Python 3.13
RUN python3.13 -m pip install --ignore-installed blinker>=1.6.2 # mlflow needs this
-RUN python3.13 -m pip install numpy>=2.1 pyarrow>=18.0.0 six==1.16.0 pandas==2.2.3 scipy coverage matplotlib openpyxl grpcio==1.67.0 grpcio-status==1.67.0 lxml jinja2 && \
+RUN python3.13 -m pip install numpy>=2.1 pyarrow>=18.0.0 six==1.16.0 pandas==2.3.0 scipy coverage matplotlib openpyxl grpcio==1.67.0 grpcio-status==1.67.0 lxml jinja2 && \
python3.13 -m pip cache purge
# Remove unused installation packages to free up disk space
diff --git a/dev/lint-scala b/dev/lint-scala
index 19c8853c787db..bfd0646f5f328 100755
--- a/dev/lint-scala
+++ b/dev/lint-scala
@@ -34,6 +34,7 @@ ERRORS=$(./build/mvn \
-pl sql/api \
-pl sql/connect/common \
-pl sql/connect/server \
+ -pl sql/connect/shims \
-pl sql/connect/client/jvm \
2>&1 | grep -e "Unformatted files found" \
)
@@ -41,7 +42,7 @@ ERRORS=$(./build/mvn \
if test ! -z "$ERRORS"; then
echo -e "The scalafmt check failed on sql/connect or sql/connect at following occurrences:\n\n$ERRORS\n"
echo "Before submitting your change, please make sure to format your code using the following command:"
- echo "./build/mvn scalafmt:format -Dscalafmt.skip=false -Dscalafmt.validateOnly=false -Dscalafmt.changedOnly=false -pl sql/api -pl sql/connect/common -pl sql/connect/server -pl sql/connect/client/jvm"
+ echo "./build/mvn scalafmt:format -Dscalafmt.skip=false -Dscalafmt.validateOnly=false -Dscalafmt.changedOnly=false -pl sql/api -pl sql/connect/common -pl sql/connect/server -pl sql/connect/shims -pl sql/connect/client/jvm"
exit 1
else
echo -e "Scalafmt checks passed."
diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh
index 39f6477e07c97..b54ea496c633b 100755
--- a/dev/make-distribution.sh
+++ b/dev/make-distribution.sh
@@ -133,7 +133,7 @@ if [ $(command -v git) ]; then
unset GITREV
fi
-if [ "$SBT_ENABLED" == "true" && ! "$(command -v "$SBT")" ]; then
+if [ "$SBT_ENABLED" == "true" ] && [ ! "$(command -v "$SBT")" ]; then
echo -e "Could not locate SBT command: '$SBT'."
echo -e "Specify the SBT command with the --sbt flag"
exit -1;
@@ -204,14 +204,6 @@ echo "Build flags: $@" >> "$DISTDIR/RELEASE"
# Copy jars
cp -r "$SPARK_HOME"/assembly/target/scala*/jars/* "$DISTDIR/jars/"
-# Only create the hive-jackson directory if they exist.
-if [ -f "$DISTDIR"/jars/jackson-core-asl-1.9.13.jar ]; then
- for f in "$DISTDIR"/jars/jackson-*-asl-*.jar; do
- mkdir -p "$DISTDIR"/hive-jackson
- mv $f "$DISTDIR"/hive-jackson/
- done
-fi
-
# Only create the yarn directory if the yarn artifacts were built.
if [ -f "$SPARK_HOME"/common/network-yarn/target/scala*/spark-*-yarn-shuffle.jar ]; then
mkdir "$DISTDIR/yarn"
diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py
index e9e82b010032f..ff38253bcf536 100755
--- a/dev/merge_spark_pr.py
+++ b/dev/merge_spark_pr.py
@@ -611,7 +611,11 @@ def main():
branch_names = sorted(branch_names, reverse=True)
branch_iter = iter(branch_names)
- pr_num = bold_input("Which pull request would you like to merge? (e.g. 34): ")
+ if len(sys.argv) == 1:
+ pr_num = bold_input("Which pull request would you like to merge? (e.g. 34): ")
+ else:
+ pr_num = sys.argv[1]
+ print("Start to merge pull request #%s" % (pr_num))
pr = get_json("%s/pulls/%s" % (GITHUB_API_BASE, pr_num))
pr_events = get_json("%s/issues/%s/events" % (GITHUB_API_BASE, pr_num))
diff --git a/dev/package-lock.json b/dev/package-lock.json
index e6ec1406a7620..4f20040d08cdb 100644
--- a/dev/package-lock.json
+++ b/dev/package-lock.json
@@ -4,9 +4,9 @@
"requires": true,
"packages": {
"": {
- "name": "dev",
"devDependencies": {
"ansi-regex": "^5.0.1",
+ "brace-expansion": "^1.1.12",
"eslint": "^7.25.0",
"minimatch": "^3.1.2"
}
@@ -237,10 +237,11 @@
"dev": true
},
"node_modules/brace-expansion": {
- "version": "1.1.11",
- "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
- "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+ "version": "1.1.12",
+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
+ "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
"dev": true,
+ "license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@@ -1407,9 +1408,9 @@
"dev": true
},
"brace-expansion": {
- "version": "1.1.11",
- "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
- "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+ "version": "1.1.12",
+ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
+ "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
"dev": true,
"requires": {
"balanced-match": "^1.0.0",
diff --git a/dev/package.json b/dev/package.json
index 98f204868dbcb..738f9c178b1b4 100644
--- a/dev/package.json
+++ b/dev/package.json
@@ -2,7 +2,8 @@
"devDependencies": {
"eslint": "^7.25.0",
"ansi-regex": "^5.0.1",
- "minimatch": "^3.1.2"
+ "minimatch": "^3.1.2",
+ "brace-expansion": "^1.1.12"
},
"resolutions": {
"optionator": "^0.9.3"
diff --git a/dev/requirements.txt b/dev/requirements.txt
index dbc517688cede..a8c3ef2c91ac2 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -2,8 +2,8 @@
py4j>=0.10.9.9
# PySpark dependencies (optional)
-numpy>=1.21
-pyarrow>=11.0.0
+numpy>=1.22
+pyarrow>=15.0.0
six==1.16.0
pandas>=2.2.0
scipy
@@ -27,6 +27,7 @@ pytest-mypy-plugins==1.9.3
flake8==3.9.0
# See SPARK-38680.
pandas-stubs<1.2.0.54
+scipy-stubs; python_version>='3.10'
types-PyYAML
# Documentation (SQL)
diff --git a/dev/run-pip-tests b/dev/run-pip-tests
index 7740481e4d832..426ec329d915d 100755
--- a/dev/run-pip-tests
+++ b/dev/run-pip-tests
@@ -55,7 +55,7 @@ if hash virtualenv 2>/dev/null && [ ! -n "$USE_CONDA" ]; then
fi
elif hash conda 2>/dev/null; then
echo "Using conda virtual environments"
- PYTHON_EXECS=('3.9')
+ PYTHON_EXECS=('3.10')
USE_CONDA=1
else
echo "Missing virtualenv & conda, skipping pip installability tests"
diff --git a/dev/run-tests.py b/dev/run-tests.py
index eb760139f9b68..32c46e0d2b94b 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -427,7 +427,7 @@ def parse_opts():
parser.add_argument(
"--python-executables",
type=str,
- default="python3.9",
+ default="python3.11",
help="A comma-separated list of Python executables to test against (default: %(default)s)",
)
parser.add_argument(
diff --git a/dev/spark-test-image/docs/Dockerfile b/dev/spark-test-image/docs/Dockerfile
index f1e33763df468..2068d53d6aba5 100644
--- a/dev/spark-test-image/docs/Dockerfile
+++ b/dev/spark-test-image/docs/Dockerfile
@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image for Documentat
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20241029
+ENV FULL_REFRESH_DATE=20250616
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
@@ -56,6 +56,7 @@ RUN apt-get update && apt-get install -y \
pandoc \
pkg-config \
qpdf \
+ tzdata \
r-base \
ruby \
ruby-dev \
@@ -74,18 +75,21 @@ RUN Rscript -e "install.packages(c('devtools', 'knitr', 'markdown', 'rmarkdown',
# See more in SPARK-39735
ENV R_LIBS_SITE="/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library"
-# Install Python 3.9
+# Install Python 3.11
RUN add-apt-repository ppa:deadsnakes/ppa
-RUN apt-get update && apt-get install -y python3.9 python3.9-distutils \
+RUN apt-get update && apt-get install -y \
+ python3.11 \
+ && apt-get autoremove --purge -y \
+ && apt-get clean \
&& rm -rf /var/lib/apt/lists/*
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
# Should unpin 'sphinxcontrib-*' after upgrading sphinx>5
# See 'ipython_genutils' in SPARK-38517
# See 'docutils<0.18.0' in SPARK-39421
-RUN python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
- ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' 'docutils<0.18.0' \
+RUN python3.11 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
+ ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.22' pyarrow 'pandas==2.3.2' 'plotly>=4.8' 'docutils<0.18.0' \
'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' \
'pandas-stubs==1.2.0.53' 'grpcio==1.67.0' 'grpcio-status==1.67.0' 'protobuf==5.29.1' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' \
- && python3.9 -m pip cache purge
+ && python3.11 -m pip cache purge
diff --git a/dev/spark-test-image/lint/Dockerfile b/dev/spark-test-image/lint/Dockerfile
index e43ca46d1409b..3b603d4ab4a68 100644
--- a/dev/spark-test-image/lint/Dockerfile
+++ b/dev/spark-test-image/lint/Dockerfile
@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image for Linter"
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20250519
+ENV FULL_REFRESH_DATE=20250618
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
@@ -93,7 +93,7 @@ RUN python3.11 -m pip install \
'pandas' \
'pandas-stubs==1.2.0.53' \
'plotly>=4.8' \
- 'pyarrow>=19.0.0' \
+ 'pyarrow>=21.0.0' \
'pytest-mypy-plugins==1.9.3' \
'pytest==7.1.3' \
&& python3.11 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu \
diff --git a/dev/spark-test-image/pypy-310/Dockerfile b/dev/spark-test-image/pypy-310/Dockerfile
index 6a309d38f1d55..cddf0f8ea10a3 100644
--- a/dev/spark-test-image/pypy-310/Dockerfile
+++ b/dev/spark-test-image/pypy-310/Dockerfile
@@ -68,4 +68,4 @@ RUN mkdir -p /usr/local/pypy/pypy3.10 && \
ln -sf /usr/local/pypy/pypy3.10/bin/pypy /usr/local/bin/pypy3.10 && \
ln -sf /usr/local/pypy/pypy3.10/bin/pypy /usr/local/bin/pypy3
RUN curl -sS https://bootstrap.pypa.io/get-pip.py | pypy3
-RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas==2.2.3' scipy coverage matplotlib lxml
+RUN pypy3 -m pip install numpy 'six==1.16.0' 'pandas==2.3.2' scipy coverage matplotlib lxml
diff --git a/dev/spark-test-image/python-309/Dockerfile b/dev/spark-test-image/python-309/Dockerfile
deleted file mode 100644
index 7fd4b604225c8..0000000000000
--- a/dev/spark-test-image/python-309/Dockerfile
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Image for building and testing Spark branches. Based on Ubuntu 22.04.
-# See also in https://hub.docker.com/_/ubuntu
-FROM ubuntu:jammy-20240911.1
-LABEL org.opencontainers.image.authors="Apache Spark project "
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark with Python 3.09"
-# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
-LABEL org.opencontainers.image.version=""
-
-ENV FULL_REFRESH_DATE=20250312
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV DEBCONF_NONINTERACTIVE_SEEN=true
-
-RUN apt-get update && apt-get install -y \
- build-essential \
- ca-certificates \
- curl \
- gfortran \
- git \
- gnupg \
- libcurl4-openssl-dev \
- libfontconfig1-dev \
- libfreetype6-dev \
- libfribidi-dev \
- libgit2-dev \
- libharfbuzz-dev \
- libjpeg-dev \
- liblapack-dev \
- libopenblas-dev \
- libpng-dev \
- libpython3-dev \
- libssl-dev \
- libtiff5-dev \
- libxml2-dev \
- openjdk-17-jdk-headless \
- pkg-config \
- qpdf \
- tzdata \
- software-properties-common \
- wget \
- zlib1g-dev
-
-# Install Python 3.9
-RUN add-apt-repository ppa:deadsnakes/ppa
-RUN apt-get update && apt-get install -y \
- python3.9 \
- python3.9-distutils \
- && apt-get autoremove --purge -y \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
-ARG BASIC_PIP_PKGS="numpy pyarrow>=19.0.0 six==1.16.0 pandas==2.2.3 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
-# Python deps for Spark Connect
-ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 protobuf==5.29.1 googleapis-common-protos==1.65.0 graphviz==0.20.3"
-
-# Install Python 3.9 packages
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9
-RUN python3.9 -m pip install --ignore-installed blinker>=1.6.2 # mlflow needs this
-RUN python3.9 -m pip install --force $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \
- python3.9 -m pip install 'torch<2.6.0' torchvision --index-url https://download.pytorch.org/whl/cpu && \
- python3.9 -m pip install torcheval && \
- python3.9 -m pip cache purge
diff --git a/dev/spark-test-image/python-310/Dockerfile b/dev/spark-test-image/python-310/Dockerfile
index 57c6c850a6219..292970e519568 100644
--- a/dev/spark-test-image/python-310/Dockerfile
+++ b/dev/spark-test-image/python-310/Dockerfile
@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark wi
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20250312
+ENV FULL_REFRESH_DATE=20250618
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
@@ -63,7 +63,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
-ARG BASIC_PIP_PKGS="numpy pyarrow>=19.0.0 six==1.16.0 pandas==2.2.3 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
+ARG BASIC_PIP_PKGS="numpy pyarrow>=21.0.0 six==1.16.0 pandas==2.3.2 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 protobuf==5.29.1 googleapis-common-protos==1.65.0 graphviz==0.20.3"
diff --git a/dev/spark-test-image/python-311-classic-only/Dockerfile b/dev/spark-test-image/python-311-classic-only/Dockerfile
index 8f2ec0b0dd1f5..e8bed6b909889 100644
--- a/dev/spark-test-image/python-311-classic-only/Dockerfile
+++ b/dev/spark-test-image/python-311-classic-only/Dockerfile
@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark Cl
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20250424
+ENV FULL_REFRESH_DATE=20250618
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
@@ -67,7 +67,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
-ARG BASIC_PIP_PKGS="numpy pyarrow>=19.0.0 pandas==2.2.3 plotly<6.0.0 matplotlib openpyxl memory-profiler>=0.61.0 mlflow>=2.8.1 scipy scikit-learn>=1.3.2"
+ARG BASIC_PIP_PKGS="numpy pyarrow>=21.0.0 pandas==2.3.2 plotly<6.0.0 matplotlib openpyxl memory-profiler>=0.61.0 mlflow>=2.8.1 scipy scikit-learn>=1.3.2"
ARG TEST_PIP_PKGS="coverage unittest-xml-reporting"
# Install Python 3.11 packages
diff --git a/dev/spark-test-image/python-311/Dockerfile b/dev/spark-test-image/python-311/Dockerfile
index 1a2caa483785b..29f80d7bae3f6 100644
--- a/dev/spark-test-image/python-311/Dockerfile
+++ b/dev/spark-test-image/python-311/Dockerfile
@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark wi
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20250312
+ENV FULL_REFRESH_DATE=20250618
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
@@ -67,7 +67,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
-ARG BASIC_PIP_PKGS="numpy pyarrow>=19.0.0 six==1.16.0 pandas==2.2.3 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
+ARG BASIC_PIP_PKGS="numpy pyarrow>=21.0.0 six==1.16.0 pandas==2.3.2 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 protobuf==5.29.1 googleapis-common-protos==1.65.0 graphviz==0.20.3"
diff --git a/dev/spark-test-image/python-312/Dockerfile b/dev/spark-test-image/python-312/Dockerfile
index f64e3e3ba30ce..d9a3944f10647 100644
--- a/dev/spark-test-image/python-312/Dockerfile
+++ b/dev/spark-test-image/python-312/Dockerfile
@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark wi
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20250312
+ENV FULL_REFRESH_DATE=20250618
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
@@ -67,7 +67,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
-ARG BASIC_PIP_PKGS="numpy pyarrow>=19.0.0 six==1.16.0 pandas==2.2.3 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
+ARG BASIC_PIP_PKGS="numpy pyarrow>=21.0.0 six==1.16.0 pandas==2.3.2 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 protobuf==5.29.1 googleapis-common-protos==1.65.0 graphviz==0.20.3"
diff --git a/dev/spark-test-image/python-313-nogil/Dockerfile b/dev/spark-test-image/python-313-nogil/Dockerfile
index cee6a4cca4d33..2819ac3114f3a 100644
--- a/dev/spark-test-image/python-313-nogil/Dockerfile
+++ b/dev/spark-test-image/python-313-nogil/Dockerfile
@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark wi
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20250407
+ENV FULL_REFRESH_DATE=20250618
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
@@ -67,7 +67,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
-ARG BASIC_PIP_PKGS="numpy pyarrow>=19.0.0 six==1.16.0 pandas==2.2.3 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
+ARG BASIC_PIP_PKGS="numpy pyarrow>=21.0.0 six==1.16.0 pandas==2.3.2 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 protobuf==5.29.1 googleapis-common-protos==1.65.0 graphviz==0.20.3"
@@ -76,5 +76,5 @@ RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13t
# TODO: Add BASIC_PIP_PKGS and CONNECT_PIP_PKGS when it supports Python 3.13 free threaded
# TODO: Add lxml, grpcio, grpcio-status back when they support Python 3.13 free threaded
RUN python3.13t -m pip install --ignore-installed blinker>=1.6.2 # mlflow needs this
-RUN python3.13t -m pip install numpy>=2.1 pyarrow>=19.0.0 six==1.16.0 pandas==2.2.3 scipy coverage matplotlib openpyxl jinja2 && \
+RUN python3.13t -m pip install numpy>=2.1 pyarrow>=19.0.0 six==1.16.0 pandas==2.3.2 scipy coverage matplotlib openpyxl jinja2 && \
python3.13t -m pip cache purge
diff --git a/dev/spark-test-image/python-313/Dockerfile b/dev/spark-test-image/python-313/Dockerfile
index aede82ac7d78c..e1ef91341a1f3 100644
--- a/dev/spark-test-image/python-313/Dockerfile
+++ b/dev/spark-test-image/python-313/Dockerfile
@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark wi
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20250312
+ENV FULL_REFRESH_DATE=20250618
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
@@ -67,7 +67,7 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
-ARG BASIC_PIP_PKGS="numpy pyarrow>=19.0.0 six==1.16.0 pandas==2.2.3 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
+ARG BASIC_PIP_PKGS="numpy pyarrow>=21.0.0 six==1.16.0 pandas==2.3.2 scipy plotly<6.0.0 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 protobuf==5.29.1 googleapis-common-protos==1.65.0 graphviz==0.20.3"
diff --git a/dev/spark-test-image/python-minimum/Dockerfile b/dev/spark-test-image/python-minimum/Dockerfile
index 59d9ebed4e40f..8f42d02023e50 100644
--- a/dev/spark-test-image/python-minimum/Dockerfile
+++ b/dev/spark-test-image/python-minimum/Dockerfile
@@ -24,11 +24,12 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For PySpark wi
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20250327
+ENV FULL_REFRESH_DATE=20250703
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
+# Should keep the installation consistent with https://apache.github.io/spark/api/python/getting_started/install.html
RUN apt-get update && apt-get install -y \
build-essential \
ca-certificates \
@@ -52,30 +53,19 @@ RUN apt-get update && apt-get install -y \
libxml2-dev \
openjdk-17-jdk-headless \
pkg-config \
+ python3.10 \
+ python3-psutil \
qpdf \
tzdata \
software-properties-common \
wget \
zlib1g-dev
-
-# Should keep the installation consistent with https://apache.github.io/spark/api/python/getting_started/install.html
-
-# Install Python 3.9
-RUN add-apt-repository ppa:deadsnakes/ppa
-RUN apt-get update && apt-get install -y \
- python3.9 \
- python3.9-distutils \
- && apt-get autoremove --purge -y \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
-
-ARG BASIC_PIP_PKGS="numpy==1.21 pyarrow==11.0.0 pandas==2.0.0 six==1.16.0 scipy scikit-learn coverage unittest-xml-reporting"
+ARG BASIC_PIP_PKGS="numpy==1.22.4 pyarrow==15.0.0 pandas==2.2.0 six==1.16.0 scipy scikit-learn coverage unittest-xml-reporting"
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 googleapis-common-protos==1.65.0 graphviz==0.20 protobuf"
# Install Python 3.9 packages
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9
-RUN python3.9 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \
- python3.9 -m pip cache purge
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+RUN python3.10 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \
+ python3.10 -m pip cache purge
diff --git a/dev/spark-test-image/python-ps-minimum/Dockerfile b/dev/spark-test-image/python-ps-minimum/Dockerfile
index 0cdf1fa6aa1f1..440fda96f0fc8 100644
--- a/dev/spark-test-image/python-ps-minimum/Dockerfile
+++ b/dev/spark-test-image/python-ps-minimum/Dockerfile
@@ -24,11 +24,12 @@ LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For Pandas API
# Overwrite this label to avoid exposing the underlying Ubuntu OS version label
LABEL org.opencontainers.image.version=""
-ENV FULL_REFRESH_DATE=20250327
+ENV FULL_REFRESH_DATE=20250708
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
+# Should keep the installation consistent with https://apache.github.io/spark/api/python/getting_started/install.html
RUN apt-get update && apt-get install -y \
build-essential \
ca-certificates \
@@ -52,6 +53,8 @@ RUN apt-get update && apt-get install -y \
libxml2-dev \
openjdk-17-jdk-headless \
pkg-config \
+ python3.10 \
+ python3-psutil \
qpdf \
tzdata \
software-properties-common \
@@ -59,23 +62,11 @@ RUN apt-get update && apt-get install -y \
zlib1g-dev
-# Should keep the installation consistent with https://apache.github.io/spark/api/python/getting_started/install.html
-
-# Install Python 3.9
-RUN add-apt-repository ppa:deadsnakes/ppa
-RUN apt-get update && apt-get install -y \
- python3.9 \
- python3.9-distutils \
- && apt-get autoremove --purge -y \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
-
-ARG BASIC_PIP_PKGS="pyarrow==11.0.0 pandas==2.2.0 six==1.16.0 numpy scipy coverage unittest-xml-reporting"
+ARG BASIC_PIP_PKGS="pyarrow==15.0.0 pandas==2.2.0 six==1.16.0 numpy scipy coverage unittest-xml-reporting"
# Python deps for Spark Connect
ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 googleapis-common-protos==1.65.0 graphviz==0.20 protobuf"
-# Install Python 3.9 packages
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9
-RUN python3.9 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \
- python3.9 -m pip cache purge
+# Install Python 3.10 packages
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+RUN python3.10 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \
+ python3.10 -m pip cache purge
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 4c748aec64278..f26a70d68d039 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -113,9 +113,20 @@ def __hash__(self):
],
)
+utils_java = Module(
+ name="utils-java",
+ dependencies=[tags],
+ source_file_regexes=[
+ "common/utils-java/",
+ ],
+ sbt_test_goals=[
+ "common-utils-java/test",
+ ],
+)
+
utils = Module(
name="utils",
- dependencies=[tags],
+ dependencies=[tags, utils_java],
source_file_regexes=[
"common/utils/",
],
@@ -137,7 +148,7 @@ def __hash__(self):
network_common = Module(
name="network-common",
- dependencies=[tags, utils],
+ dependencies=[tags, utils_java],
source_file_regexes=[
"common/network-common/",
],
@@ -509,6 +520,7 @@ def __hash__(self):
"pyspark.sql.avro.functions",
"pyspark.sql.protobuf.functions",
"pyspark.sql.pandas.conversion",
+ "pyspark.sql.pandas.functions",
"pyspark.sql.pandas.map_ops",
"pyspark.sql.pandas.group_ops",
"pyspark.sql.pandas.types",
@@ -526,6 +538,7 @@ def __hash__(self):
"pyspark.sql.tests.test_dataframe",
"pyspark.sql.tests.test_collection",
"pyspark.sql.tests.test_creation",
+ "pyspark.sql.tests.test_conversion",
"pyspark.sql.tests.test_listener",
"pyspark.sql.tests.test_observation",
"pyspark.sql.tests.test_repartition",
@@ -542,7 +555,11 @@ def __hash__(self):
"pyspark.sql.tests.arrow.test_arrow_grouped_map",
"pyspark.sql.tests.arrow.test_arrow_python_udf",
"pyspark.sql.tests.arrow.test_arrow_udf",
+ "pyspark.sql.tests.arrow.test_arrow_udf_grouped_agg",
"pyspark.sql.tests.arrow.test_arrow_udf_scalar",
+ "pyspark.sql.tests.arrow.test_arrow_udf_window",
+ "pyspark.sql.tests.arrow.test_arrow_udf_typehints",
+ "pyspark.sql.tests.arrow.test_arrow_udtf",
"pyspark.sql.tests.pandas.test_pandas_cogrouped_map",
"pyspark.sql.tests.pandas.test_pandas_grouped_map",
"pyspark.sql.tests.pandas.test_pandas_grouped_map_with_state",
@@ -568,6 +585,7 @@ def __hash__(self):
"pyspark.sql.tests.test_subquery",
"pyspark.sql.tests.test_types",
"pyspark.sql.tests.test_udf",
+ "pyspark.sql.tests.test_udf_combinations",
"pyspark.sql.tests.test_udf_profiler",
"pyspark.sql.tests.test_udtf",
"pyspark.sql.tests.test_tvf",
@@ -576,6 +594,8 @@ def __hash__(self):
"pyspark.sql.tests.plot.test_frame_plot",
"pyspark.sql.tests.plot.test_frame_plot_plotly",
"pyspark.sql.tests.test_connect_compatibility",
+ "pyspark.sql.tests.udf_type_tests.test_udf_input_types",
+ "pyspark.sql.tests.udf_type_tests.test_udf_return_types",
],
)
@@ -587,6 +607,14 @@ def __hash__(self):
# doctests
"pyspark.testing.utils",
"pyspark.testing.pandasutils",
+ # unittests
+ "pyspark.testing.tests.test_fail",
+ "pyspark.testing.tests.test_fail_in_set_up_class",
+ "pyspark.testing.tests.test_no_tests",
+ "pyspark.testing.tests.test_pass_all",
+ "pyspark.testing.tests.test_skip_all",
+ "pyspark.testing.tests.test_skip_class",
+ "pyspark.testing.tests.test_skip_set_up_class",
],
)
@@ -1077,6 +1105,7 @@ def __hash__(self):
"pyspark.sql.tests.connect.test_parity_column",
"pyspark.sql.tests.connect.test_parity_readwriter",
"pyspark.sql.tests.connect.test_parity_udf",
+ "pyspark.sql.tests.connect.test_parity_udf_combinations",
"pyspark.sql.tests.connect.test_parity_udf_profiler",
"pyspark.sql.tests.connect.test_parity_memory_profiler",
"pyspark.sql.tests.connect.test_parity_udtf",
@@ -1104,6 +1133,9 @@ def __hash__(self):
"pyspark.sql.tests.connect.arrow.test_parity_arrow_python_udf",
"pyspark.sql.tests.connect.arrow.test_parity_arrow_udf",
"pyspark.sql.tests.connect.arrow.test_parity_arrow_udf_scalar",
+ "pyspark.sql.tests.connect.arrow.test_parity_arrow_udf_grouped_agg",
+ "pyspark.sql.tests.connect.arrow.test_parity_arrow_udf_window",
+ "pyspark.sql.tests.connect.arrow.test_parity_arrow_udtf",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_map",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map",
"pyspark.sql.tests.connect.pandas.test_parity_pandas_grouped_map_with_state",
@@ -1507,6 +1539,7 @@ def __hash__(self):
source_file_regexes=["python/pyspark/pipelines"],
python_test_goals=[
"pyspark.pipelines.tests.test_block_connect_access",
+ "pyspark.pipelines.tests.test_block_session_mutations",
"pyspark.pipelines.tests.test_cli",
"pyspark.pipelines.tests.test_decorators",
"pyspark.pipelines.tests.test_graph_element_registry",
diff --git a/dev/sparktestsupport/utils.py b/dev/sparktestsupport/utils.py
index 94ca1f52ef2b8..26e7a12b69d5b 100755
--- a/dev/sparktestsupport/utils.py
+++ b/dev/sparktestsupport/utils.py
@@ -47,6 +47,12 @@ def determine_modules_for_files(filenames):
for filename in filenames:
if filename.endswith("README.md"):
continue
+ if filename in (
+ "scalastyle-config.xml",
+ "dev/checkstyle.xml",
+ "dev/checkstyle-suppressions.xml",
+ ):
+ continue
if ("GITHUB_ACTIONS" not in os.environ) and filename.startswith(".github"):
continue
matched_at_least_one_module = False
diff --git a/dev/test-classes.txt b/dev/test-classes.txt
index 5315c970c5bab..2dc6b290ad4f9 100644
--- a/dev/test-classes.txt
+++ b/dev/test-classes.txt
@@ -1,4 +1,3 @@
-repl/src/test/resources/IntSumUdf.class
sql/core/src/test/resources/artifact-tests/Hello.class
sql/core/src/test/resources/artifact-tests/IntSumUdf.class
sql/core/src/test/resources/artifact-tests/smallClassFile.class
diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh
index 563a7e1acab4f..68c61232ea2af 100755
--- a/dev/test-dependencies.sh
+++ b/dev/test-dependencies.sh
@@ -37,6 +37,9 @@ HADOOP_HIVE_PROFILES=(
hadoop-3-hive-2.3
)
+MVN_EXEC_PLUGIN_VERSION=$(build/mvn help:evaluate \
+ -Dexpression=exec-maven-plugin.version -q -DforceStdout | grep -E "[0-9]+\.[0-9]+\.[0-9]+")
+
# We'll switch the version to a temp. one, publish POMs using that new version, then switch back to
# the old version. We need to do this because the `dependency:build-classpath` task needs to
# resolve Spark's internal submodule dependencies.
@@ -47,7 +50,7 @@ OLD_VERSION=$($MVN -q \
-Dexec.executable="echo" \
-Dexec.args='${project.version}' \
--non-recursive \
- org.codehaus.mojo:exec-maven-plugin:1.6.0:exec | grep -E '[0-9]+\.[0-9]+\.[0-9]+')
+ org.codehaus.mojo:exec-maven-plugin:${MVN_EXEC_PLUGIN_VERSION}:exec | grep -E '[0-9]+\.[0-9]+\.[0-9]+')
# dependency:get for guava and jetty-io are workaround for SPARK-37302.
GUAVA_VERSION=$(build/mvn help:evaluate -Dexpression=guava.version -q -DforceStdout | grep -E "^[0-9\.]+")
build/mvn dependency:get -Dartifact=com.google.guava:guava:${GUAVA_VERSION} -q
@@ -61,7 +64,7 @@ SCALA_BINARY_VERSION=$($MVN -q \
-Dexec.executable="echo" \
-Dexec.args='${scala.binary.version}' \
--non-recursive \
- org.codehaus.mojo:exec-maven-plugin:1.6.0:exec | grep -E '[0-9]+\.[0-9]+')
+ org.codehaus.mojo:exec-maven-plugin:${MVN_EXEC_PLUGIN_VERSION}:exec | grep -E '[0-9]+\.[0-9]+')
if [[ "$SCALA_BINARY_VERSION" != "2.13" ]]; then
echo "Skip dependency testing on $SCALA_BINARY_VERSION"
exit 0
diff --git a/dev/tox.ini b/dev/tox.ini
index 05a6b16a03bd9..8a8e03ec9be3e 100644
--- a/dev/tox.ini
+++ b/dev/tox.ini
@@ -43,7 +43,8 @@ per-file-ignores =
python/pyspark/sql/tests/*.py: F403,
python/pyspark/streaming/tests/*.py: F403,
python/pyspark/tests/*.py: F403,
- python/pyspark/testing/*: F401
+ python/pyspark/testing/*.py: F401,
+ python/pyspark/testing/tests/*.py: F403
exclude =
*/target/*,
docs/.local_ruby_bundle/,
diff --git a/docs/Gemfile b/docs/Gemfile
index 68727dee9e1fb..533b4d9d78536 100644
--- a/docs/Gemfile
+++ b/docs/Gemfile
@@ -22,7 +22,7 @@ source "https://rubygems.org"
# To update the lock file, run `bundle update`.
# Version constraint reference: https://guides.rubygems.org/patterns/#declaring-dependencies
-gem "jekyll", "~> 4.3"
+gem "jekyll", "~> 4.4"
gem "jekyll-redirect-from", "~> 0.16"
# This resolves a build issue on Apple Silicon.
# See: https://issues.apache.org/jira/browse/SPARK-38488
diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
index fab7bc1bcd37b..03617ea776421 100644
--- a/docs/Gemfile.lock
+++ b/docs/Gemfile.lock
@@ -3,18 +3,18 @@ GEM
specs:
addressable (2.8.7)
public_suffix (>= 2.0.2, < 7.0)
- base64 (0.2.0)
- bigdecimal (3.1.9)
+ base64 (0.3.0)
+ bigdecimal (3.2.2)
colorator (1.1.0)
concurrent-ruby (1.3.5)
- csv (3.3.2)
+ csv (3.3.5)
em-websocket (0.5.3)
eventmachine (>= 0.12.9)
http_parser.rb (~> 0)
eventmachine (1.2.7)
- ffi (1.17.1)
+ ffi (1.17.2)
forwardable-extended (2.6.0)
- google-protobuf (4.29.3-arm64-darwin)
+ google-protobuf (4.31.1)
bigdecimal
rake (>= 13)
http_parser.rb (0.8.0)
@@ -45,7 +45,7 @@ GEM
sass-embedded (~> 1.75)
jekyll-watch (2.2.1)
listen (~> 3.0)
- json (2.9.1)
+ json (2.12.2)
kramdown (2.5.1)
rexml (>= 3.3.9)
kramdown-parser-gfm (1.1.0)
@@ -57,16 +57,17 @@ GEM
mercenary (0.4.0)
pathutil (0.16.2)
forwardable-extended (~> 2.6)
- public_suffix (6.0.1)
- rake (13.2.1)
+ public_suffix (6.0.2)
+ rake (13.3.0)
rb-fsevent (0.11.2)
rb-inotify (0.11.1)
ffi (~> 1.0)
rexml (3.3.9)
- rouge (4.5.1)
+ rouge (4.5.2)
safe_yaml (1.0.5)
- sass-embedded (1.83.4-arm64-darwin)
- google-protobuf (~> 4.29)
+ sass-embedded (1.89.2)
+ google-protobuf (~> 4.31)
+ rake (>= 13)
terminal-table (3.0.2)
unicode-display_width (>= 1.1.1, < 3)
unicode-display_width (2.6.0)
@@ -77,7 +78,7 @@ PLATFORMS
DEPENDENCIES
ffi (~> 1.15)
- jekyll (~> 4.3)
+ jekyll (~> 4.4)
jekyll-redirect-from (~> 0.16)
rexml (~> 3.3.9)
diff --git a/docs/_layouts/global.html b/docs/_layouts/global.html
index 4d6ddfc2c74a1..115f5c98ce65f 100755
--- a/docs/_layouts/global.html
+++ b/docs/_layouts/global.html
@@ -80,6 +80,7 @@
GraphX (Graph Processing)
SparkR (R on Spark)
PySpark (Python on Spark)
+ Declarative Pipelines
diff --git a/docs/_plugins/build_api_docs.rb b/docs/_plugins/build_api_docs.rb
index 0aa0db0bf9898..918d99cd2d028 100644
--- a/docs/_plugins/build_api_docs.rb
+++ b/docs/_plugins/build_api_docs.rb
@@ -45,7 +45,7 @@ def build_spark_if_necessary
print_header "Building Spark."
cd(SPARK_PROJECT_ROOT)
- command = "build/sbt -Phive -Pkinesis-asl clean package"
+ command = "NO_PROVIDED_SPARK_JARS=0 build/sbt -Phive -Pkinesis-asl clean package"
puts "Running '#{command}'; this may take a few minutes..."
system(command) || raise("Failed to build Spark")
$spark_package_is_built = true
@@ -168,7 +168,6 @@ def build_python_docs
mkdir_p "api/python"
puts "cp -r ../python/docs/build/html/. api/python"
- rm_r("../python/docs/build/html/_sources")
cp_r("../python/docs/build/html/.", "api/python")
end
diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md
index 7d51801edc67c..edd5e3e7f85d6 100644
--- a/docs/core-migration-guide.md
+++ b/docs/core-migration-guide.md
@@ -27,6 +27,7 @@ license: |
- Since Spark 4.1, Spark Master deamon provides REST API by default. To restore the behavior before Spark 4.1, you can set `spark.master.rest.enabled` to `false`.
- Since Spark 4.1, Spark will compress RDD checkpoints by default. To restore the behavior before Spark 4.1, you can set `spark.checkpoint.compress` to `false`.
- Since Spark 4.1, Spark uses Apache Hadoop Magic Committer for all S3 buckets by default. To restore the behavior before Spark 4.0, you can set `spark.hadoop.fs.s3a.committer.magic.enabled=false`.
+- Since Spark 4.1, `java.lang.InternalError` encountered during file reading will no longer fail the task if the configuration `spark.sql.files.ignoreCorruptFiles` or the data source option `ignoreCorruptFiles` is set to `true`.
## Upgrading from Core 3.5 to 4.0
@@ -58,7 +59,7 @@ license: |
- Since Spark 4.0, `spark.shuffle.unsafe.file.output.buffer` is deprecated though still works. Use `spark.shuffle.localDisk.file.output.buffer` instead.
-- Since Spark 4.0, when reading files hits `org.apache.hadoop.security.AccessControlException` and `org.apache.hadoop.hdfs.BlockMissingException`, the exception will be thrown and fail the task, even if `spark.files.ignoreCorruptFiles` is set to `true`.
+- Since Spark 4.0, `org.apache.hadoop.security.AccessControlException` or `org.apache.hadoop.hdfs.BlockMissingException` encountered during file reading will fail the task even if the configuration `spark.sql.files.ignoreCorruptFiles` or the data source option `ignoreCorruptFiles` is set to `true`.
## Upgrading from Core 3.5.3 to 3.5.4
diff --git a/docs/declarative-pipelines-programming-guide.md b/docs/declarative-pipelines-programming-guide.md
new file mode 100644
index 0000000000000..3e33153e3d252
--- /dev/null
+++ b/docs/declarative-pipelines-programming-guide.md
@@ -0,0 +1,418 @@
+---
+layout: global
+title: Spark Declarative Pipelines Programming Guide
+displayTitle: Spark Declarative Pipelines Programming Guide
+license: |
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+---
+
+* Table of contents
+{:toc}
+
+## What is Spark Declarative Pipelines (SDP)?
+
+Spark Declarative Pipelines (SDP) is a declarative framework for building reliable, maintainable, and testable data pipelines on Spark. SDP simplifies ETL development by allowing you to focus on the transformations you want to apply to your data, rather than the mechanics of pipeline execution.
+
+SDP is designed for both batch and streaming data processing, supporting common use cases such as:
+- Data ingestion from cloud storage (Amazon S3, Azure ADLS Gen2, Google Cloud Storage)
+- Data ingestion from message buses (Apache Kafka, Amazon Kinesis, Google Pub/Sub, Azure EventHub)
+- Incremental batch and streaming transformations
+
+The key advantage of SDP is its declarative approach - you define what tables should exist and what their contents should be, and SDP handles the orchestration, compute management, and error handling automatically.
+
+
+
+## Key Concepts
+
+### Flows
+
+A flow is the foundational data processing concept in SDP which supports both streaming and batch semantics. A flow reads data from a source, applies user-defined processing logic, and writes the result into a target dataset.
+
+For example, when you author a query like:
+
+```sql
+CREATE STREAMING TABLE target_table AS
+SELECT * FROM STREAM source_table
+```
+
+SDP creates the table named `target_table` along with a flow that reads new data from `source_table` and writes it to `target_table`.
+
+### Datasets
+
+A dataset is queryable object that's the output of one of more flows within a pipeline. Flows in the pipeline can also read from datasets produced in the pipeline.
+
+- **Streaming Table** – a definition of a table and one or more streaming flows written into it. Streaming tables support incremental processing of data, allowing you to process only new data as it arrives.
+- **Materialized View** – is a view that is precomputed into a table. A materialized view always has exactly one batch flow writing to it.
+- **Temporary View** – a view that is scoped to an execution of the pipeline. It can be referenced from flows within the pipeline. It's useful for encapsulating transformations and intermediate logical entities that multiple other elements of the pipeline depend on.
+
+### Pipelines
+
+A pipeline is the primary unit of development and execution in SDP. A pipeline can contain one or more flows, streaming tables, and materialized views. While your pipeline runs, it analyzes the dependencies of your defined objects and orchestrates their order of execution and parallelization automatically.
+
+### Pipeline Projects
+
+A pipeline project is a set of source files that contain code that define the datasets and flows that make up a pipeline. These source files can be `.py` or `.sql` files.
+
+A YAML-formatted pipeline spec file contains the top-level configuration for the pipeline project. It supports the following fields:
+- **definitions** (Required) - Paths where definition files can be found.
+- **database** (Optional) - The default target database for pipeline outputs.
+- **catalog** (Optional) - The default target catalog for pipeline outputs.
+- **configuration** (Optional) - Map of Spark configuration properties.
+
+An example pipeline spec file:
+
+```yaml
+name: my_pipeline
+definitions:
+ - glob:
+ include: transformations/**/*.py
+ - glob:
+ include: transformations/**/*.sql
+catalog: my_catalog
+database: my_db
+configuration:
+ spark.sql.shuffle.partitions: "1000"
+```
+
+It's conventional to name pipeline spec files `pipeline.yml`.
+
+The `spark-pipelines init` command, described below, makes it easy to generate a pipeline project with default configuration and directory structure.
+
+
+## The `spark-pipelines` Command Line Interface
+
+The `spark-pipelines` command line interface (CLI) is the primary way to execute a pipeline. It also contains an `init` subcommand for generating a pipeline project and a `dry-run` subcommand for validating a pipeline.
+
+`spark-pipelines` is built on top of `spark-submit`, meaning that it supports all cluster managers supported by `spark-submit`. It supports all `spark-submit` arguments except for `--class`.
+
+### `spark-pipelines init`
+
+`spark-pipelines init --name my_pipeline` generates a simple pipeline project, inside a directory named "my_pipeline", including a spec file and example definitions.
+
+### `spark-pipelines run`
+
+`spark-pipelines run` launches an execution of a pipeline and monitors its progress until it completes. The `--spec` parameter allows selecting the pipeline spec file. If not provided, the CLI will look in the current directory and parent directories for a file named `pipeline.yml` or `pipeline.yaml`.
+
+### `spark-pipelines dry-run`
+
+`spark-pipelines dry-run` launches an execution of a pipeline that doesn't write or read any data, but catches many kinds of errors that would be caught if the pipeline were to actually run. E.g.
+- Syntax errors – e.g. invalid Python or SQL code
+- Analysis errors – e.g. selecting from a table that doesn't exist or selecting a column that doesn't exist
+- Graph validation errors - e.g. cyclic dependencies
+
+## Programming with SDP in Python
+
+SDP Python functions are defined in the `pyspark.pipelines` module. Your pipelines implemented with the Python API must import this module. It's common to alias the module to `dp` to limit the number of characters you need to type when using its APIs.
+
+```python
+from pyspark import pipelines as dp
+```
+
+### Creating a Materialized View with Python
+
+The `@dp.materialized_view` decorator tells SDP to create a materialized view based on the results returned by a function that performs a batch read:
+
+```python
+from pyspark import pipelines as dp
+
+@dp.materialized_view
+def basic_mv():
+ return spark.table("samples.nyctaxi.trips")
+```
+
+Optionally, you can specify the table name using the `name` argument:
+
+```python
+from pyspark import pipelines as dp
+
+@dp.materialized_view(name="trips_mv")
+def basic_mv():
+ return spark.table("samples.nyctaxi.trips")
+```
+
+### Creating a Temporary View with Python
+
+The `@dp.temporary_view` decorator tells SDP to create a temporary view based on the results returned by a function that performs a batch read:
+
+```python
+from pyspark import pipelines as dp
+
+@dp.temporary_view
+def basic_tv():
+ return spark.table("samples.nyctaxi.trips")
+```
+
+This temporary view can be read by other queries within the pipeline, but can't be read outside the scope of the pipeline.
+
+### Creating a Streaming Table with Python
+
+Similarly, you can create a streaming table by using the `@dp.table` decorator with a function that performs a streaming read:
+
+```python
+from pyspark import pipelines as dp
+
+@dp.table
+def basic_st():
+ return spark.readStream.table("samples.nyctaxi.trips")
+```
+
+### Loading Data from a Streaming Source
+
+SDP supports loading data from all formats supported by Spark. For example, you can create a streaming table whose query reads from a Kafka topic:
+
+```python
+from pyspark import pipelines as dp
+
+@dp.table
+def ingestion_st():
+ return (
+ spark.readStream.format("kafka")
+ .option("kafka.bootstrap.servers", "localhost:9092")
+ .option("subscribe", "orders")
+ .load()
+ )
+```
+
+For batch reads:
+
+```python
+from pyspark import pipelines as dp
+
+@dp.materialized_view
+def batch_mv():
+ return spark.read.format("json").load("/datasets/retail-org/sales_orders")
+```
+
+### Querying Tables Defined in Your Pipeline
+
+You can reference other tables defined in your pipeline in the same way you'd reference tables defined outside your pipeline:
+
+```python
+from pyspark import pipelines as dp
+from pyspark.sql.functions import col
+
+@dp.table
+def orders():
+ return (
+ spark.readStream.format("kafka")
+ .option("kafka.bootstrap.servers", "localhost:9092")
+ .option("subscribe", "orders")
+ .load()
+ )
+
+@dp.materialized_view
+def customers():
+ return spark.read.format("csv").option("header", True).load("/datasets/retail-org/customers")
+
+@dp.materialized_view
+def customer_orders():
+ return (spark.table("orders")
+ .join(spark.table("customers"), "customer_id")
+ .select("customer_id",
+ "order_number",
+ "state",
+ col("order_datetime").cast("int").cast("timestamp").cast("date").alias("order_date"),
+ )
+ )
+
+@dp.materialized_view
+def daily_orders_by_state():
+ return (spark.table("customer_orders")
+ .groupBy("state", "order_date")
+ .count().withColumnRenamed("count", "order_count")
+ )
+```
+
+### Creating Tables in a For Loop
+
+You can use Python `for` loops to create multiple tables programmatically:
+
+```python
+from pyspark import pipelines as dp
+from pyspark.sql.functions import collect_list, col
+
+@dp.temporary_view()
+def customer_orders():
+ orders = spark.table("samples.tpch.orders")
+ customer = spark.table("samples.tpch.customer")
+
+ return (orders.join(customer, orders.o_custkey == customer.c_custkey)
+ .select(
+ col("c_custkey").alias("custkey"),
+ col("c_name").alias("name"),
+ col("c_nationkey").alias("nationkey"),
+ col("c_phone").alias("phone"),
+ col("o_orderkey").alias("orderkey"),
+ col("o_orderstatus").alias("orderstatus"),
+ col("o_totalprice").alias("totalprice"),
+ col("o_orderdate").alias("orderdate"))
+ )
+
+@dp.temporary_view()
+def nation_region():
+ nation = spark.table("samples.tpch.nation")
+ region = spark.table("samples.tpch.region")
+
+ return (nation.join(region, nation.n_regionkey == region.r_regionkey)
+ .select(
+ col("n_name").alias("nation"),
+ col("r_name").alias("region"),
+ col("n_nationkey").alias("nationkey")
+ )
+ )
+
+# Extract region names from region table
+region_list = spark.table("samples.tpch.region").select(collect_list("r_name")).collect()[0][0]
+
+# Iterate through region names to create new region-specific materialized views
+for region in region_list:
+ @dp.table(name=f"{region.lower().replace(' ', '_')}_customer_orders")
+ def regional_customer_orders(region_filter=region):
+ customer_orders = spark.table("customer_orders")
+ nation_region = spark.table("nation_region")
+
+ return (customer_orders.join(nation_region, customer_orders.nationkey == nation_region.nationkey)
+ .select(
+ col("custkey"),
+ col("name"),
+ col("phone"),
+ col("nation"),
+ col("region"),
+ col("orderkey"),
+ col("orderstatus"),
+ col("totalprice"),
+ col("orderdate")
+ ).filter(f"region = '{region_filter}'")
+ )
+```
+
+### Using Multiple Flows to Write to a Single Target
+
+You can create multiple flows that append data to the same target:
+
+```python
+from pyspark import pipelines as dp
+
+# create a streaming table
+dp.create_streaming_table("customers_us")
+
+# add the first append flow
+@dp.append_flow(target = "customers_us")
+def append1():
+ return spark.readStream.table("customers_us_west")
+
+# add the second append flow
+@dp.append_flow(target = "customers_us")
+def append2():
+ return spark.readStream.table("customers_us_east")
+```
+
+## Programming with SDP in SQL
+
+### Creating a Materialized View with SQL
+
+The basic syntax for creating a materialized view with SQL is:
+
+```sql
+CREATE MATERIALIZED VIEW basic_mv
+AS SELECT * FROM samples.nyctaxi.trips;
+```
+
+### Creating a Temporary View with SQL
+
+The basic syntax for creating a temporary view with SQL is:
+
+```sql
+CREATE TEMPORARY VIEW basic_tv
+AS SELECT * FROM samples.nyctaxi.trips;
+```
+
+### Creating a Streaming Table with SQL
+
+When creating a streaming table, use the `STREAM` keyword to indicate streaming semantics for the source:
+
+```sql
+CREATE STREAMING TABLE basic_st
+AS SELECT * FROM STREAM samples.nyctaxi.trips;
+```
+
+### Querying Tables Defined in Your Pipeline
+
+You can reference other tables defined in your pipeline:
+
+```sql
+CREATE STREAMING TABLE orders
+AS SELECT * FROM STREAM orders_source;
+
+CREATE MATERIALIZED VIEW customers
+AS SELECT * FROM customers_source;
+
+CREATE MATERIALIZED VIEW customer_orders
+AS SELECT
+ c.customer_id,
+ o.order_number,
+ c.state,
+ date(timestamp(int(o.order_datetime))) order_date
+FROM orders o
+INNER JOIN customers c
+ON o.customer_id = c.customer_id;
+
+CREATE MATERIALIZED VIEW daily_orders_by_state
+AS SELECT state, order_date, count(*) order_count
+FROM customer_orders
+GROUP BY state, order_date;
+```
+
+### Using Multiple Flows to Write to a Single Target
+
+You can create multiple flows that append data to the same target:
+
+```sql
+-- create a streaming table
+CREATE STREAMING TABLE customers_us;
+
+-- add the first append flow
+CREATE FLOW append1
+AS INSERT INTO customers_us
+SELECT * FROM STREAM(customers_us_west);
+
+-- add the second append flow
+CREATE FLOW append2
+AS INSERT INTO customers_us
+SELECT * FROM STREAM(customers_us_east);
+```
+
+## Important Considerations
+
+### Python Considerations
+
+- SDP evaluates the code that defines a pipeline multiple times during planning and pipeline runs. Python functions that define datasets should include only the code required to define the table or view.
+- The function used to define a dataset must return a Spark DataFrame.
+- Never use methods that save or write to files or tables as part of your SDP dataset code.
+
+Examples of Apache Spark operations that should never be used in SDP code:
+- `collect()`
+- `count()`
+- `toPandas()`
+- `save()`
+- `saveAsTable()`
+- `start()`
+- `toTable()`
+
+### SQL Considerations
+
+- The `PIVOT` clause is not supported in SDP SQL.
+- When using the `for` loop pattern to define datasets in Python, ensure that the list of values passed to the `for` loop is always additive.
diff --git a/docs/img/declarative-pipelines-dataflow-graph.png b/docs/img/declarative-pipelines-dataflow-graph.png
new file mode 100644
index 0000000000000..cadd2d81482ff
Binary files /dev/null and b/docs/img/declarative-pipelines-dataflow-graph.png differ
diff --git a/docs/index.md b/docs/index.md
index 88ea8ad9efc3b..cb32ddcde7e2b 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -34,7 +34,7 @@ source, visit [Building Spark](building-spark.html).
Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS), and it should run on any platform that runs a supported version of Java. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation.
-Spark runs on Java 17/21, Scala 2.13, Python 3.9+, and R 3.5+ (Deprecated).
+Spark runs on Java 17/21, Scala 2.13, Python 3.10+, and R 3.5+ (Deprecated).
When using the Scala API, it is necessary for applications to use the same version of Scala that Spark was compiled for. Since Spark 4.0.0, it's Scala 2.13.
# Running the Examples and Shell
@@ -112,6 +112,7 @@ options for deployment:
* [SparkR (Deprecated)](sparkr.html): processing data with Spark in R
* [PySpark](api/python/getting_started/index.html): processing data with Spark in Python
* [Spark SQL CLI](sql-distributed-sql-engine-spark-sql-cli.html): processing data with SQL on the command line
+* [Declarative Pipelines](declarative-pipelines-programming-guide.html): building data pipelines that create and maintain multiple tables
**API Docs:**
diff --git a/docs/quick-start.md b/docs/quick-start.md
index 5a03af98cd832..83fc3421c0381 100644
--- a/docs/quick-start.md
+++ b/docs/quick-start.md
@@ -462,7 +462,7 @@ Lines with a: 46, Lines with b: 23
-Other dependency management tools such as Conda and pip can be also used for custom classes or third-party libraries. See also [Python Package Management](api/python/user_guide/python_packaging.html).
+Other dependency management tools such as Conda and pip can be also used for custom classes or third-party libraries. See also [Python Package Management](api/python/tutorial/python_packaging.html).
# Where to Go from Here
Congratulations on running your first Spark application!
diff --git a/docs/rdd-programming-guide.md b/docs/rdd-programming-guide.md
index 400f8a512e7a7..6e69cec23a40d 100644
--- a/docs/rdd-programming-guide.md
+++ b/docs/rdd-programming-guide.md
@@ -39,7 +39,7 @@ along with if you launch Spark's interactive shell -- either `bin/spark-shell` f
-Spark {{site.SPARK_VERSION}} works with Python 3.9+. It can use the standard CPython interpreter,
+Spark {{site.SPARK_VERSION}} works with Python 3.10+. It can use the standard CPython interpreter,
so C libraries like NumPy can be used. It also works with PyPy 7.3.6+.
Spark applications in Python can either be run with the `bin/spark-submit` script which includes Spark at runtime, or by including it in your setup.py as:
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 8ad9078a6a9aa..9f385a11d92e1 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -44,7 +44,7 @@ Cluster administrators should use [Pod Security Policies](https://kubernetes.io/
# Prerequisites
-* A running Kubernetes cluster at version >= 1.31 with access configured to it using
+* A running Kubernetes cluster at version >= 1.32 with access configured to it using
[kubectl](https://kubernetes.io/docs/reference/kubectl/). If you do not already have a working Kubernetes cluster,
you may set up a test cluster on your local machine using
[minikube](https://kubernetes.io/docs/getting-started-guides/minikube/).
@@ -643,7 +643,8 @@ See the [configuration page](configuration.html) for information on Spark config
Container image to use for the Spark application.
This is usually of the form example.com/repo/spark:v1.0.0.
This configuration is required and must be provided by the user, unless explicit
- images are provided for each different container type.
+ images are provided for each different container type. Note that {{SPARK_VERSION}}
+ is the built-in variable that will be substituted with current Spark's version.
2.3.0 |
@@ -651,7 +652,8 @@ See the [configuration page](configuration.html) for information on Spark config
spark.kubernetes.driver.container.image |
(value of spark.kubernetes.container.image) |
- Custom container image to use for the driver.
+ Custom container image to use for the driver. Note that {{SPARK_VERSION}}
+ is the built-in variable that will be substituted with current Spark's version.
|
2.3.0 |
@@ -659,7 +661,8 @@ See the [configuration page](configuration.html) for information on Spark config
spark.kubernetes.executor.container.image |
(value of spark.kubernetes.container.image) |
- Custom container image to use for executors.
+ Custom container image to use for executors. Note that {{SPARK_VERSION}}
+ is the built-in variable that will be substituted with current Spark's version.
|
2.3.0 |
@@ -1988,10 +1991,10 @@ Install Apache YuniKorn:
```bash
helm repo add yunikorn https://apache.github.io/yunikorn-release
helm repo update
-helm install yunikorn yunikorn/yunikorn --namespace yunikorn --version 1.6.3 --create-namespace --set embedAdmissionController=false
+helm install yunikorn yunikorn/yunikorn --namespace yunikorn --version 1.7.0 --create-namespace --set embedAdmissionController=false
```
-The above steps will install YuniKorn v1.6.3 on an existing Kubernetes cluster.
+The above steps will install YuniKorn v1.7.0 on an existing Kubernetes cluster.
##### Get started
diff --git a/docs/sql-data-sources-hive-tables.md b/docs/sql-data-sources-hive-tables.md
index d45174425f470..977efa8f24330 100644
--- a/docs/sql-data-sources-hive-tables.md
+++ b/docs/sql-data-sources-hive-tables.md
@@ -130,7 +130,7 @@ The following options can be used to configure the version of Hive that is used
2.3.10 |
Version of the Hive metastore. Available
- options are 2.0.0 through 2.3.10, 3.0.0 through 3.1.3, and 4.0.0 through 4.0.1.
+ options are 2.0.0 through 2.3.10, 3.0.0 through 3.1.3, and 4.0.0 through 4.1.0.
|
1.4.0 |
diff --git a/docs/sql-getting-started.md b/docs/sql-getting-started.md
index c2733a0ebf012..11733a308bf0a 100644
--- a/docs/sql-getting-started.md
+++ b/docs/sql-getting-started.md
@@ -124,9 +124,9 @@ latter form, which is future proof and won't break with column names that
are also attributes on the DataFrame class.
{% include_example untyped_ops python/sql/basic.py %}
-For a complete list of the types of operations that can be performed on a DataFrame refer to the [API Documentation](api/python/reference/pyspark.sql.html#dataframe-apis).
+For a complete list of the types of operations that can be performed on a DataFrame refer to the [API Documentation](api/python/reference/pyspark.sql/dataframe.html).
-In addition to simple column references and expressions, DataFrames also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/python/reference/pyspark.sql.html#functions).
+In addition to simple column references and expressions, DataFrames also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/python/reference/pyspark.sql/functions.html).
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index edc0a61d8f1d8..d199dc31c58be 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -1059,7 +1059,7 @@ Python UDF registration is unchanged.
Spark SQL is designed to be compatible with the Hive Metastore, SerDes and UDFs.
Currently, Hive SerDes and UDFs are based on built-in Hive,
and Spark SQL can be connected to different versions of Hive Metastore
-(from 2.0.0 to 2.3.10 and 3.0.0 to 3.1.3. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
+(from 2.0.0 to 2.3.10 and 3.0.0 to 4.1.0. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore).
#### Deploying in Existing Hive Warehouses
{:.no_toc}
diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index ef79ef1871363..8d2c13beff975 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -110,19 +110,20 @@ Besides, the ANSI SQL mode disallows the following type conversions which are al
The valid combinations of source and target data type in a `CAST` expression are given by the following table.
“Y” indicates that the combination is syntactically valid without restriction and “N” indicates that the combination is not valid.
-| Source\Target | Numeric | String | Date | Timestamp | Timestamp_NTZ | Interval | Boolean | Binary | Array | Map | Struct |
-|---------------|--------------------------------------|--------------------------------------|--------------------------------------|--------------------------------------|--------------------------------------|--------------------------------------|--------------------------------------|--------|--------------------------------------|--------------------------------------|--------------------------------------|
-| Numeric | **Y** | **Y** | N | **Y** | N | **Y** | Y | N | N | N | N |
-| String | **Y** | Y | **Y** | **Y** | **Y** | **Y** | **Y** | Y | N | N | N |
-| Date | N | Y | Y | Y | Y | N | N | N | N | N | N |
-| Timestamp | **Y** | Y | Y | Y | Y | N | N | N | N | N | N |
-| Timestamp_NTZ | N | Y | Y | Y | Y | N | N | N | N | N | N |
-| Interval | **Y** | Y | N | N | N | Y | N | N | N | N | N |
-| Boolean | Y | Y | N | N | N | N | Y | N | N | N | N |
-| Binary | N | Y | N | N | N | N | N | Y | N | N | N |
-| Array | N | Y | N | N | N | N | N | N | **Y** | N | N |
-| Map | N | Y | N | N | N | N | N | N | N | **Y** | N |
-| Struct | N | Y | N | N | N | N | N | N | N | N | **Y** |
+| Source\Target | Numeric | String | Date | Time | Timestamp | Timestamp_NTZ | Interval | Boolean | Binary | Array | Map | Struct |
+|---------------|--------------------------------------|--------------------------------------|--------------------------------------|------|--------------------------------------|--------------------------------------|--------------------------------------|--------------------------------------|--------|--------------------------------------|--------------------------------------|--------------------------------------|
+| Numeric | **Y** | **Y** | N | N |**Y** | N | **Y** | Y | N | N | N | N |
+| String | **Y** | Y | **Y** | Y |**Y** | **Y** | **Y** | **Y** | Y | N | N | N |
+| Date | N | Y | Y | N | Y | Y | N | N | N | N | N | N |
+| Time | N | Y | N | Y | N | N | N | N | N | N | N | N |
+| Timestamp | **Y** | Y | Y | N | Y | Y | N | N | N | N | N | N |
+| Timestamp_NTZ | N | Y | Y | N | Y | Y | N | N | N | N | N | N |
+| Interval | **Y** | Y | N | N | N | N | Y | N | N | N | N | N |
+| Boolean | Y | Y | N | N | N | N | N | Y | N | N | N | N |
+| Binary | N | Y | N | N | N | N | N | N | Y | N | N | N |
+| Array | N | Y | N | N | N | N | N | N | N | **Y** | N | N |
+| Map | N | Y | N | N | N | N | N | N | N | N | **Y** | N |
+| Struct | N | Y | N | N | N | N | N | N | N | N | N | **Y** |
In the table above, all the `CAST`s with new syntax are marked as red **Y**:
* CAST(Numeric AS Numeric): raise an overflow exception if the value is out of the target data type's range.
@@ -198,19 +199,20 @@ While casting of a decimal with a fraction to an interval type with SECOND as th
### Store assignment
As mentioned at the beginning, when `spark.sql.storeAssignmentPolicy` is set to `ANSI`(which is the default value), Spark SQL complies with the ANSI store assignment rules on table insertions. The valid combinations of source and target data type in table insertions are given by the following table.
-| Source\Target | Numeric | String | Date | Timestamp | Timestamp_NTZ | Interval | Boolean | Binary | Array | Map | Struct |
-|:-------------:|:-------:|:------:|:----:|:---------:|---------------|:--------:|:-------:|:------:|:-----:|:---:|:------:|
-| Numeric | Y | Y | N | N | N | N | N | N | N | N | N |
-| String | N | Y | N | N | N | N | N | N | N | N | N |
-| Date | N | Y | Y | Y | Y | N | N | N | N | N | N |
-| Timestamp | N | Y | Y | Y | Y | N | N | N | N | N | N |
-| Timestamp_NTZ | N | Y | Y | Y | Y | N | N | N | N | N | N |
-| Interval | N | Y | N | N | N | N* | N | N | N | N | N |
-| Boolean | N | Y | N | N | N | N | Y | N | N | N | N |
-| Binary | N | Y | N | N | N | N | N | Y | N | N | N |
-| Array | N | N | N | N | N | N | N | N | Y** | N | N |
-| Map | N | N | N | N | N | N | N | N | N | Y** | N |
-| Struct | N | N | N | N | N | N | N | N | N | N | Y** |
+| Source\Target | Numeric | String | Date | Time | Timestamp | Timestamp_NTZ | Interval | Boolean | Binary | Array | Map | Struct |
+|:-------------:|:-------:|:------:|:----:|:----:|:---------:|---------------|:--------:|:-------:|:------:|:-----:|:---:|:------:|
+| Numeric | Y | Y | N | N | N | N | N | N | N | N | N | N |
+| String | N | Y | N | Y | N | N | N | N | N | N | N | N |
+| Date | N | Y | Y | N | Y | Y | N | N | N | N | N | N |
+| Time | N | Y | N | Y | N | N | N | N | N | N | N | N |
+| Timestamp | N | Y | Y | N | Y | Y | N | N | N | N | N | N |
+| Timestamp_NTZ | N | Y | Y | N | Y | Y | N | N | N | N | N | N |
+| Interval | N | Y | N | N | N | N | N* | N | N | N | N | N |
+| Boolean | N | Y | N | N | N | N | N | Y | N | N | N | N |
+| Binary | N | Y | N | N | N | N | N | N | Y | N | N | N |
+| Array | N | N | N | N | N | N | N | N | N | Y** | N | N |
+| Map | N | N | N | N | N | N | N | N | N | N | Y** | N |
+| Struct | N | N | N | N | N | N | N | N | N | N | N | Y** |
\* Spark doesn't support interval type table column.
@@ -239,6 +241,7 @@ At the heart of this conflict resolution is the Type Precedence List which defin
| Float | Float -> Double |
| Double | Double |
| Date | Date -> Timestamp_NTZ -> Timestamp |
+| Time | Time |
| Timestamp | Timestamp |
| String | String, Long -> Double, Date -> Timestamp_NTZ -> Timestamp , Boolean, Binary ** |
| Binary | Binary |
@@ -385,6 +388,8 @@ When ANSI mode is on, it throws exceptions for invalid operations. You can use t
- `try_make_timestamp_ntz`: identical to the function `make_timestamp_ntz`, except that it returns `NULL` result instead of throwing an exception on error.
- `try_make_interval`: identical to the function `make_interval`, except that it returns `NULL` result instead of throwing an exception on invalid interval.
- `try_to_time`: identical to the function `to_time`, except that it returns `NULL` result instead of throwing an exception on string parsing error.
+ - `try_to_date`: identical to the function `to_date`, except that it returns `NULL` result instead of throwing an exception on string parsing error.
+
### SQL Keywords (optional, disabled by default)
@@ -794,6 +799,7 @@ Below is a list of all the keywords in Spark SQL.
|WINDOW|non-reserved|non-reserved|reserved|
|WITH|reserved|non-reserved|reserved|
|WITHIN|reserved|non-reserved|reserved|
+|WITHOUT|non-reserved|non-reserved|non-reserved|
|X|non-reserved|non-reserved|non-reserved|
|YEAR|non-reserved|non-reserved|non-reserved|
|YEARS|non-reserved|non-reserved|non-reserved|
diff --git a/docs/sql-ref-literals.md b/docs/sql-ref-literals.md
index 7a10676cce237..13f310503b51d 100644
--- a/docs/sql-ref-literals.md
+++ b/docs/sql-ref-literals.md
@@ -373,7 +373,7 @@ SELECT -3.E-3D AS col;
### Datetime Literal
-A datetime literal is used to specify a date or timestamp value.
+A datetime literal is used to specify a date, time or timestamp value.
#### Date Syntax
@@ -410,6 +410,44 @@ SELECT DATE '2011-11-11' AS col;
+----------+
```
+#### Time Syntax
+
+```sql
+TIME { '[h]h:[m]m[:]' |
+ '[h]h:[m]m:[s]s[.]' |
+ '[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]'}
+```
+**Note:** defaults to `00` if hour, minute or second is not specified.
+
+#### Time Examples
+
+```sql
+SELECT TIME'12:00' as col;
++--------+
+|col |
++--------+
+|12:00:00|
++--------+
+SELECT TIME'2:0' as col;
++--------+
+|col |
++--------+
+|02:00:00|
++--------+
+SELECT TIME'2:0:3' as col;
++--------+
+|col |
++--------+
+|02:00:03|
++--------+
+SELECT TIME'23:59:59.999999' as col;
++---------------+
+|col |
++---------------+
+|23:59:59.999999|
++---------------+
+```
+
#### Timestamp Syntax
```sql
diff --git a/docs/sql-ref-syntax-ddl-create-function.md b/docs/sql-ref-syntax-ddl-create-function.md
index db54a4bd5ea69..e0e2545f5ee3f 100644
--- a/docs/sql-ref-syntax-ddl-create-function.md
+++ b/docs/sql-ref-syntax-ddl-create-function.md
@@ -1,7 +1,7 @@
---
layout: global
-title: CREATE FUNCTION
-displayTitle: CREATE FUNCTION
+title: CREATE FUNCTION (External)
+displayTitle: CREATE FUNCTION (External)
license: |
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -21,7 +21,7 @@ license: |
### Description
-The `CREATE FUNCTION` statement is used to create a temporary or permanent function
+The `CREATE FUNCTION` statement is used to create a temporary or permanent external function
in Spark. Temporary functions are scoped at a session level where as permanent
functions are created in the persistent catalog and are made available to
all sessions. The resources specified in the `USING` clause are made available
@@ -161,6 +161,7 @@ SELECT simple_udf(c1) AS function_return_value FROM test;
### Related Statements
+* [CREATE FUNCTION (SQL)](sql-ref-syntax-ddl-create-sql-function.html)
* [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
* [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
diff --git a/docs/sql-ref-syntax-ddl-create-sql-function.md b/docs/sql-ref-syntax-ddl-create-sql-function.md
new file mode 100644
index 0000000000000..649cd895a1974
--- /dev/null
+++ b/docs/sql-ref-syntax-ddl-create-sql-function.md
@@ -0,0 +1,303 @@
+---
+layout: global
+title: CREATE FUNCTION (SQL)
+displayTitle: CREATE FUNCTION (SQL)
+license: |
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+---
+
+### Description
+The `CREATE FUNCTION` statement creates a SQL function that can be used in SQL statements. The function can be temporary or permanent, and can return either a scalar value or a table result. The function body can be defined either a SQL expression or a query.
+
+When `TEMPORARY` is specified, the function is only available for the current session. Otherwise, it is persisted in the catalog and available across sessions. The `OR REPLACE` option allows updating an existing function definition, while `IF NOT EXISTS` prevents errors when creating a function that already exists.
+
+The function parameters must be specified with their data types. The return type can be either a scalar data type or a table with an optional schema definition.
+
+
+### Syntax
+
+```sql
+CREATE [OR REPLACE] [TEMPORARY] FUNCTION [IF NOT EXISTS]
+ function_name ( [ function_parameter [, ...] ] )
+ { [ RETURNS data_type ] |
+ RETURNS TABLE [ ( column_spec [, ...]) ] }
+ [ characteristic [...] ]
+ RETURN { expression | query }
+
+function_parameter
+ parameter_name data_type [DEFAULT default_expression] [COMMENT parameter_comment]
+
+column_spec
+ column_name data_type [COMMENT column_comment]
+
+characteristic
+ { LANGUAGE SQL |
+ [NOT] DETERMINISTIC |
+ COMMENT function_comment |
+ [CONTAINS SQL | READS SQL DATA] }
+```
+
+### Parameters
+
+- **OR REPLACE**
+
+ If specified, the function with the same name and signature (number of parameters and parameter types) is replaced. You cannot replace an existing function with a different signature or a procedure. This is mainly useful to update the function body and the return type of the function. You cannot specify this parameter with `IF NOT EXISTS`.
+
+- **TEMPORARY**
+
+ The scope of the function being created. When you specify `TEMPORARY`, the created function is valid and visible in the current session. No persistent entry is made in the catalog.
+
+- **IF NOT EXISTS**
+
+ If specified, creates the function only when it does not exist. The creation of the function succeeds (no error is thrown) if the specified function already exists in the system. You cannot specify this parameter with `OR REPLACE`.
+
+- **function_name**
+
+ A name for the function. For a permanent function, you can optionally qualify the function name, or it will be created under the current catalog and namespace.
+ If the name is not qualified the permanent function is created in the current schema.
+
+ **Syntax:** `[ database_name. ] function_name`
+
+- **function_parameter**
+
+ Specifies a parameter of the function.
+
+ - **[parameter_name](sql-ref-identifier.md)**
+
+ The parameter name must be unique within the function.
+
+ - **[data_type](sql-ref-datatypes.md)**
+
+ Any supported data type.
+
+ - **DEFAULT default_expression**
+
+ An optional default to be used when a function invocation does not assign an argument to the parameter.
+ `default_expression` must be castable to `data_type`.
+ The expression must not reference another parameter or contain a subquery.
+
+ When you specify a default for one parameter, all following parameters must also have a default.
+
+ - **COMMENT comment**
+
+ An optional description of the parameter. `comment` must be a `STRING` literal.
+
+- **RETURNS [data_type](sql-ref-datatypes.md)**
+
+ The return data type of the scalar function. This clause is optional. The data type will be derived from the SQL function body if it is not provided.
+
+- **RETURNS TABLE [ (column_spec [,…] ) ]**
+
+ This clause marks the function as a table function.
+ Optionally it also specifies the signature of the result of the table function.
+ If no column_spec is specified it will be derived from the body of the SQL UDF.
+
+ - **[column_name](sql-ref-identifier.md)**
+
+ The column name must be unique within the signature.
+
+ - **[data_type](sql-ref-datatypes.md)**
+
+ Any supported data type.
+
+ - **COMMENT column_comment**
+
+ An optional description of the column. `comment` must be a `STRING` literal.
+
+- **RETURN \{ expression| query \}**
+
+ The body of the function. For a scalar function, it can either be a query or an expression. For a table function, it can only be a query. The expression cannot contain:
+
+ - [Aggregate functions](sql-ref-functions-builtin.md#aggregate-functions)
+ - [Window functions](sql-ref-functions-builtin.md#analytic-window-functions)
+ - [Ranking functions](sql-ref-functions-builtin.md#ranking-window-functions)
+ - Row producing functions such as `explode`
+
+ Within the body of the function you can refer to parameter by its unqualified name or by qualifying the parameter with the function name.
+
+- **characteristic**
+
+ All characteristic clauses are optional.
+ You can specify any number of them in any order, but you can specify each clause only once.
+
+ - **LANGUAGE SQL**
+
+ The language of the function implementation.
+
+ - **[NOT] DETERMINISTIC**
+
+ Whether the function is deterministic.
+ A function is deterministic when it returns only one result for a given set of arguments.
+ You may mark a function as `DETERMINISTIC` when its body is not and vice versa.
+ A reason for this may be to encourage or discourage query optimizations such as constant
+ folding or query caching.
+ If you do not specify ths option it is derived from the function body.
+
+ - **COMMENT function_comment**
+
+ A comment for the function. `function_comment` must be String literal.
+
+ - **CONTAINS SQL** or **READS SQL DATA**
+
+ Whether a function reads data directly or indirectly from a table or a view.
+ When the function reads SQL data, you cannot specify `CONTAINS SQL`.
+ If you don't specify either clause, the property is derived from the function body.
+
+## Examples
+
+
+### Create and use a SQL scalar function
+
+```sql
+> CREATE VIEW t(c1, c2) AS VALUES (0, 1), (1, 2);
+
+-- Create a temporary function with no parameter.
+> CREATE TEMPORARY FUNCTION hello() RETURNS STRING
+ RETURN 'Hello World!';
+
+> SELECT hello();
+ Hello World!
+
+-- Create a permanent function with parameters.
+> CREATE FUNCTION area(x DOUBLE, y DOUBLE) RETURNS DOUBLE RETURN x * y;
+
+-- Use a SQL function in the SELECT clause of a query.
+> SELECT area(c1, c2) AS area FROM t;
+ 1.0
+ 1.0
+
+-- Use a SQL function in the WHERE clause of a query.
+> SELECT * FROM t WHERE area(c1, c2) > 0;
+ 1 2
+
+-- Compose SQL functions.
+> CREATE FUNCTION square(x DOUBLE) RETURNS DOUBLE RETURN area(x, x);
+
+> SELECT c1, square(c1) AS square FROM t;
+ 0 0.0
+ 1 1.0
+
+-- Create a non-deterministic function
+> CREATE FUNCTION roll_dice()
+ RETURNS INT
+ NOT DETERMINISTIC
+ CONTAINS SQL
+ COMMENT 'Roll a single 6 sided die'
+ RETURN (rand() * 6)::INT + 1;
+-- Roll a single 6-sided die
+> SELECT roll_dice();
+ 3
+```
+
+### Create a SQL table function
+
+```sql
+-- Produce all weekdays between two dates
+> CREATE FUNCTION weekdays(start DATE, end DATE)
+ RETURNS TABLE(day_of_week STRING, day DATE)
+ RETURN SELECT extract(DAYOFWEEK_ISO FROM day), day
+ FROM (SELECT sequence(weekdays.start, weekdays.end)) AS T(days)
+ LATERAL VIEW explode(days) AS day
+ WHERE extract(DAYOFWEEK_ISO FROM day) BETWEEN 1 AND 5;
+
+-- Return all weekdays
+> SELECT weekdays.day_of_week, day
+ FROM weekdays(DATE'2022-01-01', DATE'2022-01-14');
+ 1 2022-01-03
+ 2 2022-01-04
+ 3 2022-01-05
+ 4 2022-01-06
+ 5 2022-01-07
+ 1 2022-01-10
+ 2 2022-01-11
+ 3 2022-01-12
+ 4 2022-01-13
+ 5 2022-01-14
+
+-- Return weekdays for date ranges originating from a LATERAL correlation
+> SELECT weekdays.*
+ FROM VALUES (DATE'2020-01-01'),
+ (DATE'2021-01-01'),
+ (DATE'2022-01-01') AS starts(start),
+ LATERAL weekdays(start, start + INTERVAL '7' DAYS);
+ 3 2020-01-01
+ 4 2020-01-02
+ 5 2020-01-03
+ 1 2020-01-06
+ 2 2020-01-07
+ 3 2020-01-08
+ 5 2021-01-01
+ 1 2021-01-04
+ 2 2021-01-05
+ 3 2021-01-06
+ 4 2021-01-07
+ 5 2021-01-08
+ 1 2022-01-03
+ 2 2022-01-04
+ 3 2022-01-05
+ 4 2022-01-06
+ 5 2022-01-07
+```
+
+### Replace a SQL function
+
+```sql
+-- Replace a SQL scalar function.
+> CREATE OR REPLACE FUNCTION square(x DOUBLE) RETURNS DOUBLE RETURN x * x;
+
+-- Replace a SQL table function.
+> CREATE OR REPLACE FUNCTION getemps(deptno INT)
+ RETURNS TABLE (name STRING)
+ RETURN SELECT name FROM employee e WHERE e.deptno = getemps.deptno;
+
+-- Describe a SQL table function.
+> DESCRIBE FUNCTION getemps;
+ Function: default.getemps
+ Type: TABLE
+ Input: deptno INT
+ Returns: id INT
+ name STRING
+```
+
+### Describe a SQL function
+
+```sql
+> DESCRIBE FUNCTION hello;
+ Function: hello
+ Type: SCALAR
+ Input: ()
+ Returns: STRING
+
+> DESCRIBE FUNCTION area;
+ Function: default.area
+ Type: SCALAR
+ Input: x DOUBLE
+ y DOUBLE
+ Returns: DOUBLE
+
+> DESCRIBE FUNCTION roll_dice;
+ Function: default.roll_dice
+ Type: SCALAR
+ Input: num_dice INT
+ num_sides INT
+ Returns: INT
+```
+
+### Related Statements
+
+* [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
diff --git a/docs/submitting-applications.md b/docs/submitting-applications.md
index 071fbf5549398..922c0a953d423 100644
--- a/docs/submitting-applications.md
+++ b/docs/submitting-applications.md
@@ -36,7 +36,7 @@ script as shown here while passing your jar.
For Python, you can use the `--py-files` argument of `spark-submit` to add `.py`, `.zip` or `.egg`
files to be distributed with your application. If you depend on multiple Python files we recommend
packaging them into a `.zip` or `.egg`. For third-party Python dependencies,
-see [Python Package Management](api/python/user_guide/python_packaging.html).
+see [Python Package Management](api/python/tutorial/python_packaging.html).
# Launching Applications with spark-submit
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaStratifiedSamplingExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaStratifiedSamplingExample.java
index b06b2cceccfcd..3d50b6917c77c 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaStratifiedSamplingExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaStratifiedSamplingExample.java
@@ -17,7 +17,6 @@
package org.apache.spark.examples.mllib;
-import com.google.common.collect.ImmutableMap;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
@@ -48,7 +47,7 @@ public static void main(String[] args) {
JavaPairRDD data = jsc.parallelizePairs(list);
// specify the exact fraction desired from each key Map
- ImmutableMap fractions = ImmutableMap.of(1, 0.1, 2, 0.6, 3, 0.3);
+ Map fractions = Map.of(1, 0.1, 2, 0.6, 3, 0.3);
// Get an approximate sample from each stratum
JavaPairRDD approxSample = data.sampleByKey(false, fractions);
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
index efbed9e591eca..e7d52ce03e860 100644
--- a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
@@ -17,7 +17,6 @@
package org.apache.spark.examples.sql;
// $example on:schema_merging$
-import com.google.common.collect.Lists;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -534,7 +533,7 @@ private static void runXmlDatasetExample(SparkSession spark) {
"" +
"laglangyueDeveloper28" +
"");
- Dataset otherPeopleDataset = spark.createDataset(Lists.newArrayList(xmlData),
+ Dataset otherPeopleDataset = spark.createDataset(new ArrayList<>(xmlData),
Encoders.STRING());
Dataset otherPeople = spark.read()
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
index 1052f47ea496e..633b1ed5f5edf 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
@@ -18,16 +18,14 @@
package org.apache.spark.examples.streaming;
import java.io.File;
-import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;
-import com.google.common.io.FileWriteMode;
import scala.Tuple2;
-import com.google.common.io.Files;
-
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.*;
@@ -153,8 +151,7 @@ private static JavaStreamingContext createContext(String ip,
System.out.println(output);
System.out.println("Dropped " + droppedWordsCounter.value() + " word(s) totally");
System.out.println("Appending to " + outputFile.getAbsolutePath());
- Files.asCharSink(outputFile, Charset.defaultCharset(), FileWriteMode.APPEND)
- .write(output + "\n");
+ Files.writeString(outputFile.toPath(), output + "\n", StandardOpenOption.APPEND);
});
return ssc;
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
index 1ec6ee4abd327..d4f2c78999d96 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
@@ -19,9 +19,8 @@
package org.apache.spark.examples.streaming
import java.io.File
-import java.nio.charset.Charset
-
-import com.google.common.io.{Files, FileWriteMode}
+import java.nio.file.Files
+import java.nio.file.StandardOpenOption
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.broadcast.Broadcast
@@ -134,8 +133,7 @@ object RecoverableNetworkWordCount {
println(output)
println(s"Dropped ${droppedWordsCounter.value} word(s) totally")
println(s"Appending to ${outputFile.getAbsolutePath}")
- Files.asCharSink(outputFile, Charset.defaultCharset(), FileWriteMode.APPEND)
- .write(output + "\n")
+ Files.writeString(outputFile.toPath, output + "\n", StandardOpenOption.APPEND)
}
ssc
}
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala b/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
index 045149922c8ed..0c17e0b3bc4ac 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/GraphLoader.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.TimeUnit
import org.apache.spark.SparkContext
import org.apache.spark.graphx.impl.{EdgePartitionBuilder, GraphImpl}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.TOTAL_TIME
import org.apache.spark.storage.StorageLevel
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala b/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala
index 823143f9b9abb..5b1e1ca6565b4 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/Pregel.scala
@@ -20,7 +20,7 @@ package org.apache.spark.graphx
import scala.reflect.ClassTag
import org.apache.spark.graphx.util.PeriodicGraphCheckpointer
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.NUM_ITERATIONS
import org.apache.spark.rdd.RDD
import org.apache.spark.rdd.util.PeriodicRDDCheckpointer
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
index 4fe010bfce785..2b3636ac1e333 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/PageRank.scala
@@ -22,7 +22,7 @@ import scala.reflect.ClassTag
import breeze.linalg.{Vector => BV}
import org.apache.spark.graphx._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.NUM_ITERATIONS
import org.apache.spark.ml.linalg.{Vector, Vectors}
diff --git a/hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManager.scala b/hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManager.scala
index 599361009fcc5..e64ba7882b918 100644
--- a/hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManager.scala
+++ b/hadoop-cloud/src/main/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManager.scala
@@ -24,9 +24,9 @@ import scala.util.control.NonFatal
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs._
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
-import org.apache.spark.sql.execution.streaming.AbstractFileContextBasedCheckpointFileManager
-import org.apache.spark.sql.execution.streaming.CheckpointFileManager.CancellableFSDataOutputStream
+import org.apache.spark.internal.{Logging, LogKeys}
+import org.apache.spark.sql.execution.streaming.checkpointing.AbstractFileContextBasedCheckpointFileManager
+import org.apache.spark.sql.execution.streaming.checkpointing.CheckpointFileManager.CancellableFSDataOutputStream
class AbortableStreamBasedCheckpointFileManager(path: Path, hadoopConf: Configuration)
extends AbstractFileContextBasedCheckpointFileManager(path, hadoopConf) with Logging {
diff --git a/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManagerSuite.scala b/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManagerSuite.scala
index 0dbc650fc8c73..0b7e65ff78d98 100644
--- a/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManagerSuite.scala
+++ b/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/AbortableStreamBasedCheckpointFileManagerSuite.scala
@@ -27,8 +27,8 @@ import org.scalatest.BeforeAndAfter
import org.apache.spark.internal.Logging
import org.apache.spark.internal.io.cloud.abortable.AbortableFileSystem
-import org.apache.spark.sql.execution.streaming.CheckpointFileManager
import org.apache.spark.sql.execution.streaming.CheckpointFileManagerTests
+import org.apache.spark.sql.execution.streaming.checkpointing.CheckpointFileManager
class AbortableStreamBasedCheckpointFileManagerSuite
extends CheckpointFileManagerTests with Logging {
diff --git a/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/CommitterBindingSuite.scala b/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/CommitterBindingSuite.scala
index 2a2d25af758ec..ee89adab94fb9 100644
--- a/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/CommitterBindingSuite.scala
+++ b/hadoop-cloud/src/test/scala/org/apache/spark/internal/io/cloud/CommitterBindingSuite.scala
@@ -21,7 +21,6 @@ import java.io.{File, FileInputStream, FileOutputStream, IOException, ObjectInpu
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{Path, StreamCapabilities}
-import org.apache.hadoop.io.IOUtils
import org.apache.hadoop.mapreduce.{Job, JobStatus, MRJobConfig, TaskAttemptContext, TaskAttemptID}
import org.apache.hadoop.mapreduce.lib.output.{BindingPathOutputCommitter, FileOutputFormat}
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
@@ -29,6 +28,7 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
import org.apache.spark.SparkFunSuite
import org.apache.spark.internal.io.{FileCommitProtocol, FileNameSpec}
import org.apache.spark.internal.io.cloud.PathOutputCommitProtocol.{CAPABILITY_DYNAMIC_PARTITIONING, OUTPUTCOMMITTER_FACTORY_SCHEME}
+import org.apache.spark.network.util.JavaUtils
class CommitterBindingSuite extends SparkFunSuite {
@@ -130,7 +130,8 @@ class CommitterBindingSuite extends SparkFunSuite {
assert(committer.destPath === committer2.destPath,
"destPath mismatch on round trip")
} finally {
- IOUtils.closeStreams(out, in)
+ JavaUtils.closeQuietly(out)
+ JavaUtils.closeQuietly(in)
serData.delete()
}
}
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index d4d3faf83e3b0..caa2d2b5854e1 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -49,6 +49,7 @@ abstract class AbstractCommandBuilder {
String master;
String remote;
protected String propertiesFile;
+ protected boolean loadSparkDefaults;
final List appArgs;
final List jars;
final List files;
@@ -362,21 +363,35 @@ Map getEffectiveConfig() throws IOException {
}
/**
- * Loads the configuration file for the application, if it exists. This is either the
- * user-specified properties file, or the spark-defaults.conf file under the Spark configuration
- * directory.
+ * Load the configuration file(s) for the application - from the user-specified properties
+ * file, and/or the spark-defaults.conf file under the Spark configuration directory, if exists.
+ * Configurations from user-specified properties file take precedence over spark-defaults.conf.
*/
private Properties loadPropertiesFile() throws IOException {
Properties props = new Properties();
- File propsFile;
if (propertiesFile != null) {
- propsFile = new File(propertiesFile);
+ File propsFile = new File(propertiesFile);
checkArgument(propsFile.isFile(), "Invalid properties file '%s'.", propertiesFile);
- } else {
- propsFile = new File(getConfDir(), DEFAULT_PROPERTIES_FILE);
+ props = loadPropertiesFile(propsFile);
}
- if (propsFile.isFile()) {
+ Properties defaultsProps = new Properties();
+ if (propertiesFile == null || loadSparkDefaults) {
+ defaultsProps = loadPropertiesFile(new File(getConfDir(), DEFAULT_PROPERTIES_FILE));
+ }
+
+ for (Map.Entry entry : defaultsProps.entrySet()) {
+ if (!props.containsKey(entry.getKey())) {
+ props.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ return props;
+ }
+
+ private Properties loadPropertiesFile(File propsFile) throws IOException {
+ Properties props = new Properties();
+ if (propsFile != null && propsFile.isFile()) {
try (InputStreamReader isr = new InputStreamReader(
new FileInputStream(propsFile), StandardCharsets.UTF_8)) {
props.load(isr);
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java
index 8d95bc06d7a7d..d7d10d486e3f5 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java
@@ -99,6 +99,7 @@ public List buildCommand(Map env)
String mem = firstNonEmpty(memKey != null ? System.getenv(memKey) : null, DEFAULT_MEM);
cmd.add("-Xmx" + mem);
+ addOptionString(cmd, JavaModuleOptions.defaultModuleOptions());
cmd.add(className);
cmd.addAll(classArgs);
return cmd;
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java
index 67f76a73c214d..c3cc470e8871d 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java
@@ -59,7 +59,7 @@ public class SparkLauncher extends AbstractLauncher {
/** Configuration key for the driver default extra class path. */
public static final String DRIVER_DEFAULT_EXTRA_CLASS_PATH =
"spark.driver.defaultExtraClassPath";
- public static final String DRIVER_DEFAULT_EXTRA_CLASS_PATH_VALUE = "hive-jackson/*";
+ public static final String DRIVER_DEFAULT_EXTRA_CLASS_PATH_VALUE = "";
/** Configuration key for the driver class path. */
public static final String DRIVER_EXTRA_CLASSPATH = "spark.driver.extraClassPath";
/** Configuration key for the default driver VM options. */
@@ -74,7 +74,7 @@ public class SparkLauncher extends AbstractLauncher {
/** Configuration key for the executor default extra class path. */
public static final String EXECUTOR_DEFAULT_EXTRA_CLASS_PATH =
"spark.executor.defaultExtraClassPath";
- public static final String EXECUTOR_DEFAULT_EXTRA_CLASS_PATH_VALUE = "hive-jackson/*";
+ public static final String EXECUTOR_DEFAULT_EXTRA_CLASS_PATH_VALUE = "";
/** Configuration key for the executor class path. */
public static final String EXECUTOR_EXTRA_CLASSPATH = "spark.executor.extraClassPath";
/** Configuration key for the default executor VM options. */
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
index 5efa3bef78bcc..7b9f90ac7b7a6 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
@@ -172,6 +172,16 @@ private static String getApiMode(Map conf) {
@Override
public List buildCommand(Map env)
throws IOException, IllegalArgumentException {
+ for (Map.Entry entry : getEffectiveConfig().entrySet()) {
+ // If both spark.remote and spark.master are set, the error will be thrown later
+ // when the application is started.
+ if (entry.getKey().equals("spark.remote")) {
+ isRemote = true;
+ } else if (entry.getKey().equals(SparkLauncher.SPARK_API_MODE)) {
+ // Respects if the API mode is explicitly set.
+ isRemote = entry.getValue().equalsIgnoreCase("connect");
+ }
+ }
if (PYSPARK_SHELL.equals(appResource) && !isSpecialCommand) {
return buildPySparkShellCommand(env);
} else if (SPARKR_SHELL.equals(appResource) && !isSpecialCommand) {
@@ -241,6 +251,10 @@ List buildSparkSubmitArgs(boolean includeRemote) {
args.add(propertiesFile);
}
+ if (loadSparkDefaults) {
+ args.add(parser.LOAD_SPARK_DEFAULTS);
+ }
+
if (isExample) {
jars.addAll(findExamplesJars());
}
@@ -540,6 +554,7 @@ protected boolean handle(String opt, String value) {
}
case DEPLOY_MODE -> deployMode = value;
case PROPERTIES_FILE -> propertiesFile = value;
+ case LOAD_SPARK_DEFAULTS -> loadSparkDefaults = true;
case DRIVER_MEMORY -> conf.put(SparkLauncher.DRIVER_MEMORY, value);
case DRIVER_JAVA_OPTIONS -> conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, value);
case DRIVER_LIBRARY_PATH -> conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, value);
@@ -550,14 +565,6 @@ protected boolean handle(String opt, String value) {
checkArgument(value != null, "Missing argument to %s", CONF);
String[] setConf = value.split("=", 2);
checkArgument(setConf.length == 2, "Invalid argument to %s: %s", CONF, value);
- // If both spark.remote and spark.mater are set, the error will be thrown later when
- // the application is started.
- if (setConf[0].equals("spark.remote")) {
- isRemote = true;
- } else if (setConf[0].equals(SparkLauncher.SPARK_API_MODE)) {
- // Respects if the API mode is explicitly set.
- isRemote = setConf[1].equalsIgnoreCase("connect");
- }
conf.put(setConf[0], setConf[1]);
}
case CLASS -> {
diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkClassCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkClassCommandBuilderSuite.java
index 76a5e23d4693a..23ebbfc098aca 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/SparkClassCommandBuilderSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/SparkClassCommandBuilderSuite.java
@@ -33,6 +33,7 @@ public void testBeelineBuilder() throws Exception {
SparkClassCommandBuilder builder =
new SparkClassCommandBuilder("org.apache.hive.beeline.BeeLine", args);
List strings = builder.buildCommand(new HashMap<>());
+ assertTrue(strings.containsAll(Arrays.asList(JavaModuleOptions.defaultModuleOptionArray())));
assertTrue(strings.contains("-DmyKey=yourValue"));
assertTrue(strings.contains("myBeelineArg"));
}
diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
index 31318f1318ca0..d1dba85a534f2 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
@@ -18,12 +18,16 @@
package org.apache.spark.launcher;
import java.io.File;
+import java.io.FileReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.regex.Pattern;
import org.junit.jupiter.api.*;
@@ -33,29 +37,106 @@
public class SparkSubmitCommandBuilderSuite extends BaseSuite {
private static File dummyPropsFile;
+ private static File connectPropsFile;
+ private static File driverMemPropsFile;
private static SparkSubmitOptionParser parser;
@BeforeAll
public static void setUp() throws Exception {
dummyPropsFile = File.createTempFile("spark", "properties");
+ connectPropsFile = File.createTempFile("spark", "properties");
+ Files.writeString(connectPropsFile.toPath(), "spark.remote=sc://connect-server:15002");
+ driverMemPropsFile = File.createTempFile("spark", "properties");
+ Files.writeString(driverMemPropsFile.toPath(),
+ "spark.driver.memory=4g\nspark.driver.memoryOverhead=768m");
parser = new SparkSubmitOptionParser();
}
@AfterAll
public static void cleanUp() throws Exception {
dummyPropsFile.delete();
+ connectPropsFile.delete();
+ driverMemPropsFile.delete();
+ }
+
+ @Test
+ public void testGetEffectiveConfig() throws Exception {
+ doTestGetEffectiveConfig(null, true, true);
+ doTestGetEffectiveConfig(null, true, false);
+ doTestGetEffectiveConfig(null, false, true);
+ doTestGetEffectiveConfig(null, false, false);
+ doTestGetEffectiveConfig(driverMemPropsFile, true, true);
+ doTestGetEffectiveConfig(driverMemPropsFile, true, false);
+ doTestGetEffectiveConfig(driverMemPropsFile, false, true);
+ doTestGetEffectiveConfig(driverMemPropsFile, false, false);
+ }
+
+ private void doTestGetEffectiveConfig(
+ File propertiesFile, boolean loadSparkDefaults, boolean confDriverMemory) throws Exception {
+ SparkSubmitCommandBuilder launcher =
+ newCommandBuilder(Collections.emptyList());
+ launcher.loadSparkDefaults = loadSparkDefaults;
+ launcher.conf.put("spark.foo", "bar");
+ launcher.childEnv.put("SPARK_CONF_DIR", System.getProperty("spark.test.home")
+ + "/launcher/src/test/resources");
+
+ if (propertiesFile != null) {
+ launcher.setPropertiesFile(propertiesFile.getAbsolutePath());
+ }
+
+ if (confDriverMemory) {
+ launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "2g");
+ }
+
+ Map effectiveConfig = launcher.getEffectiveConfig();
+
+ assertEquals("bar", effectiveConfig.get("spark.foo"));
+ if (confDriverMemory) {
+ assertEquals("2g", effectiveConfig.get(SparkLauncher.DRIVER_MEMORY));
+ } else if (propertiesFile != null) {
+ try (FileReader reader = new FileReader(propertiesFile, StandardCharsets.UTF_8)) {
+ Properties props = new Properties();
+ props.load(reader);
+ if (props.containsKey(SparkLauncher.DRIVER_MEMORY)) {
+ assertEquals(props.getProperty(SparkLauncher.DRIVER_MEMORY),
+ effectiveConfig.get(SparkLauncher.DRIVER_MEMORY));
+ }
+ }
+ } else {
+ assertEquals("1g", effectiveConfig.get(SparkLauncher.DRIVER_MEMORY));
+ }
+
+ if (propertiesFile != null) {
+ try (FileReader reader = new FileReader(propertiesFile, StandardCharsets.UTF_8)) {
+ Properties props = new Properties();
+ props.load(reader);
+ if (props.containsKey("spark.driver.memoryOverhead")) {
+ assertEquals(props.getProperty("spark.driver.memoryOverhead"),
+ effectiveConfig.get("spark.driver.memoryOverhead"));
+ }
+ }
+ if (loadSparkDefaults) {
+ assertEquals("/driver", effectiveConfig.get(SparkLauncher.DRIVER_EXTRA_CLASSPATH));
+ } else {
+ assertFalse(effectiveConfig.containsKey(SparkLauncher.DRIVER_EXTRA_CLASSPATH));
+ }
+ } else {
+ assertEquals("/driver", effectiveConfig.get(SparkLauncher.DRIVER_EXTRA_CLASSPATH));
+ }
}
@Test
public void testDriverCmdBuilder() throws Exception {
- testCmdBuilder(true, true);
- testCmdBuilder(true, false);
+ testCmdBuilder(true, null);
+ testCmdBuilder(true, dummyPropsFile);
+ testCmdBuilder(true, connectPropsFile);
}
@Test
public void testClusterCmdBuilder() throws Exception {
- testCmdBuilder(false, true);
- testCmdBuilder(false, false);
+ testCmdBuilder(false, null);
+ testCmdBuilder(false, dummyPropsFile);
+ testCmdBuilder(false, connectPropsFile);
}
@Test
@@ -307,7 +388,7 @@ public void testIsClientMode() {
assertTrue(builder.isClientMode(userProps));
}
- private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) throws Exception {
+ private void testCmdBuilder(boolean isDriver, File propertiesFile) throws Exception {
final String DRIVER_DEFAULT_PARAM = "-Ddriver-default";
final String DRIVER_EXTRA_PARAM = "-Ddriver-extra";
String deployMode = isDriver ? "client" : "cluster";
@@ -325,16 +406,16 @@ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) th
launcher.appArgs.add("bar");
launcher.conf.put("spark.foo", "foo");
// either set the property through "--conf" or through default property file
- if (!useDefaultPropertyFile) {
- launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath());
+ if (propertiesFile == null) {
+ launcher.childEnv.put("SPARK_CONF_DIR", System.getProperty("spark.test.home")
+ + "/launcher/src/test/resources");
+ } else {
+ launcher.setPropertiesFile(propertiesFile.getAbsolutePath());
launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g");
launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, "/driver");
launcher.conf.put(SparkLauncher.DRIVER_DEFAULT_JAVA_OPTIONS, DRIVER_DEFAULT_PARAM);
launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, DRIVER_EXTRA_PARAM);
launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native");
- } else {
- launcher.childEnv.put("SPARK_CONF_DIR", System.getProperty("spark.test.home")
- + "/launcher/src/test/resources");
}
Map env = new HashMap<>();
@@ -348,13 +429,7 @@ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) th
"Driver default options should be configured.");
assertTrue(cmd.contains(DRIVER_EXTRA_PARAM), "Driver extra options should be configured.");
} else {
- boolean found = false;
- for (String arg : cmd) {
- if (arg.startsWith("-Xmx")) {
- found = true;
- break;
- }
- }
+ boolean found = cmd.stream().anyMatch(arg -> arg.startsWith("-Xmx"));
assertFalse(found, "Memory arguments should not be set.");
assertFalse(cmd.contains(DRIVER_DEFAULT_PARAM),
"Driver default options should not be configured.");
@@ -379,8 +454,15 @@ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) th
}
// Checks below are the same for both driver and non-driver mode.
- if (!useDefaultPropertyFile) {
- assertEquals(dummyPropsFile.getAbsolutePath(), findArgValue(cmd, parser.PROPERTIES_FILE));
+ if (propertiesFile != null) {
+ assertEquals(propertiesFile.getAbsolutePath(), findArgValue(cmd, parser.PROPERTIES_FILE));
+ try (FileReader reader = new FileReader(propertiesFile, StandardCharsets.UTF_8)) {
+ Properties props = new Properties();
+ props.load(reader);
+ if (props.containsKey("spark.remote")) {
+ assertTrue(launcher.isRemote);
+ }
+ }
}
assertEquals("yarn", findArgValue(cmd, parser.MASTER));
assertEquals(deployMode, findArgValue(cmd, parser.DEPLOY_MODE));
diff --git a/licenses-binary/LICENSE-jodd.txt b/licenses-binary/LICENSE-jodd.txt
deleted file mode 100644
index cc6b458adb386..0000000000000
--- a/licenses-binary/LICENSE-jodd.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-Copyright (c) 2003-present, Jodd Team (https://jodd.org)
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice,
-this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/mllib-local/benchmarks/BLASBenchmark-jdk21-results.txt b/mllib-local/benchmarks/BLASBenchmark-jdk21-results.txt
index dac9751d9afc4..7eba64d4ec8fd 100644
--- a/mllib-local/benchmarks/BLASBenchmark-jdk21-results.txt
+++ b/mllib-local/benchmarks/BLASBenchmark-jdk21-results.txt
@@ -2,311 +2,311 @@
daxpy
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
daxpy: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 151 178 26 661.0 1.5 1.0X
-java 142 174 24 706.6 1.4 1.1X
+f2j 142 152 5 702.9 1.4 1.0X
+java 142 146 4 704.9 1.4 1.0X
================================================================================================
saxpy
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
saxpy: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 77 94 14 1298.8 0.8 1.0X
-java 71 84 13 1410.4 0.7 1.1X
+f2j 75 83 6 1325.6 0.8 1.0X
+java 69 71 1 1459.8 0.7 1.1X
================================================================================================
dcopy
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dcopy: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 142 178 25 702.4 1.4 1.0X
-java 150 183 32 664.8 1.5 0.9X
+f2j 129 148 8 776.0 1.3 1.0X
+java 127 144 12 788.7 1.3 1.0X
================================================================================================
scopy
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
scopy: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 72 91 14 1394.3 0.7 1.0X
-java 72 88 19 1387.3 0.7 1.0X
+f2j 69 78 6 1444.8 0.7 1.0X
+java 69 72 2 1449.4 0.7 1.0X
================================================================================================
ddot
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
ddot: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 95 100 7 1050.9 1.0 1.0X
-java 46 57 9 2171.0 0.5 2.1X
+f2j 96 96 0 1041.9 1.0 1.0X
+java 47 49 1 2121.9 0.5 2.0X
================================================================================================
sdot
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sdot: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 94 94 1 1068.5 0.9 1.0X
-java 23 29 5 4399.8 0.2 4.1X
+f2j 92 94 0 1090.5 0.9 1.0X
+java 24 25 1 4188.1 0.2 3.8X
================================================================================================
dnrm2
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dnrm2: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 143 144 2 698.6 1.4 1.0X
-java 33 38 7 3012.7 0.3 4.3X
+f2j 140 144 1 713.5 1.4 1.0X
+java 32 33 1 3151.0 0.3 4.4X
================================================================================================
snrm2
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
snrm2: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 125 125 2 802.5 1.2 1.0X
-java 17 20 4 6044.7 0.2 7.5X
+f2j 121 125 1 824.1 1.2 1.0X
+java 16 16 0 6405.3 0.2 7.8X
================================================================================================
dscal
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dscal: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 136 160 24 736.1 1.4 1.0X
-java 122 145 21 821.1 1.2 1.1X
+f2j 133 136 1 753.4 1.3 1.0X
+java 121 124 1 828.0 1.2 1.1X
================================================================================================
sscal
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sscal: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 80 97 14 1254.5 0.8 1.0X
-java 59 75 12 1707.3 0.6 1.4X
+f2j 77 84 8 1300.9 0.8 1.0X
+java 55 63 6 1808.1 0.6 1.4X
================================================================================================
dgemv[N]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemv[N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 100 102 4 1002.2 1.0 1.0X
-java 23 29 5 4265.7 0.2 4.3X
+f2j 100 100 0 1003.6 1.0 1.0X
+java 22 23 0 4444.7 0.2 4.4X
================================================================================================
dgemv[T]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemv[T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 94 96 3 1064.1 0.9 1.0X
-java 23 29 6 4285.4 0.2 4.0X
+f2j 94 94 1 1067.3 0.9 1.0X
+java 22 23 0 4448.5 0.2 4.2X
================================================================================================
sgemv[N]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemv[N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 96 96 0 1043.1 1.0 1.0X
-java 12 15 2 8498.9 0.1 8.1X
+f2j 96 96 0 1042.9 1.0 1.0X
+java 11 11 0 9050.4 0.1 8.7X
================================================================================================
sgemv[T]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemv[T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 93 94 0 1071.3 0.9 1.0X
-java 12 15 2 8475.7 0.1 7.9X
+f2j 93 93 0 1071.7 0.9 1.0X
+java 11 12 0 9015.6 0.1 8.4X
================================================================================================
dger
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dger: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 145 169 21 691.0 1.4 1.0X
-java 122 141 19 817.2 1.2 1.2X
+f2j 140 145 2 712.8 1.4 1.0X
+java 125 128 2 800.7 1.2 1.1X
================================================================================================
dspmv[U]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dspmv[U]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 92 93 0 542.4 1.8 1.0X
-java 12 15 2 4102.6 0.2 7.6X
+f2j 92 92 0 543.0 1.8 1.0X
+java 11 12 0 4393.4 0.2 8.1X
================================================================================================
dspr[U]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dspr[U]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 96 106 10 520.2 1.9 1.0X
-java 90 106 12 558.6 1.8 1.1X
+f2j 93 97 1 537.4 1.9 1.0X
+java 86 94 8 578.1 1.7 1.1X
================================================================================================
dsyr[U]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dsyr[U]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 140 161 17 356.5 2.8 1.0X
-java 143 160 17 349.1 2.9 1.0X
+f2j 136 139 2 368.8 2.7 1.0X
+java 136 139 2 367.1 2.7 1.0X
================================================================================================
dgemm[N,N]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemm[N,N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 671 673 1 1490.1 0.7 1.0X
-java 63 70 4 15790.7 0.1 10.6X
+f2j 664 665 1 1505.2 0.7 1.0X
+java 64 70 4 15702.2 0.1 10.4X
================================================================================================
dgemm[N,T]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemm[N,T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 673 675 1 1484.9 0.7 1.0X
-java 64 70 4 15709.3 0.1 10.6X
+f2j 674 676 1 1482.6 0.7 1.0X
+java 64 70 4 15614.9 0.1 10.5X
================================================================================================
dgemm[T,N]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemm[T,N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 906 908 1 1103.5 0.9 1.0X
-java 63 69 4 15850.6 0.1 14.4X
+f2j 908 909 0 1101.1 0.9 1.0X
+java 64 69 4 15734.9 0.1 14.3X
================================================================================================
dgemm[T,T]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemm[T,T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 950 952 1 1052.5 1.0 1.0X
-java 63 70 4 15793.7 0.1 15.0X
+f2j 951 953 3 1051.2 1.0 1.0X
+java 63 69 4 15754.9 0.1 15.0X
================================================================================================
sgemm[N,N]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemm[N,N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 653 655 1 1530.9 0.7 1.0X
-java 40 41 1 25152.3 0.0 16.4X
+f2j 653 654 1 1531.6 0.7 1.0X
+java 40 41 1 24917.5 0.0 16.3X
================================================================================================
sgemm[N,T]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemm[N,T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 655 656 1 1527.4 0.7 1.0X
-java 40 41 1 24811.8 0.0 16.2X
+f2j 656 657 1 1524.6 0.7 1.0X
+java 41 41 1 24687.2 0.0 16.2X
================================================================================================
sgemm[T,N]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemm[T,N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 906 907 1 1104.1 0.9 1.0X
-java 39 40 1 25372.9 0.0 23.0X
+f2j 907 907 1 1102.7 0.9 1.0X
+java 40 40 1 25305.9 0.0 22.9X
================================================================================================
sgemm[T,T]
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemm[T,T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 946 948 1 1057.3 0.9 1.0X
-java 40 41 1 25019.0 0.0 23.7X
+f2j 947 948 2 1056.1 0.9 1.0X
+java 40 41 1 24894.4 0.0 23.6X
diff --git a/mllib-local/benchmarks/BLASBenchmark-results.txt b/mllib-local/benchmarks/BLASBenchmark-results.txt
index 16dfba14e3168..794c61b2567f4 100644
--- a/mllib-local/benchmarks/BLASBenchmark-results.txt
+++ b/mllib-local/benchmarks/BLASBenchmark-results.txt
@@ -2,311 +2,311 @@
daxpy
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
daxpy: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 147 151 2 681.6 1.5 1.0X
-java 142 144 1 705.3 1.4 1.0X
+f2j 144 160 9 693.6 1.4 1.0X
+java 137 148 9 728.3 1.4 1.1X
================================================================================================
saxpy
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
saxpy: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 85 86 1 1176.1 0.9 1.0X
-java 71 73 1 1401.3 0.7 1.2X
+f2j 81 86 4 1227.9 0.8 1.0X
+java 70 76 4 1433.1 0.7 1.2X
================================================================================================
dcopy
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dcopy: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 143 146 2 700.2 1.4 1.0X
-java 142 145 2 705.7 1.4 1.0X
+f2j 140 152 7 713.2 1.4 1.0X
+java 138 152 10 726.9 1.4 1.0X
================================================================================================
scopy
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
scopy: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 82 83 1 1219.9 0.8 1.0X
-java 72 74 1 1395.8 0.7 1.1X
+f2j 78 83 4 1279.4 0.8 1.0X
+java 69 80 12 1443.0 0.7 1.1X
================================================================================================
ddot
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
ddot: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 96 97 0 1037.6 1.0 1.0X
-java 46 48 1 2158.4 0.5 2.1X
+f2j 96 96 1 1045.6 1.0 1.0X
+java 47 50 3 2143.6 0.5 2.1X
================================================================================================
sdot
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sdot: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 94 94 0 1068.1 0.9 1.0X
-java 23 23 0 4433.6 0.2 4.2X
+f2j 94 94 0 1067.6 0.9 1.0X
+java 23 25 2 4360.3 0.2 4.1X
================================================================================================
dnrm2
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dnrm2: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 143 144 1 696.9 1.4 1.0X
-java 32 32 0 3105.3 0.3 4.5X
+f2j 144 144 0 696.5 1.4 1.0X
+java 33 34 2 3071.3 0.3 4.4X
================================================================================================
snrm2
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
snrm2: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 121 121 0 829.1 1.2 1.0X
-java 16 16 0 6203.2 0.2 7.5X
+f2j 121 121 0 828.5 1.2 1.0X
+java 16 17 1 6148.4 0.2 7.4X
================================================================================================
dscal
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dscal: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 140 144 2 713.0 1.4 1.0X
-java 129 131 2 777.2 1.3 1.1X
+f2j 131 145 8 760.5 1.3 1.0X
+java 122 130 7 821.8 1.2 1.1X
================================================================================================
sscal
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sscal: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 79 81 1 1258.7 0.8 1.0X
-java 65 67 1 1548.0 0.6 1.2X
+f2j 75 79 4 1337.5 0.7 1.0X
+java 61 66 4 1645.9 0.6 1.2X
================================================================================================
dgemv[N]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemv[N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 96 96 0 1038.5 1.0 1.0X
-java 22 23 0 4459.6 0.2 4.3X
+f2j 96 97 1 1036.5 1.0 1.0X
+java 23 26 2 4283.2 0.2 4.1X
================================================================================================
dgemv[T]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemv[T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 94 95 0 1060.3 0.9 1.0X
-java 23 24 1 4422.6 0.2 4.2X
+f2j 95 95 1 1057.4 0.9 1.0X
+java 23 25 2 4311.6 0.2 4.1X
================================================================================================
sgemv[N]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemv[N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 96 96 0 1044.2 1.0 1.0X
-java 11 11 0 8998.8 0.1 8.6X
+f2j 96 96 0 1043.9 1.0 1.0X
+java 12 13 1 8673.8 0.1 8.3X
================================================================================================
sgemv[T]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemv[T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 93 94 0 1070.6 0.9 1.0X
-java 11 12 0 8924.2 0.1 8.3X
+f2j 93 94 0 1070.8 0.9 1.0X
+java 12 13 1 8342.2 0.1 7.8X
================================================================================================
dger
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dger: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 146 149 2 684.3 1.5 1.0X
-java 127 131 2 787.4 1.3 1.2X
+f2j 137 157 24 732.5 1.4 1.0X
+java 119 129 7 843.7 1.2 1.2X
================================================================================================
dspmv[U]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dspmv[U]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 100 100 0 501.3 2.0 1.0X
-java 11 12 0 4366.1 0.2 8.7X
+f2j 100 100 0 500.1 2.0 1.0X
+java 12 14 2 4169.4 0.2 8.3X
================================================================================================
dspr[U]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dspr[U]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 95 97 1 524.0 1.9 1.0X
-java 95 97 1 525.3 1.9 1.0X
+f2j 92 96 3 545.3 1.8 1.0X
+java 92 96 3 546.1 1.8 1.0X
================================================================================================
dsyr[U]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dsyr[U]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 143 145 1 350.1 2.9 1.0X
-java 143 144 1 350.1 2.9 1.0X
+f2j 140 155 16 357.9 2.8 1.0X
+java 138 147 6 361.9 2.8 1.0X
================================================================================================
dgemm[N,N]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemm[N,N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 668 670 1 1496.5 0.7 1.0X
-java 64 72 4 15627.5 0.1 10.4X
+f2j 669 670 1 1495.7 0.7 1.0X
+java 77 78 2 13051.7 0.1 8.7X
================================================================================================
dgemm[N,T]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemm[N,T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 676 678 1 1478.6 0.7 1.0X
-java 65 72 4 15494.6 0.1 10.5X
+f2j 676 678 1 1478.9 0.7 1.0X
+java 73 74 1 13704.1 0.1 9.3X
================================================================================================
dgemm[T,N]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemm[T,N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 926 928 1 1079.9 0.9 1.0X
-java 64 71 4 15716.6 0.1 14.6X
+f2j 926 929 2 1079.5 0.9 1.0X
+java 76 78 2 13187.8 0.1 12.2X
================================================================================================
dgemm[T,T]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
dgemm[T,T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 960 962 1 1042.0 1.0 1.0X
-java 64 72 4 15644.9 0.1 15.0X
+f2j 960 962 3 1042.2 1.0 1.0X
+java 77 78 2 13051.9 0.1 12.5X
================================================================================================
sgemm[N,N]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemm[N,N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 652 653 1 1533.2 0.7 1.0X
-java 41 41 1 24534.0 0.0 16.0X
+f2j 652 656 3 1533.4 0.7 1.0X
+java 41 42 1 24439.5 0.0 15.9X
================================================================================================
sgemm[N,T]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemm[N,T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 656 658 1 1524.1 0.7 1.0X
-java 41 42 1 24202.1 0.0 15.9X
+f2j 655 657 1 1525.7 0.7 1.0X
+java 41 43 1 24129.1 0.0 15.8X
================================================================================================
sgemm[T,N]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemm[T,N]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 906 907 0 1104.1 0.9 1.0X
-java 40 41 1 24801.4 0.0 22.5X
+f2j 906 907 1 1103.9 0.9 1.0X
+java 40 42 1 24767.2 0.0 22.4X
================================================================================================
sgemm[T,T]
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
sgemm[T,T]: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-f2j 956 958 2 1046.3 1.0 1.0X
-java 41 42 1 24445.7 0.0 23.4X
+f2j 955 956 1 1047.4 1.0 1.0X
+java 41 42 1 24444.2 0.0 23.3X
diff --git a/mllib-local/src/main/scala/scala/collection/compat/package.scala b/mllib-local/src/main/scala/scala/collection/compat/package.scala
new file mode 100644
index 0000000000000..8ce1cfff886f6
--- /dev/null
+++ b/mllib-local/src/main/scala/scala/collection/compat/package.scala
@@ -0,0 +1,26 @@
+// scalastyle:off license
+/*
+ * Scala (https://www.scala-lang.org)
+ *
+ * Copyright EPFL and Lightbend, Inc. dba Akka
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+// scalastyle:on license
+
+package scala.collection
+
+package object compat {
+ type Factory[-A, +C] = scala.collection.Factory[A, C]
+ val Factory = scala.collection.Factory
+
+ type BuildFrom[-From, -A, +C] = scala.collection.BuildFrom[From, A, C]
+ val BuildFrom = scala.collection.BuildFrom
+
+ type IterableOnce[+X] = scala.collection.IterableOnce[X]
+ val IterableOnce = scala.collection.IterableOnce
+}
diff --git a/mllib/benchmarks/UDTSerializationBenchmark-jdk21-results.txt b/mllib/benchmarks/UDTSerializationBenchmark-jdk21-results.txt
index 1d84715344887..e7bc62b022135 100644
--- a/mllib/benchmarks/UDTSerializationBenchmark-jdk21-results.txt
+++ b/mllib/benchmarks/UDTSerializationBenchmark-jdk21-results.txt
@@ -2,11 +2,11 @@
VectorUDT de/serialization
================================================================================================
-OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
VectorUDT de/serialization: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-serialize 90 94 4 0.0 89629.8 1.0X
-deserialize 67 70 2 0.0 66956.8 1.3X
+serialize 101 105 4 0.0 100568.0 1.0X
+deserialize 72 77 3 0.0 72050.7 1.4X
diff --git a/mllib/benchmarks/UDTSerializationBenchmark-results.txt b/mllib/benchmarks/UDTSerializationBenchmark-results.txt
index 84d023b405cf1..af2e5f972bcf1 100644
--- a/mllib/benchmarks/UDTSerializationBenchmark-results.txt
+++ b/mllib/benchmarks/UDTSerializationBenchmark-results.txt
@@ -2,11 +2,11 @@
VectorUDT de/serialization
================================================================================================
-OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1020-azure
+OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure
AMD EPYC 7763 64-Core Processor
VectorUDT de/serialization: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-serialize 93 95 2 0.0 92851.3 1.0X
-deserialize 71 74 2 0.0 71022.3 1.3X
+serialize 99 104 3 0.0 98606.8 1.0X
+deserialize 71 76 2 0.0 71492.9 1.4X
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
index 83b77510602b2..17d6be0ce7cd5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{LogKeys, MDC}
+import org.apache.spark.internal.{LogKeys}
import org.apache.spark.ml.linalg.VectorUDT
import org.apache.spark.ml.param._
import org.apache.spark.ml.param.shared._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
index 7883a0dea54f1..d9238479e8031 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.classification
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{LogKeys, MDC}
+import org.apache.spark.internal.{LogKeys}
import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams}
import org.apache.spark.ml.linalg.{Vector, VectorUDT}
import org.apache.spark.ml.param.ParamMap
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
index cefa13b2bbe71..b653383161e74 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
@@ -224,17 +224,8 @@ class FMClassifier @Since("3.0.0") (
factors: Matrix,
objectiveHistory: Array[Double]): FMClassificationModel = {
val model = copyValues(new FMClassificationModel(uid, intercept, linear, factors))
- val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
-
- val (summaryModel, probabilityColName, predictionColName) = model.findSummaryModel()
- val summary = new FMClassificationTrainingSummaryImpl(
- summaryModel.transform(dataset),
- probabilityColName,
- predictionColName,
- $(labelCol),
- weightColName,
- objectiveHistory)
- model.setSummary(Some(summary))
+ model.createSummary(dataset, objectiveHistory)
+ model
}
@Since("3.0.0")
@@ -343,6 +334,42 @@ class FMClassificationModel private[classification] (
s"uid=${super.toString}, numClasses=$numClasses, numFeatures=$numFeatures, " +
s"factorSize=${$(factorSize)}, fitLinear=${$(fitLinear)}, fitIntercept=${$(fitIntercept)}"
}
+
+ private[spark] def createSummary(
+ dataset: Dataset[_], objectiveHistory: Array[Double]
+ ): Unit = {
+ val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+
+ val (summaryModel, probabilityColName, predictionColName) = findSummaryModel()
+ val summary = new FMClassificationTrainingSummaryImpl(
+ summaryModel.transform(dataset),
+ probabilityColName,
+ predictionColName,
+ $(labelCol),
+ weightColName,
+ objectiveHistory)
+ setSummary(Some(summary))
+ }
+
+ override private[spark] def saveSummary(path: String): Unit = {
+ ReadWriteUtils.saveObjectToLocal[Tuple1[Array[Double]]](
+ path, Tuple1(summary.objectiveHistory),
+ (data, dos) => {
+ ReadWriteUtils.serializeDoubleArray(data._1, dos)
+ }
+ )
+ }
+
+ override private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ val Tuple1(objectiveHistory: Array[Double])
+ = ReadWriteUtils.loadObjectFromLocal[Tuple1[Array[Double]]](
+ path,
+ dis => {
+ Tuple1(ReadWriteUtils.deserializeDoubleArray(dis))
+ }
+ )
+ createSummary(dataset, objectiveHistory)
+ }
}
@Since("3.0.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
index a50346ae88f4c..89f1f7792c075 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{COUNT, RANGE}
import org.apache.spark.ml.feature._
import org.apache.spark.ml.linalg._
@@ -277,17 +277,8 @@ class LinearSVC @Since("2.2.0") (
intercept: Double,
objectiveHistory: Array[Double]): LinearSVCModel = {
val model = copyValues(new LinearSVCModel(uid, coefficients, intercept))
- val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
-
- val (summaryModel, rawPredictionColName, predictionColName) = model.findSummaryModel()
- val summary = new LinearSVCTrainingSummaryImpl(
- summaryModel.transform(dataset),
- rawPredictionColName,
- predictionColName,
- $(labelCol),
- weightColName,
- objectiveHistory)
- model.setSummary(Some(summary))
+ model.createSummary(dataset, objectiveHistory)
+ model
}
private def trainImpl(
@@ -445,6 +436,42 @@ class LinearSVCModel private[classification] (
override def toString: String = {
s"LinearSVCModel: uid=$uid, numClasses=$numClasses, numFeatures=$numFeatures"
}
+
+ private[spark] def createSummary(
+ dataset: Dataset[_], objectiveHistory: Array[Double]
+ ): Unit = {
+ val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+
+ val (summaryModel, rawPredictionColName, predictionColName) = findSummaryModel()
+ val summary = new LinearSVCTrainingSummaryImpl(
+ summaryModel.transform(dataset),
+ rawPredictionColName,
+ predictionColName,
+ $(labelCol),
+ weightColName,
+ objectiveHistory)
+ setSummary(Some(summary))
+ }
+
+ override private[spark] def saveSummary(path: String): Unit = {
+ ReadWriteUtils.saveObjectToLocal[Tuple1[Array[Double]]](
+ path, Tuple1(summary.objectiveHistory),
+ (data, dos) => {
+ ReadWriteUtils.serializeDoubleArray(data._1, dos)
+ }
+ )
+ }
+
+ override private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ val Tuple1(objectiveHistory: Array[Double])
+ = ReadWriteUtils.loadObjectFromLocal[Tuple1[Array[Double]]](
+ path,
+ dis => {
+ Tuple1(ReadWriteUtils.deserializeDoubleArray(dis))
+ }
+ )
+ createSummary(dataset, objectiveHistory)
+ }
}
@Since("2.2.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 58a2652d0eab9..d0824925780c0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.LogKeys.{COUNT, RANGE}
import org.apache.spark.ml.feature._
import org.apache.spark.ml.impl.Utils
@@ -718,29 +718,8 @@ class LogisticRegression @Since("1.2.0") (
objectiveHistory: Array[Double]): LogisticRegressionModel = {
val model = copyValues(new LogisticRegressionModel(uid, coefficientMatrix, interceptVector,
numClasses, checkMultinomial(numClasses)))
- val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
-
- val (summaryModel, probabilityColName, predictionColName) = model.findSummaryModel()
- val logRegSummary = if (numClasses <= 2) {
- new BinaryLogisticRegressionTrainingSummaryImpl(
- summaryModel.transform(dataset),
- probabilityColName,
- predictionColName,
- $(labelCol),
- $(featuresCol),
- weightColName,
- objectiveHistory)
- } else {
- new LogisticRegressionTrainingSummaryImpl(
- summaryModel.transform(dataset),
- probabilityColName,
- predictionColName,
- $(labelCol),
- $(featuresCol),
- weightColName,
- objectiveHistory)
- }
- model.setSummary(Some(logRegSummary))
+ model.createSummary(dataset, objectiveHistory)
+ model
}
private def createBounds(
@@ -1323,6 +1302,54 @@ class LogisticRegressionModel private[spark] (
override def toString: String = {
s"LogisticRegressionModel: uid=$uid, numClasses=$numClasses, numFeatures=$numFeatures"
}
+
+ private[spark] def createSummary(
+ dataset: Dataset[_], objectiveHistory: Array[Double]
+ ): Unit = {
+ val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+
+ val (summaryModel, probabilityColName, predictionColName) = findSummaryModel()
+ val logRegSummary = if (numClasses <= 2) {
+ new BinaryLogisticRegressionTrainingSummaryImpl(
+ summaryModel.transform(dataset),
+ probabilityColName,
+ predictionColName,
+ $(labelCol),
+ $(featuresCol),
+ weightColName,
+ objectiveHistory)
+ } else {
+ new LogisticRegressionTrainingSummaryImpl(
+ summaryModel.transform(dataset),
+ probabilityColName,
+ predictionColName,
+ $(labelCol),
+ $(featuresCol),
+ weightColName,
+ objectiveHistory)
+ }
+ setSummary(Some(logRegSummary))
+ }
+
+ override private[spark] def saveSummary(path: String): Unit = {
+ ReadWriteUtils.saveObjectToLocal[Tuple1[Array[Double]]](
+ path, Tuple1(summary.objectiveHistory),
+ (data, dos) => {
+ ReadWriteUtils.serializeDoubleArray(data._1, dos)
+ }
+ )
+ }
+
+ override private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ val Tuple1(objectiveHistory: Array[Double])
+ = ReadWriteUtils.loadObjectFromLocal[Tuple1[Array[Double]]](
+ path,
+ dis => {
+ Tuple1(ReadWriteUtils.deserializeDoubleArray(dis))
+ }
+ )
+ createSummary(dataset, objectiveHistory)
+ }
}
@Since("1.6.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
index 6bd46cff815d7..5e52d62fb83cb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
@@ -251,14 +251,8 @@ class MultilayerPerceptronClassifier @Since("1.5.0") (
objectiveHistory: Array[Double]): MultilayerPerceptronClassificationModel = {
val model = copyValues(new MultilayerPerceptronClassificationModel(uid, weights))
- val (summaryModel, _, predictionColName) = model.findSummaryModel()
- val summary = new MultilayerPerceptronClassificationTrainingSummaryImpl(
- summaryModel.transform(dataset),
- predictionColName,
- $(labelCol),
- "",
- objectiveHistory)
- model.setSummary(Some(summary))
+ model.createSummary(dataset, objectiveHistory)
+ model
}
}
@@ -365,6 +359,39 @@ class MultilayerPerceptronClassificationModel private[ml] (
s"MultilayerPerceptronClassificationModel: uid=$uid, numLayers=${$(layers).length}, " +
s"numClasses=$numClasses, numFeatures=$numFeatures"
}
+
+ private[spark] def createSummary(
+ dataset: Dataset[_], objectiveHistory: Array[Double]
+ ): Unit = {
+ val (summaryModel, _, predictionColName) = findSummaryModel()
+ val summary = new MultilayerPerceptronClassificationTrainingSummaryImpl(
+ summaryModel.transform(dataset),
+ predictionColName,
+ $(labelCol),
+ "",
+ objectiveHistory)
+ setSummary(Some(summary))
+ }
+
+ override private[spark] def saveSummary(path: String): Unit = {
+ ReadWriteUtils.saveObjectToLocal[Tuple1[Array[Double]]](
+ path, Tuple1(summary.objectiveHistory),
+ (data, dos) => {
+ ReadWriteUtils.serializeDoubleArray(data._1, dos)
+ }
+ )
+ }
+
+ override private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ val Tuple1(objectiveHistory: Array[Double])
+ = ReadWriteUtils.loadObjectFromLocal[Tuple1[Array[Double]]](
+ path,
+ dis => {
+ Tuple1(ReadWriteUtils.deserializeDoubleArray(dis))
+ }
+ )
+ createSummary(dataset, objectiveHistory)
+ }
}
@Since("2.0.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
index 929bd1541ec6e..f62b951fd6505 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
@@ -29,7 +29,7 @@ import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{LogKeys, MDC}
+import org.apache.spark.internal.{LogKeys}
import org.apache.spark.ml._
import org.apache.spark.ml.attribute._
import org.apache.spark.ml.functions._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
index 61fab02cb4518..ea2c79d8a2181 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.classification
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{LogKeys, MDC}
+import org.apache.spark.internal.{LogKeys}
import org.apache.spark.ml.linalg.{DenseVector, Vector, VectorUDT}
import org.apache.spark.ml.param.ParamMap
import org.apache.spark.ml.param.shared._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
index f64e2a6d4efc3..8b580b1e075c5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
@@ -182,26 +182,8 @@ class RandomForestClassifier @Since("1.4.0") (
numFeatures: Int,
numClasses: Int): RandomForestClassificationModel = {
val model = copyValues(new RandomForestClassificationModel(uid, trees, numFeatures, numClasses))
- val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
-
- val (summaryModel, probabilityColName, predictionColName) = model.findSummaryModel()
- val rfSummary = if (numClasses <= 2) {
- new BinaryRandomForestClassificationTrainingSummaryImpl(
- summaryModel.transform(dataset),
- probabilityColName,
- predictionColName,
- $(labelCol),
- weightColName,
- Array(0.0))
- } else {
- new RandomForestClassificationTrainingSummaryImpl(
- summaryModel.transform(dataset),
- predictionColName,
- $(labelCol),
- weightColName,
- Array(0.0))
- }
- model.setSummary(Some(rfSummary))
+ model.createSummary(dataset)
+ model
}
@Since("1.4.1")
@@ -393,6 +375,35 @@ class RandomForestClassificationModel private[ml] (
@Since("2.0.0")
override def write: MLWriter =
new RandomForestClassificationModel.RandomForestClassificationModelWriter(this)
+
+ private[spark] def createSummary(dataset: Dataset[_]): Unit = {
+ val weightColName = if (!isDefined(weightCol)) "weightCol" else $(weightCol)
+
+ val (summaryModel, probabilityColName, predictionColName) = findSummaryModel()
+ val rfSummary = if (numClasses <= 2) {
+ new BinaryRandomForestClassificationTrainingSummaryImpl(
+ summaryModel.transform(dataset),
+ probabilityColName,
+ predictionColName,
+ $(labelCol),
+ weightColName,
+ Array(0.0))
+ } else {
+ new RandomForestClassificationTrainingSummaryImpl(
+ summaryModel.transform(dataset),
+ predictionColName,
+ $(labelCol),
+ weightColName,
+ Array(0.0))
+ }
+ setSummary(Some(rfSummary))
+ }
+
+ override private[spark] def saveSummary(path: String): Unit = {}
+
+ override private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ createSummary(dataset)
+ }
}
@Since("2.0.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index 3248b4b391d0a..9e09ee00c3e30 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -180,6 +180,9 @@ class BisectingKMeansModel private[ml] (
override def summary: BisectingKMeansSummary = super.summary
override def estimatedSize: Long = SizeEstimator.estimate(parentModel)
+
+ // BisectingKMeans model hasn't supported offloading, so put an empty `saveSummary` here for now
+ override private[spark] def saveSummary(path: String): Unit = {}
}
object BisectingKMeansModel extends MLReadable[BisectingKMeansModel] {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index a94b8a87d8fc7..5d3e36be28082 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path
import org.apache.spark.annotation.Since
import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.{LogKeys, MDC}
+import org.apache.spark.internal.{LogKeys}
import org.apache.spark.ml.{Estimator, Model}
import org.apache.spark.ml.impl.Utils.{unpackUpperTriangular, EPSILON}
import org.apache.spark.ml.linalg._
@@ -223,6 +223,36 @@ class GaussianMixtureModel private[ml] (
override def summary: GaussianMixtureSummary = super.summary
override def estimatedSize: Long = SizeEstimator.estimate((weights, gaussians))
+
+ private[spark] def createSummary(
+ predictions: DataFrame, logLikelihood: Double, iteration: Int
+ ): Unit = {
+ val summary = new GaussianMixtureSummary(predictions,
+ $(predictionCol), $(probabilityCol), $(featuresCol), $(k), logLikelihood, iteration)
+ setSummary(Some(summary))
+ }
+
+ override private[spark] def saveSummary(path: String): Unit = {
+ ReadWriteUtils.saveObjectToLocal[(Double, Int)](
+ path, (summary.logLikelihood, summary.numIter),
+ (data, dos) => {
+ dos.writeDouble(data._1)
+ dos.writeInt(data._2)
+ }
+ )
+ }
+
+ override private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ val (logLikelihood: Double, numIter: Int) = ReadWriteUtils.loadObjectFromLocal[(Double, Int)](
+ path,
+ dis => {
+ val logLikelihood = dis.readDouble()
+ val numIter = dis.readInt()
+ (logLikelihood, numIter)
+ }
+ )
+ createSummary(dataset, logLikelihood, numIter)
+ }
}
@Since("2.0.0")
@@ -453,11 +483,10 @@ class GaussianMixture @Since("2.0.0") (
val model = copyValues(new GaussianMixtureModel(uid, weights, gaussianDists))
.setParent(this)
- val summary = new GaussianMixtureSummary(model.transform(dataset),
- $(predictionCol), $(probabilityCol), $(featuresCol), $(k), logLikelihood, iteration)
+ model.createSummary(model.transform(dataset), logLikelihood, iteration)
instr.logNamedValue("logLikelihood", logLikelihood)
- instr.logNamedValue("clusterSizes", summary.clusterSizes)
- model.setSummary(Some(summary))
+ instr.logNamedValue("clusterSizes", model.summary.clusterSizes)
+ model
}
private def trainImpl(
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index f3ac58e670e5a..2abd82c712960 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -25,7 +25,6 @@ import org.apache.hadoop.fs.Path
import org.apache.spark.annotation.Since
import org.apache.spark.internal.LogKeys.{COST, INIT_MODE, NUM_ITERATIONS, TOTAL_TIME}
-import org.apache.spark.internal.MDC
import org.apache.spark.ml.{Estimator, Model, PipelineStage}
import org.apache.spark.ml.feature.{Instance, InstanceBlock}
import org.apache.spark.ml.linalg._
@@ -215,6 +214,42 @@ class KMeansModel private[ml] (
override def summary: KMeansSummary = super.summary
override def estimatedSize: Long = SizeEstimator.estimate(parentModel.clusterCenters)
+
+ private[spark] def createSummary(
+ predictions: DataFrame, numIter: Int, trainingCost: Double
+ ): Unit = {
+ val summary = new KMeansSummary(
+ predictions,
+ $(predictionCol),
+ $(featuresCol),
+ $(k),
+ numIter,
+ trainingCost)
+
+ setSummary(Some(summary))
+ }
+
+ override private[spark] def saveSummary(path: String): Unit = {
+ ReadWriteUtils.saveObjectToLocal[(Int, Double)](
+ path, (summary.numIter, summary.trainingCost),
+ (data, dos) => {
+ dos.writeInt(data._1)
+ dos.writeDouble(data._2)
+ }
+ )
+ }
+
+ override private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ val (numIter: Int, trainingCost: Double) = ReadWriteUtils.loadObjectFromLocal[(Int, Double)](
+ path,
+ dis => {
+ val numIter = dis.readInt()
+ val trainingCost = dis.readDouble()
+ (numIter, trainingCost)
+ }
+ )
+ createSummary(dataset, numIter, trainingCost)
+ }
}
/** Helper class for storing model data */
@@ -414,16 +449,9 @@ class KMeans @Since("1.5.0") (
}
val model = copyValues(new KMeansModel(uid, oldModel).setParent(this))
- val summary = new KMeansSummary(
- model.transform(dataset),
- $(predictionCol),
- $(featuresCol),
- $(k),
- oldModel.numIter,
- oldModel.trainingCost)
- model.setSummary(Some(summary))
- instr.logNamedValue("clusterSizes", summary.clusterSizes)
+ model.createSummary(model.transform(dataset), oldModel.numIter, oldModel.trainingCost)
+ instr.logNamedValue("clusterSizes", model.summary.clusterSizes)
model
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Binarizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Binarizer.scala
index 8123438fd8878..52ed90415f1cd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Binarizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Binarizer.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable.ArrayBuilder
import org.apache.spark.{SparkException, SparkIllegalArgumentException}
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{LogKeys, MDC}
+import org.apache.spark.internal.{LogKeys}
import org.apache.spark.ml.Transformer
import org.apache.spark.ml.attribute._
import org.apache.spark.ml.linalg._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala
index 12d957ea360f7..289217e1b8179 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala
@@ -231,7 +231,6 @@ object IDFModel extends MLReadable[IDFModel] {
override def load(path: String): IDFModel = {
val metadata = DefaultParamsReader.loadMetadata(path, sparkSession, className)
val dataPath = new Path(path, "data").toString
- val data = sparkSession.read.parquet(dataPath)
val model = if (majorVersion(metadata.sparkVersion) >= 3) {
val data = ReadWriteUtils.loadObject[Data](dataPath, sparkSession, deserializeData)
@@ -240,6 +239,7 @@ object IDFModel extends MLReadable[IDFModel] {
new feature.IDFModel(OldVectors.fromML(data.idf), data.docFreq, data.numDocs)
)
} else {
+ val data = sparkSession.read.parquet(dataPath)
val Row(idf: Vector) = MLUtils.convertVectorColumnsToML(data, "idf")
.select("idf")
.head()
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
index cb9d8b32f0064..4bd1bfaf582d0 100755
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala
@@ -20,7 +20,7 @@ package org.apache.spark.ml.feature
import java.util.Locale
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.ml.Transformer
import org.apache.spark.ml.param._
import org.apache.spark.ml.param.shared.{HasInputCol, HasInputCols, HasOutputCol, HasOutputCols}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
index db3749558f473..6ba57f583ae31 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path
import org.apache.spark.{SparkException, SparkIllegalArgumentException}
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{LogKeys, MDC}
+import org.apache.spark.internal.{LogKeys}
import org.apache.spark.ml.{Estimator, Model, Transformer}
import org.apache.spark.ml.attribute.{Attribute, NominalAttribute}
import org.apache.spark.ml.param._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala
index 17adf61e46e9d..346595f80a9c4 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala
@@ -17,13 +17,12 @@
package org.apache.spark.ml.optim
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NUM_ITERATIONS, RELATIVE_TOLERANCE}
-import org.apache.spark.internal.MDC
import org.apache.spark.ml.feature.{Instance, OffsetInstance}
import org.apache.spark.ml.linalg._
import org.apache.spark.ml.util.OptionalInstrumentation
import org.apache.spark.rdd.RDD
-import org.apache.spark.util.MavenUtils.LogStringContext
/**
* Model fitted by [[IterativelyReweightedLeastSquares]].
@@ -64,7 +63,7 @@ private[ml] class IterativelyReweightedLeastSquares(
val fitIntercept: Boolean,
val regParam: Double,
val maxIter: Int,
- val tol: Double) extends Serializable {
+ val tol: Double) extends Serializable with Logging {
def fit(
instances: RDD[OffsetInstance],
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala
index 31db7b18efb3a..177f75b898c1b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala
@@ -17,13 +17,12 @@
package org.apache.spark.ml.optim
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.COUNT
-import org.apache.spark.internal.MDC
import org.apache.spark.ml.feature.Instance
import org.apache.spark.ml.linalg._
import org.apache.spark.ml.util.OptionalInstrumentation
import org.apache.spark.rdd.RDD
-import org.apache.spark.util.MavenUtils.LogStringContext
/**
* Model fitted by [[WeightedLeastSquares]].
@@ -85,7 +84,7 @@ private[ml] class WeightedLeastSquares(
val solverType: WeightedLeastSquares.Solver = WeightedLeastSquares.Auto,
val maxIter: Int = 100,
val tol: Double = 1e-6
- ) extends Serializable {
+ ) extends Serializable with Logging {
import WeightedLeastSquares._
require(regParam >= 0.0, s"regParam cannot be negative: $regParam")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/RWrapperUtils.scala b/mllib/src/main/scala/org/apache/spark/ml/r/RWrapperUtils.scala
index 9e66647ef35fb..87948d1796d2f 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/r/RWrapperUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/r/RWrapperUtils.scala
@@ -17,7 +17,7 @@
package org.apache.spark.ml.r
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{FEATURE_COLUMN, LABEL_COLUMN, NEW_FEATURE_COLUMN_NAME, NEW_LABEL_COLUMN_NAME}
import org.apache.spark.ml.attribute.{Attribute, AttributeGroup, NominalAttribute}
import org.apache.spark.ml.feature.{RFormula, RFormulaModel}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index 276c7630d2d51..538ad03820754 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -33,7 +33,7 @@ import org.json4s.JsonDSL._
import org.apache.spark.{Partitioner, SparkException}
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PATH
import org.apache.spark.ml.{Estimator, Model}
import org.apache.spark.ml.linalg.BLAS
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index 3aee34a148adf..d96500ea84ab9 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -26,7 +26,7 @@ import breeze.optimize.{CachedDiffFunction, LBFGS => BreezeLBFGS}
import org.apache.hadoop.fs.Path
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.ml.PredictorParams
import org.apache.spark.ml.feature._
import org.apache.spark.ml.linalg._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
index 2c40a2f353b7d..f049e9a44cc28 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.scala
@@ -22,7 +22,7 @@ import org.json4s.{DefaultFormats, JObject}
import org.json4s.JsonDSL._
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{LogKeys, MDC}
+import org.apache.spark.internal.{LogKeys}
import org.apache.spark.ml.feature.Instance
import org.apache.spark.ml.linalg.Vector
import org.apache.spark.ml.param.ParamMap
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
index 163921c703903..c8fa97bfccce0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GBTRegressor.scala
@@ -21,7 +21,7 @@ import org.json4s.{DefaultFormats, JObject}
import org.json4s.JsonDSL._
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.ml.linalg.{BLAS, Vector}
import org.apache.spark.ml.param.ParamMap
import org.apache.spark.ml.tree._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
index 14467c761b216..f4d6e4e3562af 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
@@ -22,12 +22,11 @@ import java.util.Locale
import breeze.stats.{distributions => dist}
import breeze.stats.distributions.Rand.FixedSeed.randBasis
-import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.ml.PredictorParams
import org.apache.spark.ml.attribute._
import org.apache.spark.ml.feature.{Instance, OffsetInstance}
@@ -42,6 +41,7 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Column, DataFrame, Dataset, Row}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types.{DataType, DoubleType, StructType}
+import org.apache.spark.util.Utils
/**
* Params for Generalized Linear Regression.
@@ -419,9 +419,8 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
val model = copyValues(
new GeneralizedLinearRegressionModel(uid, wlsModel.coefficients, wlsModel.intercept)
.setParent(this))
- val trainingSummary = new GeneralizedLinearRegressionTrainingSummary(dataset, model,
- wlsModel.diagInvAtWA.toArray, 1, getSolver)
- model.setSummary(Some(trainingSummary))
+ model.createSummary(dataset, wlsModel.diagInvAtWA.toArray, 1)
+ model
} else {
val instances = validated.rdd.map {
case Row(label: Double, weight: Double, offset: Double, features: Vector) =>
@@ -436,9 +435,8 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
val model = copyValues(
new GeneralizedLinearRegressionModel(uid, irlsModel.coefficients, irlsModel.intercept)
.setParent(this))
- val trainingSummary = new GeneralizedLinearRegressionTrainingSummary(dataset, model,
- irlsModel.diagInvAtWA.toArray, irlsModel.numIterations, getSolver)
- model.setSummary(Some(trainingSummary))
+ model.createSummary(dataset, irlsModel.diagInvAtWA.toArray, irlsModel.numIterations)
+ model
}
model
@@ -1140,6 +1138,39 @@ class GeneralizedLinearRegressionModel private[ml] (
s"GeneralizedLinearRegressionModel: uid=$uid, family=${$(family)}, link=${$(link)}, " +
s"numFeatures=$numFeatures"
}
+
+ private[spark] def createSummary(
+ dataset: Dataset[_], diagInvAtWA: Array[Double], numIter: Int
+ ): Unit = {
+ val summary = new GeneralizedLinearRegressionTrainingSummary(
+ dataset, this, diagInvAtWA, numIter, $(solver)
+ )
+
+ setSummary(Some(summary))
+ }
+
+ override private[spark] def saveSummary(path: String): Unit = {
+ ReadWriteUtils.saveObjectToLocal[(Array[Double], Int)](
+ path, (summary.diagInvAtWA, summary.numIterations),
+ (data, dos) => {
+ ReadWriteUtils.serializeDoubleArray(data._1, dos)
+ dos.writeInt(data._2)
+ }
+ )
+ }
+
+ override private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ val (diagInvAtWA: Array[Double], numIterations: Int) =
+ ReadWriteUtils.loadObjectFromLocal[(Array[Double], Int)](
+ path,
+ dis => {
+ val diagInvAtWA = ReadWriteUtils.deserializeDoubleArray(dis)
+ val numIterations = dis.readInt()
+ (diagInvAtWA, numIterations)
+ }
+ )
+ createSummary(dataset, diagInvAtWA, numIterations)
+ }
}
@Since("2.0.0")
@@ -1467,7 +1498,7 @@ class GeneralizedLinearRegressionSummary private[regression] (
class GeneralizedLinearRegressionTrainingSummary private[regression] (
dataset: Dataset[_],
origModel: GeneralizedLinearRegressionModel,
- private val diagInvAtWA: Array[Double],
+ private[spark] val diagInvAtWA: Array[Double],
@Since("2.0.0") val numIterations: Int,
@Since("2.0.0") val solver: String)
extends GeneralizedLinearRegressionSummary(dataset, origModel) with Serializable {
@@ -1605,12 +1636,12 @@ class GeneralizedLinearRegressionTrainingSummary private[regression] (
// Output coefficients with statistics
sb.append("Coefficients:\n")
colNames.zipWithIndex.map { case (colName: String, i: Int) =>
- StringUtils.leftPad(colName, colWidths(i))
+ Utils.leftPad(colName, colWidths(i))
}.addString(sb, "", " ", "\n")
data.foreach { case strRow: Array[String] =>
strRow.zipWithIndex.map { case (cell: String, i: Int) =>
- StringUtils.leftPad(cell, colWidths(i))
+ Utils.leftPad(cell, colWidths(i))
}.addString(sb, "", " ", "\n")
}
@@ -1623,9 +1654,9 @@ class GeneralizedLinearRegressionTrainingSummary private[regression] (
val rd = s"Residual deviance: ${round(deviance)} on $residualDegreeOfFreedom degrees of " +
"freedom"
val l = math.max(nd.length, rd.length)
- sb.append(StringUtils.leftPad(nd, l))
+ sb.append(Utils.leftPad(nd, l))
sb.append("\n")
- sb.append(StringUtils.leftPad(rd, l))
+ sb.append(Utils.leftPad(rd, l))
if (family.name != "tweedie") {
sb.append("\n")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
index b06140e48338c..822df270c0bf7 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
@@ -433,15 +433,8 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
}
val model = createModel(parameters, yMean, yStd, featuresMean, featuresStd)
-
- // Handle possible missing or invalid prediction columns
- val (summaryModel, predictionColName) = model.findSummaryModelAndPredictionCol()
- val trainingSummary = new LinearRegressionTrainingSummary(
- summaryModel.transform(dataset), predictionColName, $(labelCol), $(featuresCol),
- summaryModel.get(summaryModel.weightCol).getOrElse(""),
- summaryModel.numFeatures, summaryModel.getFitIntercept,
- Array(0.0), objectiveHistory)
- model.setSummary(Some(trainingSummary))
+ model.createSummary(dataset, Array(0.0), objectiveHistory, Array.emptyDoubleArray)
+ model
}
private def trainWithNormal(
@@ -459,20 +452,16 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
// attach returned model.
val lrModel = copyValues(new LinearRegressionModel(
uid, model.coefficients.compressed, model.intercept))
- val (summaryModel, predictionColName) = lrModel.findSummaryModelAndPredictionCol()
- val coefficientArray = if (summaryModel.getFitIntercept) {
- summaryModel.coefficients.toArray ++ Array(summaryModel.intercept)
+ val coefficientArray = if (lrModel.getFitIntercept) {
+ lrModel.coefficients.toArray ++ Array(lrModel.intercept)
} else {
- summaryModel.coefficients.toArray
+ lrModel.coefficients.toArray
}
- val trainingSummary = new LinearRegressionTrainingSummary(
- summaryModel.transform(dataset), predictionColName, $(labelCol), $(featuresCol),
- summaryModel.get(summaryModel.weightCol).getOrElse(""),
- summaryModel.numFeatures, summaryModel.getFitIntercept,
- model.diagInvAtWA.toArray, model.objectiveHistory, coefficientArray)
-
- lrModel.setSummary(Some(trainingSummary))
+ lrModel.createSummary(
+ dataset, model.diagInvAtWA.toArray, model.objectiveHistory, coefficientArray
+ )
+ lrModel
}
private def trainWithConstantLabel(
@@ -497,16 +486,9 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
val intercept = yMean
val model = copyValues(new LinearRegressionModel(uid, coefficients, intercept))
- // Handle possible missing or invalid prediction columns
- val (summaryModel, predictionColName) = model.findSummaryModelAndPredictionCol()
- val trainingSummary = new LinearRegressionTrainingSummary(
- summaryModel.transform(dataset), predictionColName, $(labelCol), $(featuresCol),
- summaryModel.get(summaryModel.weightCol).getOrElse(""),
- summaryModel.numFeatures, summaryModel.getFitIntercept,
- Array(0.0), Array(0.0))
-
- model.setSummary(Some(trainingSummary))
+ model.createSummary(dataset, Array(0.0), Array(0.0), Array.emptyDoubleArray)
+ model
}
private def createOptimizer(
@@ -800,6 +782,53 @@ class LinearRegressionModel private[ml] (
override def toString: String = {
s"LinearRegressionModel: uid=$uid, numFeatures=$numFeatures"
}
+
+ private[spark] def createSummary(
+ dataset: Dataset[_],
+ diagInvAtWA: Array[Double],
+ objectiveHistory: Array[Double],
+ coefficientArray: Array[Double]
+ ): Unit = {
+ // Handle possible missing or invalid prediction columns
+ val (summaryModel, predictionColName) = findSummaryModelAndPredictionCol()
+
+ val trainingSummary = new LinearRegressionTrainingSummary(
+ summaryModel.transform(dataset), predictionColName, $(labelCol), $(featuresCol),
+ summaryModel.get(summaryModel.weightCol).getOrElse(""),
+ summaryModel.numFeatures, summaryModel.getFitIntercept,
+ diagInvAtWA, objectiveHistory, coefficientArray)
+
+ setSummary(Some(trainingSummary))
+ }
+
+ override private[spark] def saveSummary(path: String): Unit = {
+ ReadWriteUtils.saveObjectToLocal[(Array[Double], Array[Double], Array[Double])](
+ path, (summary.diagInvAtWA, summary.objectiveHistory, summary.coefficientArray),
+ (data, dos) => {
+ ReadWriteUtils.serializeDoubleArray(data._1, dos)
+ ReadWriteUtils.serializeDoubleArray(data._2, dos)
+ ReadWriteUtils.serializeDoubleArray(data._3, dos)
+ }
+ )
+ }
+
+ override private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ val (
+ diagInvAtWA: Array[Double],
+ objectiveHistory: Array[Double],
+ coefficientArray: Array[Double]
+ )
+ = ReadWriteUtils.loadObjectFromLocal[(Array[Double], Array[Double], Array[Double])](
+ path,
+ dis => {
+ val diagInvAtWA = ReadWriteUtils.deserializeDoubleArray(dis)
+ val objectiveHistory = ReadWriteUtils.deserializeDoubleArray(dis)
+ val coefficientArray = ReadWriteUtils.deserializeDoubleArray(dis)
+ (diagInvAtWA, objectiveHistory, coefficientArray)
+ }
+ )
+ createSummary(dataset, diagInvAtWA, objectiveHistory, coefficientArray)
+ }
}
private[ml] case class LinearModelData(intercept: Double, coefficients: Vector, scale: Double)
@@ -926,7 +955,7 @@ class LinearRegressionTrainingSummary private[regression] (
private val fitIntercept: Boolean,
diagInvAtWA: Array[Double],
val objectiveHistory: Array[Double],
- private val coefficientArray: Array[Double] = Array.emptyDoubleArray)
+ override private[regression] val coefficientArray: Array[Double] = Array.emptyDoubleArray)
extends LinearRegressionSummary(
predictions,
predictionCol,
@@ -972,8 +1001,8 @@ class LinearRegressionSummary private[regression] (
private val weightCol: String,
private val numFeatures: Int,
private val fitIntercept: Boolean,
- private val diagInvAtWA: Array[Double],
- private val coefficientArray: Array[Double] = Array.emptyDoubleArray)
+ private[regression] val diagInvAtWA: Array[Double],
+ private[regression] val coefficientArray: Array[Double] = Array.emptyDoubleArray)
extends Summary with Serializable {
@transient private val metrics = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
index 04e36c7f27ff3..a9e2c47a3229a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
@@ -21,7 +21,7 @@ import org.json4s.{DefaultFormats, JObject}
import org.json4s.JsonDSL._
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{LogKeys, MDC}
+import org.apache.spark.internal.{LogKeys}
import org.apache.spark.ml.feature.Instance
import org.apache.spark.ml.linalg.Vector
import org.apache.spark.ml.param.ParamMap
diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala b/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala
index f44cbd5b3acca..10a3101dbb608 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala
@@ -17,7 +17,7 @@
package org.apache.spark.ml.source.image
-import com.google.common.io.{ByteStreams, Closeables}
+import com.google.common.io.Closeables
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.mapreduce.Job
@@ -76,7 +76,7 @@ private[image] case class ImageFileFormat() extends FileFormat with DataSourceRe
val fs = path.getFileSystem(broadcastedHadoopConf.value.value)
val stream = fs.open(path)
val bytes = try {
- ByteStreams.toByteArray(stream)
+ stream.readAllBytes()
} finally {
Closeables.close(stream, true)
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala
index 697d989538391..b68a5e079dc44 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala
@@ -134,7 +134,7 @@ class LeafNode private[ml] (
override private[tree] def numDescendants: Int = 0
override private[tree] def subtreeToString(indentFactor: Int = 0): String = {
- val prefix: String = " " * indentFactor
+ val prefix: String = " ".repeat(indentFactor)
prefix + s"Predict: $prediction\n"
}
@@ -212,7 +212,7 @@ class InternalNode private[ml] (
}
override private[tree] def subtreeToString(indentFactor: Int = 0): String = {
- val prefix: String = " " * indentFactor
+ val prefix: String = " ".repeat(indentFactor)
prefix + s"If (${InternalNode.splitToString(split, left = true)})\n" +
leftChild.subtreeToString(indentFactor + 1) +
prefix + s"Else (${InternalNode.splitToString(split, left = false)})\n" +
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala
index 2f63f4ae073e5..181168385011d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala
@@ -20,7 +20,7 @@ package org.apache.spark.ml.tree.impl
import scala.collection.mutable
import scala.util.Try
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.ml.feature.Instance
import org.apache.spark.ml.tree.TreeEnsembleParams
import org.apache.spark.mllib.tree.configuration.Algo._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala
index d1cad44a15c8a..81ffa5c86a9fa 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.tree.impl
import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.TIMER
import org.apache.spark.ml.EstimatorUtils
import org.apache.spark.ml.feature.Instance
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
index 118d6d7a063ac..cabbc497571b6 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
import scala.util.Random
import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{MAX_MEMORY_SIZE, MEMORY_SIZE, NUM_CLASSES, NUM_EXAMPLES, NUM_FEATURES, NUM_NODES, NUM_WEIGHTED_EXAMPLES, TIMER}
import org.apache.spark.ml.EstimatorUtils
import org.apache.spark.ml.classification.DecisionTreeClassificationModel
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala
index bc6b747344e31..119cc47a56f4f 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.Path
import org.json4s.DefaultFormats
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CROSS_VALIDATION_METRIC, CROSS_VALIDATION_METRICS, ESTIMATOR_PARAM_MAP}
import org.apache.spark.ml.{Estimator, Model}
import org.apache.spark.ml.evaluation.Evaluator
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tuning/TrainValidationSplit.scala b/mllib/src/main/scala/org/apache/spark/ml/tuning/TrainValidationSplit.scala
index 324a08ba0b5ab..6ee64ef99a668 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tuning/TrainValidationSplit.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tuning/TrainValidationSplit.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.Path
import org.json4s.DefaultFormats
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ESTIMATOR_PARAM_MAP, TRAIN_VALIDATION_SPLIT_METRIC, TRAIN_VALIDATION_SPLIT_METRICS}
import org.apache.spark.ml.{Estimator, Model}
import org.apache.spark.ml.evaluation.Evaluator
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/DatasetUtils.scala b/mllib/src/main/scala/org/apache/spark/ml/util/DatasetUtils.scala
index d5b6396e5ba80..06de43260b303 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/DatasetUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/DatasetUtils.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.util
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLASS_NAME, LABEL_COLUMN, NUM_CLASSES}
import org.apache.spark.ml.PredictorParams
import org.apache.spark.ml.classification.ClassifierParams
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/HasTrainingSummary.scala b/mllib/src/main/scala/org/apache/spark/ml/util/HasTrainingSummary.scala
index 0ba8ce072ab4a..c6f6babf71a2b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/HasTrainingSummary.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/HasTrainingSummary.scala
@@ -19,6 +19,7 @@ package org.apache.spark.ml.util
import org.apache.spark.SparkException
import org.apache.spark.annotation.Since
+import org.apache.spark.sql.DataFrame
/**
@@ -49,4 +50,14 @@ private[spark] trait HasTrainingSummary[T] {
this.trainingSummary = summary
this
}
+
+ private[spark] def loadSummary(path: String, dataset: DataFrame): Unit = {
+ throw new SparkException(
+ s"No loadSummary implementation for this ${this.getClass.getSimpleName}")
+ }
+
+ private[spark] def saveSummary(path: String): Unit = {
+ throw new SparkException(
+ s"No saveSummary implementation for this ${this.getClass.getSimpleName}")
+ }
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala b/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala
index 9413605a31ced..ddd63736023a3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala
@@ -27,7 +27,7 @@ import org.json4s._
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._
-import org.apache.spark.internal.{LogEntry, Logging, MDC}
+import org.apache.spark.internal.{LogEntry, Logging}
import org.apache.spark.internal.LogKeys.{CLASS_NAME, NUM_PARTITIONS, PIPELINE_STAGE_UID, STORAGE_LEVEL}
import org.apache.spark.ml.{MLEvents, PipelineStage}
import org.apache.spark.ml.param.{Param, Params}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
index 3a6f7fc00d6f9..e3f31874a4c23 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
@@ -30,7 +30,6 @@ import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import scala.util.{Failure, Success, Try, Using}
-import org.apache.commons.io.FileUtils
import org.apache.hadoop.fs.Path
import org.json4s._
import org.json4s.{DefaultFormats, JObject}
@@ -39,7 +38,7 @@ import org.json4s.jackson.JsonMethods._
import org.apache.spark.{SparkContext, SparkException}
import org.apache.spark.annotation.{Since, Unstable}
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.PATH
import org.apache.spark.ml._
import org.apache.spark.ml.classification.{OneVsRest, OneVsRestModel}
@@ -822,7 +821,7 @@ private[spark] class FileSystemOverwrite extends Logging {
val filePath = new File(path)
if (filePath.exists()) {
if (shouldOverwrite) {
- FileUtils.deleteDirectory(filePath)
+ Utils.deleteRecursively(filePath)
} else {
throw new IOException(errMsg)
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
index c826654f0893c..0d94c76640291 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
@@ -24,7 +24,7 @@ import scala.collection.mutable
import org.apache.spark.annotation.Since
import org.apache.spark.api.java.JavaRDD
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLUSTER_LEVEL, COST, DIVISIBLE_CLUSTER_INDICES_SIZE, FEATURE_DIMENSION, MIN_POINT_PER_CLUSTER, NUM_POINT}
import org.apache.spark.ml.util.Instrumentation
import org.apache.spark.mllib.linalg.{Vector, Vectors}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
index 52fbc7a5a47f5..21c2af721428c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable.ArrayBuffer
import org.apache.spark.annotation.Since
import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{COST, INIT_MODE, NUM_ITERATIONS, TOTAL_TIME}
import org.apache.spark.ml.util.Instrumentation
import org.apache.spark.mllib.linalg.{Vector, Vectors}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LocalKMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LocalKMeans.scala
index ea83be1237298..ac31c0d3be479 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LocalKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LocalKMeans.scala
@@ -19,7 +19,7 @@ package org.apache.spark.mllib.clustering
import scala.util.Random
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{NUM_ITERATIONS, POINT_OF_CENTER}
import org.apache.spark.mllib.linalg.BLAS.{axpy, scal}
import org.apache.spark.mllib.linalg.Vectors
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.scala
index 639e762ef3c87..10e8a992c784e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.scala
@@ -25,7 +25,7 @@ import org.apache.spark.{SparkContext, SparkException}
import org.apache.spark.annotation.Since
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.graphx._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{DELTA, DIFF_DELTA, NORM, NUM_ITERATIONS}
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.util.{Loader, MLUtils, Saveable}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala
index 641b4fa4048a6..c05ca9ecff3e9 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/StreamingKMeans.scala
@@ -21,7 +21,7 @@ import scala.reflect.ClassTag
import org.apache.spark.annotation.Since
import org.apache.spark.api.java.JavaSparkContext._
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.LogKeys.{CLUSTER_CENTROIDS, CLUSTER_LABEL, CLUSTER_WEIGHT, LARGEST_CLUSTER_INDEX, SMALLEST_CLUSTER_INDEX}
import org.apache.spark.mllib.linalg.{BLAS, Vector, Vectors}
import org.apache.spark.rdd.RDD
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala
index a203d3fc73537..42007e96ca3a2 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala
@@ -18,7 +18,7 @@
package org.apache.spark.mllib.evaluation
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{COUNT, NUM_BIN}
import org.apache.spark.mllib.evaluation.binary._
import org.apache.spark.rdd.RDD
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
index 100fa13db5180..d99dfd504cb80 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
@@ -31,7 +31,7 @@ import org.apache.spark.SparkContext
import org.apache.spark.annotation.Since
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{ALPHA, COUNT, NUM_TRAIN_WORD, VOCAB_SIZE}
import org.apache.spark.internal.config.Kryo.KRYO_SERIALIZER_MAX_BUFFER_SIZE
import org.apache.spark.ml.linalg.BLAS
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
index 9c16ac2ecd526..a81bd823415af 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
@@ -33,7 +33,7 @@ import org.apache.spark.SparkContext
import org.apache.spark.annotation.Since
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{MIN_NUM_FREQUENT_PATTERN, NUM_FREQUENT_ITEMS, NUM_LOCAL_FREQUENT_PATTERN, NUM_PREFIXES, NUM_SEQUENCES}
import org.apache.spark.mllib.util.{Loader, Saveable}
import org.apache.spark.rdd.RDD
@@ -460,7 +460,6 @@ object PrefixSpan extends Logging {
def genPrefixItems: Iterator[(Int, Long)] = {
val n1 = items.length - 1
// For each unique item (subject to sign) in this sequence, we output exact one split.
- // TODO: use PrimitiveKeyOpenHashMap
val prefixes = mutable.Map.empty[Int, Long]
// a) items that can be assembled to the last itemset of the prefix
partialStarts.foreach { start =>
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
index fc0c8d42579a9..a0b31d3d9282b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
@@ -17,7 +17,7 @@
package org.apache.spark.mllib.linalg
-import java.util.{Arrays, Random}
+import java.util.{Arrays, Objects, Random}
import scala.collection.mutable.{ArrayBuffer, ArrayBuilder => MArrayBuilder, HashSet => MHashSet}
import scala.language.implicitConversions
@@ -314,7 +314,7 @@ class DenseMatrix @Since("1.3.0") (
}
override def hashCode: Int = {
- com.google.common.base.Objects.hashCode(numRows: Integer, numCols: Integer, toArray)
+ Objects.hash(numRows: Integer, numCols: Integer, toArray)
}
private[mllib] def asBreeze: BM[Double] = {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
index 3329682d3b550..50ce6f008f07e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
@@ -17,13 +17,15 @@
package org.apache.spark.mllib.linalg.distributed
+import java.util.Objects
+
import scala.collection.mutable.ArrayBuffer
import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV, Matrix => BM}
import org.apache.spark.{Partitioner, PartitionIdPassthrough, SparkException}
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.mllib.linalg._
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
@@ -90,7 +92,7 @@ private[mllib] class GridPartitioner(
}
override def hashCode: Int = {
- com.google.common.base.Objects.hashCode(
+ Objects.hash(
rows: java.lang.Integer,
cols: java.lang.Integer,
rowsPerPart: java.lang.Integer,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
index 7621489bb1f0c..01abf22ad87c6 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
@@ -26,7 +26,7 @@ import breeze.linalg.{axpy => brzAxpy, inv, svd => brzSvd, DenseMatrix => BDM, D
import breeze.numerics.{sqrt => brzSqrt}
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.MAX_RESULT_SIZE
import org.apache.spark.mllib.linalg._
import org.apache.spark.mllib.stat._
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
index d4ff30c6add04..c7ed8b377fdb0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable.ArrayBuffer
import breeze.linalg.{norm, DenseVector => BDV}
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.mllib.linalg.{Vector, Vectors}
import org.apache.spark.rdd.RDD
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
index 3a8cde3ca1e89..2d3eab2a5708c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
@@ -22,7 +22,7 @@ import scala.collection.mutable
import breeze.linalg.{DenseVector => BDV}
import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS}
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.mllib.linalg.{Vector, Vectors}
import org.apache.spark.mllib.linalg.BLAS.axpy
import org.apache.spark.rdd.RDD
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
index fb9e8ac7c8920..9d239b5b7a503 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
@@ -30,7 +30,7 @@ import org.json4s.jackson.JsonMethods._
import org.apache.spark.SparkContext
import org.apache.spark.annotation.Since
import org.apache.spark.api.java.{JavaPairRDD, JavaRDD}
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.mllib.linalg.BLAS
import org.apache.spark.mllib.rdd.MLPairRDDFunctions._
import org.apache.spark.mllib.util.{Loader, Saveable}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala
index 2fa1339bc72a9..124a4a6596d27 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala
@@ -21,7 +21,7 @@ import scala.reflect.ClassTag
import org.apache.spark.annotation.Since
import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{MODEL_WEIGHTS, TIME}
import org.apache.spark.mllib.linalg.Vector
import org.apache.spark.streaming.api.java.{JavaDStream, JavaPairDStream}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/ChiSqTest.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/ChiSqTest.scala
index 2059a9f785381..15f7a700be312 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/ChiSqTest.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/ChiSqTest.scala
@@ -20,7 +20,7 @@ package org.apache.spark.mllib.stat.test
import org.apache.commons.math3.distribution.ChiSquaredDistribution
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.mllib.linalg._
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.rdd.RDD
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
index 2acc49e218f2d..299fd1073f8ed 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
@@ -26,7 +26,7 @@ import org.json4s.jackson.JsonMethods._
import org.apache.spark.SparkContext
import org.apache.spark.annotation.Since
import org.apache.spark.api.java.JavaRDD
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.mllib.linalg.Vector
import org.apache.spark.mllib.tree.configuration.{Algo, FeatureType}
import org.apache.spark.mllib.tree.configuration.Algo._
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala
index 54d95a3ffad1b..5a29516c926e8 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala
@@ -17,6 +17,8 @@
package org.apache.spark.mllib.tree.model
+import java.util.Objects
+
import org.apache.spark.annotation.Since
import org.apache.spark.mllib.tree.impurity.ImpurityCalculator
@@ -56,7 +58,7 @@ class InformationGainStats(
}
override def hashCode: Int = {
- com.google.common.base.Objects.hashCode(
+ Objects.hash(
gain: java.lang.Double,
impurity: java.lang.Double,
leftImpurity: java.lang.Double,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala
index bac36ce6937b0..a0eec8b2afb9c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala
@@ -137,7 +137,7 @@ class Node @Since("1.2.0") (
}
}
}
- val prefix: String = " " * indentFactor
+ val prefix: String = " ".repeat(indentFactor)
if (isLeaf) {
prefix + s"Predict: ${predict.predict}\n"
} else {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala
index 44bc625d217f1..f3d66c4f5ba3e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala
@@ -17,6 +17,8 @@
package org.apache.spark.mllib.tree.model
+import java.util.Objects
+
import org.apache.spark.annotation.Since
/**
@@ -39,6 +41,6 @@ class Predict @Since("1.2.0") (
}
override def hashCode: Int = {
- com.google.common.base.Objects.hashCode(predict: java.lang.Double, prob: java.lang.Double)
+ Objects.hash(predict: java.lang.Double, prob: java.lang.Double)
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
index af922dda13f6b..8737cd1617be0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
@@ -26,7 +26,7 @@ import org.json4s.jackson.JsonMethods._
import org.apache.spark.SparkContext
import org.apache.spark.annotation.Since
import org.apache.spark.api.java.JavaRDD
-import org.apache.spark.internal.{Logging, LogKeys, MDC}
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.ml.linalg.BLAS
import org.apache.spark.mllib.linalg.Vector
import org.apache.spark.mllib.regression.LabeledPoint
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala
index 4857c9b00f421..d7ed3ab1e2abf 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala
@@ -18,7 +18,7 @@
package org.apache.spark.mllib.util
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{COUNT, RANGE}
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.rdd.RDD
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
index b8fcb1ffcbfe1..5a213e6803486 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
@@ -22,7 +22,7 @@ import scala.reflect.ClassTag
import org.apache.spark.{SparkContext, SparkException}
import org.apache.spark.annotation.Since
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.OPTIMIZER_CLASS_NAME
import org.apache.spark.ml.linalg.{MatrixUDT => MLMatrixUDT, VectorUDT => MLVectorUDT}
import org.apache.spark.ml.util.Instrumentation
diff --git a/mllib/src/test/java/org/apache/spark/ml/source/libsvm/JavaLibSVMRelationSuite.java b/mllib/src/test/java/org/apache/spark/ml/source/libsvm/JavaLibSVMRelationSuite.java
index 5f0d22ea2a8aa..9d6fef9a55977 100644
--- a/mllib/src/test/java/org/apache/spark/ml/source/libsvm/JavaLibSVMRelationSuite.java
+++ b/mllib/src/test/java/org/apache/spark/ml/source/libsvm/JavaLibSVMRelationSuite.java
@@ -19,9 +19,7 @@
import java.io.File;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-
-import com.google.common.io.Files;
+import java.nio.file.Files;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -50,7 +48,7 @@ public void setUp() throws IOException {
tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "datasource");
File file = new File(tempDir, "part-00000");
String s = "1 1:1.0 3:2.0 5:3.0\n0\n0 2:4.0 4:5.0 6:6.0";
- Files.asCharSink(file, StandardCharsets.UTF_8).write(s);
+ Files.writeString(file.toPath(), s);
path = tempDir.toURI().toString();
}
diff --git a/mllib/src/test/java/org/apache/spark/mllib/feature/JavaWord2VecSuite.java b/mllib/src/test/java/org/apache/spark/mllib/feature/JavaWord2VecSuite.java
index 6543373a5a4d9..044b92666d9b5 100644
--- a/mllib/src/test/java/org/apache/spark/mllib/feature/JavaWord2VecSuite.java
+++ b/mllib/src/test/java/org/apache/spark/mllib/feature/JavaWord2VecSuite.java
@@ -20,8 +20,6 @@
import java.util.Arrays;
import java.util.List;
-import com.google.common.base.Strings;
-
import scala.Tuple2;
import org.junit.jupiter.api.Assertions;
@@ -35,7 +33,7 @@ public class JavaWord2VecSuite extends SharedSparkSession {
@Test
public void word2Vec() {
// The tests are to check Java compatibility.
- String sentence = Strings.repeat("a b ", 100) + Strings.repeat("a c ", 10);
+ String sentence = "a b ".repeat(100) + "a c ".repeat(10);
List words = Arrays.asList(sentence.split(" "));
List> localDoc = Arrays.asList(words, words);
JavaRDD> doc = jsc.parallelize(localDoc);
diff --git a/mllib/src/test/scala/org/apache/spark/ml/MLEventsSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/MLEventsSuite.scala
index 0b7426c1701f6..e28b26011593e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/MLEventsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/MLEventsSuite.scala
@@ -26,7 +26,6 @@ import org.mockito.Mockito.when
import org.scalatest.concurrent.Eventually
import org.scalatestplus.mockito.MockitoSugar.mock
-import org.apache.spark.SparkConf
import org.apache.spark.SparkFunSuite
import org.apache.spark.ml.param.ParamMap
import org.apache.spark.ml.util.{DefaultParamsReader, DefaultParamsWriter, MLWriter}
@@ -44,7 +43,6 @@ class MLEventsSuite extends SparkFunSuite with MLlibTestSparkContext with Eventu
case _ =>
}
}
- private val jsonProtocol = new JsonProtocol(new SparkConf())
override def beforeAll(): Unit = {
super.beforeAll()
@@ -145,8 +143,8 @@ class MLEventsSuite extends SparkFunSuite with MLlibTestSparkContext with Eventu
}
// Test if they can be ser/de via JSON protocol.
assert(events.nonEmpty)
- events.map(jsonProtocol.sparkEventToJsonString).foreach { event =>
- assert(jsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
+ events.map(JsonProtocol.sparkEventToJsonString).foreach { event =>
+ assert(JsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
}
}
@@ -203,8 +201,8 @@ class MLEventsSuite extends SparkFunSuite with MLlibTestSparkContext with Eventu
}
// Test if they can be ser/de via JSON protocol.
assert(events.nonEmpty)
- events.map(jsonProtocol.sparkEventToJsonString).foreach { event =>
- assert(jsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
+ events.map(JsonProtocol.sparkEventToJsonString).foreach { event =>
+ assert(JsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
}
}
@@ -235,8 +233,8 @@ class MLEventsSuite extends SparkFunSuite with MLlibTestSparkContext with Eventu
// Test if they can be ser/de via JSON protocol.
eventually(timeout(10.seconds), interval(1.second)) {
assert(events.nonEmpty)
- events.map(jsonProtocol.sparkEventToJsonString).foreach { event =>
- assert(jsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
+ events.map(JsonProtocol.sparkEventToJsonString).foreach { event =>
+ assert(JsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
}
}
sc.listenerBus.waitUntilEmpty(timeoutMillis = 10000)
@@ -263,8 +261,8 @@ class MLEventsSuite extends SparkFunSuite with MLlibTestSparkContext with Eventu
// Test if they can be ser/de via JSON protocol.
eventually(timeout(10.seconds), interval(1.second)) {
assert(events.nonEmpty)
- events.map(jsonProtocol.sparkEventToJsonString).foreach { event =>
- assert(jsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
+ events.map(JsonProtocol.sparkEventToJsonString).foreach { event =>
+ assert(JsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
}
}
}
@@ -298,8 +296,8 @@ class MLEventsSuite extends SparkFunSuite with MLlibTestSparkContext with Eventu
// Test if they can be ser/de via JSON protocol.
eventually(timeout(10.seconds), interval(1.second)) {
assert(events.nonEmpty)
- events.map(jsonProtocol.sparkEventToJsonString).foreach { event =>
- assert(jsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
+ events.map(JsonProtocol.sparkEventToJsonString).foreach { event =>
+ assert(JsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
}
}
sc.listenerBus.waitUntilEmpty(timeoutMillis = 10000)
@@ -326,8 +324,8 @@ class MLEventsSuite extends SparkFunSuite with MLlibTestSparkContext with Eventu
// Test if they can be ser/de via JSON protocol.
eventually(timeout(10.seconds), interval(1.second)) {
assert(events.nonEmpty)
- events.map(jsonProtocol.sparkEventToJsonString).foreach { event =>
- assert(jsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
+ events.map(JsonProtocol.sparkEventToJsonString).foreach { event =>
+ assert(JsonProtocol.sparkEventFromJson(event).isInstanceOf[MLEvent])
}
}
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala
index 582a11df793ed..4f5c8b25de772 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala
@@ -120,9 +120,12 @@ class IDFSuite extends MLTest with DefaultReadWriteTest {
new OldIDFModel(Vectors.dense(1.0, 2.0), Array(1, 2), 2))
.setInputCol("myInputCol")
.setOutputCol("myOutputCol")
- val newInstance = testDefaultReadWrite(instance)
- assert(newInstance.idf === instance.idf)
- assert(newInstance.docFreq === instance.docFreq)
- assert(newInstance.numDocs === instance.numDocs)
+
+ for (testSaveToLocal <- Seq(false, true)) {
+ val newInstance = testDefaultReadWrite(instance, testSaveToLocal = testSaveToLocal)
+ assert(newInstance.idf === instance.idf)
+ assert(newInstance.docFreq === instance.docFreq)
+ assert(newInstance.numDocs === instance.numDocs)
+ }
}
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
index 53be2444ecb60..58ddbebbe77d8 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.internal.config.Kryo._
import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.serializer.KryoSerializer
+import org.apache.spark.util.collection.Utils.createArray
class InstanceSuite extends SparkFunSuite {
test("Kryo class register") {
@@ -98,7 +99,7 @@ class InstanceSuite extends SparkFunSuite {
}
// instances larger than maxMemUsage
- val denseInstance = Instance(-1.0, 2.0, Vectors.dense(Array.fill(1000)(1.0)))
+ val denseInstance = Instance(-1.0, 2.0, Vectors.dense(createArray(1000, 1.0)))
InstanceBlock.blokifyWithMaxMemUsage(Iterator.single(denseInstance), 64).size
InstanceBlock.blokifyWithMaxMemUsage(Iterator.fill(10)(denseInstance), 64).size
@@ -109,7 +110,7 @@ class InstanceSuite extends SparkFunSuite {
// nnz = 10
val sparseInstance = Instance(-2.0, 3.0,
- Vectors.sparse(1000, Array.range(0, 1000, 100), Array.fill(10)(0.1)))
+ Vectors.sparse(1000, Array.range(0, 1000, 100), createArray(10, 0.1)))
// normally, memory usage of a block does not exceed maxMemUsage too much
val maxMemUsage = 1 << 18
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/RobustScalerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/RobustScalerSuite.scala
index 6f24b9db3b3eb..b3eb595b971aa 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/RobustScalerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/RobustScalerSuite.scala
@@ -22,6 +22,7 @@ import org.apache.spark.ml.param.ParamsSuite
import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
import org.apache.spark.ml.util.TestingUtils._
import org.apache.spark.sql.Row
+import org.apache.spark.util.collection.Utils.createArray
class RobustScalerSuite extends MLTest with DefaultReadWriteTest {
@@ -140,19 +141,19 @@ class RobustScalerSuite extends MLTest with DefaultReadWriteTest {
// 3st quartile = [3.0, ...]
// quantile range = IQR = [2.0, ...]
highDimData = Array(
- Vectors.dense(Array.fill(2000)(0.0)),
- Vectors.dense(Array.fill(2000)(1.0)),
- Vectors.dense(Array.fill(2000)(2.0)),
- Vectors.dense(Array.fill(2000)(3.0)),
- Vectors.dense(Array.fill(2000)(4.0))
+ Vectors.dense(createArray(2000, 0.0)),
+ Vectors.dense(createArray(2000, 1.0)),
+ Vectors.dense(createArray(2000, 2.0)),
+ Vectors.dense(createArray(2000, 3.0)),
+ Vectors.dense(createArray(2000, 4.0))
)
highDimRes = Array(
- Vectors.dense(Array.fill(2000)(0.0)),
- Vectors.dense(Array.fill(2000)(0.5)),
- Vectors.dense(Array.fill(2000)(1.0)),
- Vectors.dense(Array.fill(2000)(1.5)),
- Vectors.dense(Array.fill(2000)(2.0))
+ Vectors.dense(createArray(2000, 0.0)),
+ Vectors.dense(createArray(2000, 0.5)),
+ Vectors.dense(createArray(2000, 1.0)),
+ Vectors.dense(createArray(2000, 1.5)),
+ Vectors.dense(createArray(2000, 2.0))
)
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorIndexerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorIndexerSuite.scala
index 384fcf6ceb859..0c97e48079599 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorIndexerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorIndexerSuite.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.feature
import org.apache.spark.SparkException
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CATEGORICAL_FEATURES, MAX_CATEGORIES}
import org.apache.spark.ml.attribute._
import org.apache.spark.ml.linalg.{SparseVector, Vector, Vectors}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorSizeHintSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorSizeHintSuite.scala
index 14faa81f7bffa..ade7823326f42 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorSizeHintSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorSizeHintSuite.scala
@@ -21,7 +21,7 @@ import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.attribute.AttributeGroup
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest}
-import org.apache.spark.sql.execution.streaming.MemoryStream
+import org.apache.spark.sql.execution.streaming.runtime.MemoryStream
import org.apache.spark.sql.streaming.StreamTest
class VectorSizeHintSuite
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
index d15462f6f403f..0e38b782f5074 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
@@ -36,7 +36,7 @@ class Word2VecSuite extends MLTest with DefaultReadWriteTest {
}
test("Word2Vec") {
- val sentence = "a b " * 100 + "a c " * 10
+ val sentence = "a b ".repeat(100) + "a c ".repeat(10)
val numOfWords = sentence.split(" ").length
val doc = sc.parallelize(Seq(sentence, sentence)).map(line => line.split(" "))
@@ -76,7 +76,7 @@ class Word2VecSuite extends MLTest with DefaultReadWriteTest {
}
test("getVectors") {
- val sentence = "a b " * 100 + "a c " * 10
+ val sentence = "a b ".repeat(100) + "a c ".repeat(10)
val doc = sc.parallelize(Seq(sentence, sentence)).map(line => line.split(" "))
val docDF = doc.zip(doc).toDF("text", "alsotext")
@@ -106,7 +106,7 @@ class Word2VecSuite extends MLTest with DefaultReadWriteTest {
test("findSynonyms") {
- val sentence = "a b " * 100 + "a c " * 10
+ val sentence = "a b ".repeat(100) + "a c ".repeat(10)
val doc = sc.parallelize(Seq(sentence, sentence)).map(line => line.split(" "))
val docDF = doc.zip(doc).toDF("text", "alsotext")
@@ -138,7 +138,7 @@ class Word2VecSuite extends MLTest with DefaultReadWriteTest {
test("window size") {
- val sentence = "a q s t q s t b b b s t m s t m q " * 100 + "a c " * 10
+ val sentence = "a q s t q s t b b b s t m s t m q ".repeat(100) + "a c ".repeat(10)
val doc = sc.parallelize(Seq(sentence, sentence)).map(line => line.split(" "))
val docDF = doc.zip(doc).toDF("text", "alsotext")
diff --git a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index 4da67a92d7075..e6e15bb1b3487 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -24,11 +24,8 @@ import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
-import org.apache.commons.io.FileUtils
-import org.apache.commons.io.filefilter.TrueFileFilter
-
import org.apache.spark._
-import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{RMSE, TEST_SIZE, TRAINING_SIZE}
import org.apache.spark.ml.linalg.{BLAS, Vectors}
import org.apache.spark.ml.recommendation.ALS._
@@ -1027,13 +1024,7 @@ class ALSCleanerSuite extends SparkFunSuite with LocalRootDirsTest {
val conf = new SparkConf()
val localDir = Utils.createTempDir()
val checkpointDir = Utils.createTempDir()
- def getAllFiles: Set[File] = {
- val files = FileUtils.listFiles(
- localDir,
- TrueFileFilter.INSTANCE,
- TrueFileFilter.INSTANCE).asScala.toSet
- files
- }
+ def getAllFiles: Set[File] = Utils.listFiles(localDir).asScala.toSet
try {
conf.set("spark.local.dir", localDir.getAbsolutePath)
val sc = new SparkContext("local[2]", "ALSCleanerSuite", conf)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
index 6a0d7b1237ee4..e6650fa580135 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
@@ -18,9 +18,7 @@
package org.apache.spark.ml.source.libsvm
import java.io.{File, IOException}
-import java.nio.charset.StandardCharsets
-
-import com.google.common.io.Files
+import java.nio.file.Files
import org.apache.spark.SparkFunSuite
import org.apache.spark.ml.attribute.AttributeGroup
@@ -65,9 +63,9 @@ class LibSVMRelationSuite
val succ = new File(dir, "_SUCCESS")
val file0 = new File(dir, "part-00000")
val file1 = new File(dir, "part-00001")
- Files.asCharSink(succ, StandardCharsets.UTF_8).write("")
- Files.asCharSink(file0, StandardCharsets.UTF_8).write(lines0)
- Files.asCharSink(file1, StandardCharsets.UTF_8).write(lines1)
+ Files.writeString(succ.toPath, "")
+ Files.writeString(file0.toPath, lines0)
+ Files.writeString(file1.toPath, lines1)
path = dir.getPath
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala b/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala
index 18cce169b4ce9..a35b19b2816e4 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/util/MLTest.scala
@@ -28,7 +28,7 @@ import org.apache.spark.ml.attribute._
import org.apache.spark.ml.classification._
import org.apache.spark.ml.linalg._
import org.apache.spark.sql.{DataFrame, Dataset, Encoder, Row}
-import org.apache.spark.sql.execution.streaming.MemoryStream
+import org.apache.spark.sql.execution.streaming.runtime.MemoryStream
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.streaming.StreamTest
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/feature/PCASuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/feature/PCASuite.scala
index ea02f58ca906c..cf1f6005b4af9 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/feature/PCASuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/feature/PCASuite.scala
@@ -23,6 +23,7 @@ import org.apache.spark.mllib.linalg.distributed.RowMatrix
import org.apache.spark.mllib.util.MLlibTestSparkContext
import org.apache.spark.mllib.util.TestingUtils._
import org.apache.spark.util.ArrayImplicits._
+import org.apache.spark.util.collection.Utils.createArray
class PCASuite extends SparkFunSuite with MLlibTestSparkContext {
@@ -58,8 +59,8 @@ class PCASuite extends SparkFunSuite with MLlibTestSparkContext {
test("number of features more than 65535") {
val data1 = sc.parallelize(Seq(
- Vectors.dense(Array.fill(100000)(2.0)),
- Vectors.dense(Array.fill(100000)(0.0))
+ Vectors.dense(createArray(100000, 2.0)),
+ Vectors.dense(createArray(100000, 0.0))
), 2)
val pca = new PCA(2).fit(data1)
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/feature/Word2VecSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/feature/Word2VecSuite.scala
index 0b81a7c8aaf30..5e305262d6f40 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/feature/Word2VecSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/feature/Word2VecSuite.scala
@@ -29,7 +29,7 @@ class Word2VecSuite extends SparkFunSuite with MLlibTestSparkContext {
// TODO: add more tests
test("Word2Vec") {
- val sentence = "a b " * 100 + "a c " * 10
+ val sentence = "a b ".repeat(100) + "a c ".repeat(10)
val localDoc = Seq(sentence, sentence)
val doc = sc.parallelize(localDoc)
.map(line => line.split(" ").toSeq)
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
index 37bbe76b9b531..22c514a0a0170 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
@@ -19,6 +19,7 @@ package org.apache.spark.mllib.linalg
import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.util.collection.Utils.createArray
/**
* Serialization benchmark for VectorUDT.
@@ -45,7 +46,7 @@ object UDTSerializationBenchmark extends BenchmarkBase {
val fromRow = encoder.createDeserializer()
val vectors = (1 to numRows).map { i =>
- Vectors.dense(Array.fill(1e5.toInt)(1.0 * i))
+ Vectors.dense(createArray(1e5.toInt, 1.0 * i))
}.toArray
val rows = vectors.map(toRow)
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
index 135d7e26c6d8c..5f681317a58b6 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
@@ -26,7 +26,6 @@ import org.json4s.jackson.JsonMethods.{parse => parseJson}
import org.apache.spark.{SparkConf, SparkException, SparkFunSuite}
import org.apache.spark.internal.LogKeys.MALFORMATTED_STRING
-import org.apache.spark.internal.MDC
import org.apache.spark.internal.config.Kryo._
import org.apache.spark.ml.{linalg => newlinalg}
import org.apache.spark.mllib.util.TestingUtils._
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/tree/GradientBoostedTreesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/tree/GradientBoostedTreesSuite.scala
index e654eac83649c..4240da825ce57 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/tree/GradientBoostedTreesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/tree/GradientBoostedTreesSuite.scala
@@ -18,8 +18,8 @@
package org.apache.spark.mllib.tree
import org.apache.spark.SparkFunSuite
-import org.apache.spark.internal.{MDC, MessageWithContext}
import org.apache.spark.internal.LogKeys.{LEARNING_RATE, NUM_ITERATIONS, SUBSAMPLING_RATE}
+import org.apache.spark.internal.MessageWithContext
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.tree.configuration.{BoostingStrategy, Strategy}
import org.apache.spark.mllib.tree.configuration.Algo._
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/util/MLUtilsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/util/MLUtilsSuite.scala
index c4dc3b1715720..5f6ac1b716cc9 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/util/MLUtilsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/util/MLUtilsSuite.scala
@@ -18,12 +18,10 @@
package org.apache.spark.mllib.util
import java.io.File
-import java.nio.charset.StandardCharsets
+import java.nio.file.Files
import scala.io.Source
-import com.google.common.io.Files
-
import org.apache.spark.{SparkException, SparkFunSuite, SparkRuntimeException}
import org.apache.spark.mllib.linalg.{DenseVector, Matrices, SparseVector, Vector, Vectors}
import org.apache.spark.mllib.regression.LabeledPoint
@@ -93,7 +91,7 @@ class MLUtilsSuite extends SparkFunSuite with MLlibTestSparkContext {
""".stripMargin
val tempDir = Utils.createTempDir()
val file = new File(tempDir.getPath, "part-00000")
- Files.asCharSink(file, StandardCharsets.UTF_8).write(lines)
+ Files.writeString(file.toPath, lines)
val path = tempDir.toURI.toString
val pointsWithNumFeatures = loadLibSVMFile(sc, path, 6).collect()
@@ -126,7 +124,7 @@ class MLUtilsSuite extends SparkFunSuite with MLlibTestSparkContext {
""".stripMargin
val tempDir = Utils.createTempDir()
val file = new File(tempDir.getPath, "part-00000")
- Files.asCharSink(file, StandardCharsets.UTF_8).write(lines)
+ Files.writeString(file.toPath, lines)
val path = tempDir.toURI.toString
intercept[SparkException] {
@@ -143,7 +141,7 @@ class MLUtilsSuite extends SparkFunSuite with MLlibTestSparkContext {
""".stripMargin
val tempDir = Utils.createTempDir()
val file = new File(tempDir.getPath, "part-00000")
- Files.asCharSink(file, StandardCharsets.UTF_8).write(lines)
+ Files.writeString(file.toPath, lines)
val path = tempDir.toURI.toString
intercept[SparkException] {
diff --git a/pom.xml b/pom.xml
index 2ad0603357a21..e6cb98665d109 100644
--- a/pom.xml
+++ b/pom.xml
@@ -82,6 +82,7 @@
common/network-shuffle
common/unsafe
common/utils
+ common/utils-java
common/variant
common/tags
sql/connect/shims
@@ -116,20 +117,21 @@
UTF-8
UTF-8
17
+ 17.0.11
${java.version}
- 3.9.10
- 3.5.0
+ 3.9.11
+ 3.5.1
spark
9.8
2.0.17
- 2.24.3
+ 2.25.1
- 3.4.1
+ 3.4.2
4.29.3
3.11.4
3.9.3
- 5.7.1
+ 5.9.0
org.apache.hive
core
@@ -139,7 +141,7 @@
10.16.1.1
1.15.2
- 2.1.2
+ 2.2.0
shaded-protobuf
11.0.25
5.0.0
@@ -153,7 +155,7 @@
If you change codahale.metrics.version, you also need to change
the link to metrics.dropwizard.io in docs/monitoring.md.
-->
- 4.2.30
+ 4.2.33
1.12.0
1.12.0
@@ -163,7 +165,7 @@
0.12.8
- hadoop3-2.2.26
+ hadoop3-2.2.28
4.5.14
4.4.16
@@ -174,35 +176,33 @@
2.13.16
2.13
2.2.0
- 4.9.2
+ 4.9.5
false
2.18.0
true
true
true
- 1.9.13
- 2.19.0
- 2.19.0
+ 2.19.2
+ 2.19.2
2.3.1
- 1.1.10.7
- 3.0.3
- 1.18.0
- 1.27.1
- 2.19.0
+ 1.1.10.8
+ 3.0.4
+ 1.19.0
+ 1.28.0
+ 2.20.0
2.6
- 3.17.0
+ 3.18.0
2.12.1
4.1.17
33.4.0-jre
2.11.0
3.1.9
- 3.0.17
- 2.13.0
- 3.5.2
+ 3.0.18
+ 2.14.0
3.0.0
2.2.11
0.16.0
@@ -212,27 +212,27 @@
4.21.0
3.1.0
1.1.0
- 1.9.0
- 1.80
+ 1.10.0
+ 1.81
1.16.0
6.2.0
- 4.1.121.Final
- 2.0.70.Final
+ 4.1.124.Final
+ 2.0.72.Final
77.1
- 5.12.2
- 1.12.2
+ 5.13.1
+ 1.13.1
- 0.14.0
+ 0.15.0
18.3.0
- 3.0.1
+ 3.0.2
0.12.6
@@ -281,7 +281,6 @@
compile
2.8.1
compile
- compile
compile
test
compile
@@ -310,7 +309,7 @@
6.0.53
- 3.0-9
+ 4.0-10
128m
@@ -340,7 +339,7 @@
2.7.12
9.2.0
- 42.7.5
+ 42.7.7
11.5.9.0
12.8.1.jre11
23.6.0.24.10
@@ -518,12 +517,12 @@