diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index 2cc8b18fbfc..fcf1fbbbf77 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -20,6 +20,7 @@ import org.apache.gluten.config.{GlutenConfig, GlutenCoreConfig, VeloxConfig} import org.apache.gluten.expression.VeloxDummyExpression import org.apache.spark.SparkConf +import org.apache.spark.shuffle.GlutenShuffleUtils import org.apache.spark.sql.{DataFrame, Row} import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AQEShuffleReadExec, ShuffleQueryStageExec} @@ -2250,4 +2251,30 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa } } } + + test("GLUTEN-11539: unsupported spark.io.compression.codec throws with actionable message") { + val conf = spark.sparkContext.getConf.clone().set("spark.io.compression.codec", "snappy") + val ex = intercept[IllegalArgumentException] { + GlutenShuffleUtils.getCompressionCodec(conf) + } + assert(ex.getMessage.contains("snappy is not supported")) + assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key)) + } + + test("GLUTEN-11539: spark.io.compression.codec=none throws with actionable message") { + val conf = spark.sparkContext.getConf.clone().set("spark.io.compression.codec", "none") + val ex = intercept[IllegalArgumentException] { + GlutenShuffleUtils.getCompressionCodec(conf) + } + assert(ex.getMessage.contains("none is not supported")) + assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key)) + } + + test("GLUTEN-11539: supported spark.io.compression.codec is accepted") { + Seq("lz4", "zstd").foreach { + codec => + val conf = spark.sparkContext.getConf.clone().set("spark.io.compression.codec", codec) + assert(GlutenShuffleUtils.getCompressionCodec(conf) === codec) + } + } } diff --git a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala index 213b2831f46..4bd89582c18 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala @@ -74,10 +74,14 @@ object GlutenShuffleUtils { conf .get(sparkCodecKey, IO_COMPRESSION_CODEC.defaultValueString) .toLowerCase(Locale.ROOT) - checkCodecValues( - sparkCodecKey, - codec, - BackendsApiManager.getSettings.shuffleSupportedCodec()) + val supportedCodecs = BackendsApiManager.getSettings.shuffleSupportedCodec() + if (!supportedCodecs.contains(codec)) { + throw new IllegalArgumentException( + s"Gluten shuffle only supports ${supportedCodecs.mkString(", ")}. " + + s"$codec is not supported. " + + s"You may configure ${GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key} " + + s"to ${supportedCodecs.mkString(" or ")}.") + } codec } }