Skip to content

[MINOR] Refactor operator-enable configs and remove dead code #12212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
philo-he wants to merge 2 commits into
base: main
Choose a base branch
from
+15 −30
Open

[MINOR] Refactor operator-enable configs and remove dead code #12212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f3cf8f0
Initial
philo-he Jun 1, 2026
80915bd
Fix comment
philo-he Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,8 +363,6 @@ object CHBackendSettings extends BackendSettingsApi with Logging {

override def structFieldToLowerCase(): Boolean = false

override def supportExpandExec(): Boolean = true

override def excludeScanExecFromCollapsedStage(): Boolean =
SQLConf.get
.getConfString(GLUTEN_CLICKHOUSE_SEP_SCAN_RDD, GLUTEN_CLICKHOUSE_SEP_SCAN_RDD_DEFAULT)
Expand Down
2 changes: 1 addition & 1 deletion ...lickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ case class FallbackBroadcastHashJoin(session: SparkSession) extends Rule[SparkPl
GlutenConfig.get.enableColumnarBroadcastExchange

private val enableColumnarBroadcastNestedLoopJoin: Boolean =
GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled &&
GlutenConfig.get.enableColumnarBroadcastNestedLoopJoin &&
GlutenConfig.get.enableColumnarBroadcastExchange

override def apply(plan: SparkPlan): SparkPlan = {
Expand Down
2 changes: 0 additions & 2 deletions backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,8 +397,6 @@ object VeloxBackendSettings extends BackendSettingsApi {
}
}

override def supportExpandExec(): Boolean = true

override def supportSortExec(): Boolean = true

override def supportSortMergeJoinExec(): Boolean = {
Expand Down
4 changes: 2 additions & 2 deletions docs/Configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@ nav_order: 15
| spark.gluten.soft-affinity.replications.num | 🔄 Dynamic | 2 | Calculate the number of the replications for scheduling to the target executors per file |
| spark.gluten.sql.adaptive.costEvaluator.enabled | ⚓ Static | true | If true, use org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator as custom cost evaluator class, else follow the configuration spark.sql.adaptive.customCostEvaluatorClass. |
| spark.gluten.sql.ansiFallback.enabled | 🔄 Dynamic | true | When true (default), Gluten will fall back to Spark when ANSI mode is enabled. When false, Gluten will attempt to execute in ANSI mode. |
| spark.gluten.sql.broadcastNestedLoopJoinTransformerEnabled | 🔄 Dynamic | true | Config to enable BroadcastNestedLoopJoinExecTransformer. |
| spark.gluten.sql.cacheWholeStageTransformerContext | 🔄 Dynamic | false | When true, `WholeStageTransformer` will cache the `WholeStageTransformerContext` when executing. It is used to get substrait plan node and native plan string. |
| spark.gluten.sql.cartesianProductTransformerEnabled | 🔄 Dynamic | true | Config to enable CartesianProductExecTransformer. |
| spark.gluten.sql.collapseGetJsonObject.enabled | 🔄 Dynamic | false | Collapse nested get_json_object functions as one for optimization. |
| spark.gluten.sql.columnar.appendData | 🔄 Dynamic | true | Enable or disable columnar v2 command append data. |
| spark.gluten.sql.columnar.arrowUdf | 🔄 Dynamic | true | Enable or disable columnar arrow udf. |
| spark.gluten.sql.columnar.batchscan | 🔄 Dynamic | true | Enable or disable columnar batchscan. |
| spark.gluten.sql.columnar.broadcastExchange | 🔄 Dynamic | true | Enable or disable columnar broadcastExchange. |
| spark.gluten.sql.columnar.broadcastJoin | 🔄 Dynamic | true | Enable or disable columnar broadcastJoin. |
| spark.gluten.sql.columnar.broadcastNestedLoopJoin.enabled | 🔄 Dynamic | true | Enable or disable columnar broadcastNestedLoopJoin. |
| spark.gluten.sql.columnar.cartesianProduct.enabled | 🔄 Dynamic | true | Enable or disable columnar cartesianProduct. |
| spark.gluten.sql.columnar.cast.avg | 🔄 Dynamic | true |
| spark.gluten.sql.columnar.coalesce | 🔄 Dynamic | true | Enable or disable columnar coalesce. |
| spark.gluten.sql.columnar.collectLimit | 🔄 Dynamic | true | Enable or disable columnar collectLimit. |
Expand Down
2 changes: 0 additions & 2 deletions gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ trait BackendSettingsApi {

def supportNativeRowIndexColumn(): Boolean = true

def supportExpandExec(): Boolean = false

def supportSortExec(): Boolean = false

def supportSortMergeJoinExec(): Boolean = true
Expand Down
20 changes: 10 additions & 10 deletions gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,11 +285,11 @@ class GlutenConfig(conf: SQLConf) extends GlutenCoreConfig(conf) {

def fallbackPreferColumnar: Boolean = getConf(COLUMNAR_FALLBACK_PREFER_COLUMNAR)

def cartesianProductTransformerEnabled: Boolean =
getConf(CARTESIAN_PRODUCT_TRANSFORMER_ENABLED)
def enableColumnarCartesianProduct: Boolean =
getConf(COLUMNAR_CARTESIAN_PRODUCT_ENABLED)

def broadcastNestedLoopJoinTransformerTransformerEnabled: Boolean =
getConf(BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED)
def enableColumnarBroadcastNestedLoopJoin: Boolean =
getConf(COLUMNAR_BROADCAST_NESTED_LOOP_JOIN_ENABLED)

def transformPlanLogLevel: String = getConf(TRANSFORM_PLAN_LOG_LEVEL)

Expand Down Expand Up @@ -1462,15 +1462,15 @@ object GlutenConfig extends ConfigRegistry {
.booleanConf
.createWithDefault(true)

val CARTESIAN_PRODUCT_TRANSFORMER_ENABLED =
buildConf("spark.gluten.sql.cartesianProductTransformerEnabled")
.doc("Config to enable CartesianProductExecTransformer.")
val COLUMNAR_CARTESIAN_PRODUCT_ENABLED =
buildConf("spark.gluten.sql.columnar.cartesianProduct.enabled")
.doc("Enable or disable columnar cartesianProduct.")
.booleanConf
.createWithDefault(true)

val BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED =
buildConf("spark.gluten.sql.broadcastNestedLoopJoinTransformerEnabled")
.doc("Config to enable BroadcastNestedLoopJoinExecTransformer.")
val COLUMNAR_BROADCAST_NESTED_LOOP_JOIN_ENABLED =
buildConf("spark.gluten.sql.columnar.broadcastNestedLoopJoin.enabled")
.doc("Enable or disable columnar broadcastNestedLoopJoin.")
.booleanConf
.createWithDefault(true)

Expand Down
6 changes: 0 additions & 6 deletions ...t/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package org.apache.gluten.execution

import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.metrics.MetricsUpdater
import org.apache.gluten.substrait.{JoinParams, SubstraitContext}
import org.apache.gluten.utils.SubstraitUtil
Expand Down Expand Up @@ -167,11 +166,6 @@ abstract class BroadcastNestedLoopJoinExecTransformer(
}

override protected def doValidateInternal(): ValidationResult = {
if (!GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled) {
return ValidationResult.failed(
s"Config ${GlutenConfig.BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED.key} not enabled")
}

if (substraitJoinType == CrossRel.JoinType.UNRECOGNIZED) {
return ValidationResult.failed(
s"$joinType join is not supported with BroadcastNestedLoopJoin")
Expand Down
3 changes: 0 additions & 3 deletions gluten-substrait/src/main/scala/org/apache/gluten/execution/ExpandExecTransformer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,6 @@ case class ExpandExecTransformer(
}

override protected def doValidateInternal(): ValidationResult = {
if (!BackendsApiManager.getSettings.supportExpandExec()) {
return ValidationResult.failed("Current backend does not support expand")
}
if (projections.isEmpty) {
return ValidationResult.failed("Current backend does not support empty projections in expand")
}
Expand Down
4 changes: 2 additions & 2 deletions ...-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ object Validators {
fail(p)
case p: GenerateExec if !glutenConf.enableColumnarGenerate => fail(p)
case p: CoalesceExec if !glutenConf.enableColumnarCoalesce => fail(p)
case p: CartesianProductExec if !glutenConf.cartesianProductTransformerEnabled => fail(p)
case p: CartesianProductExec if !glutenConf.enableColumnarCartesianProduct => fail(p)
case p: TakeOrderedAndProjectExec
if !(glutenConf.enableTakeOrderedAndProject && glutenConf.enableColumnarSort &&
glutenConf.enableColumnarShuffle && glutenConf.enableColumnarProject) =>
Expand All @@ -198,7 +198,7 @@ object Validators {
fail(p)
case p: BroadcastNestedLoopJoinExec
if !(glutenConf.enableColumnarBroadcastJoin &&
glutenConf.broadcastNestedLoopJoinTransformerTransformerEnabled) =>
glutenConf.enableColumnarBroadcastNestedLoopJoin) =>
fail(p)
case p @ (_: HashAggregateExec | _: SortAggregateExec | _: ObjectHashAggregateExec)
if !glutenConf.enableColumnarHashAgg =>
Expand Down
Loading