From 35329bd24010184b2979a7e7e16da2aec1ccf05b Mon Sep 17 00:00:00 2001 From: Boxuan Li Date: Mon, 19 May 2025 22:26:24 -0700 Subject: [PATCH] Strengthen data source v2 operators' equality checks --- .../spark/sql/execution/datasources/v2/BatchScanExec.scala | 4 ++-- .../sql/execution/datasources/v2/ContinuousScanExec.scala | 2 +- .../sql/execution/datasources/v2/MicroBatchScanExec.scala | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala index 6a502a44fad58..44776f9e66c84 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala @@ -48,8 +48,8 @@ case class BatchScanExec( // TODO: unify the equal/hashCode implementation for all data source v2 query plans. override def equals(other: Any): Boolean = other match { case other: BatchScanExec => - this.batch != null && this.batch == other.batch && - this.runtimeFilters == other.runtimeFilters && + this.getClass == other.getClass && this.batch != null && + this.batch == other.batch && this.runtimeFilters == other.runtimeFilters && this.spjParams == other.spjParams case _ => false diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ContinuousScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ContinuousScanExec.scala index 288233e691453..f9a6792dc9014 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ContinuousScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ContinuousScanExec.scala @@ -38,7 +38,7 @@ case class ContinuousScanExec( // TODO: unify the equal/hashCode implementation for all data source v2 query plans. override def equals(other: Any): Boolean = other match { - case other: ContinuousScanExec => this.stream == other.stream + case other: ContinuousScanExec => this.getClass == other.getClass && this.stream == other.stream case _ => false } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MicroBatchScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MicroBatchScanExec.scala index f81ca001fbe29..0d856a367b0b2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MicroBatchScanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MicroBatchScanExec.scala @@ -41,7 +41,7 @@ case class MicroBatchScanExec( // TODO: unify the equal/hashCode implementation for all data source v2 query plans. override def equals(other: Any): Boolean = other match { - case other: MicroBatchScanExec => this.stream == other.stream + case other: MicroBatchScanExec => this.getClass == other.getClass && this.stream == other.stream case _ => false }