Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ class VeloxColumnarCacheSuite extends VeloxWholeStageTransformerSuite with Adapt
}
}

testWithMinSparkVersion("input row", "3.2") {
test("input row") {
withTable("t") {
sql("CREATE TABLE t USING json AS SELECT * FROM values(1, 'a', (2, 'b'), (3, 'c'))")
runQueryAndCompare("SELECT * FROM t", cache = true) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

// NameMapping is supported in Delta 2.0 (related to Spark3.2.0)
testWithMinSparkVersion("column mapping mode = name", "3.2") {
test("column mapping mode = name") {
Comment thread
LuciferYang marked this conversation as resolved.
withTable("delta_cm2") {
spark.sql(s"""
|create table delta_cm2 (id int, name string) using delta
Expand Down Expand Up @@ -110,9 +109,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
// broke `PreparedDeltaFileIndex.matchingFiles` and silently returned all files.
Seq("name", "id").foreach {
mode =>
testWithMinSparkVersion(
s"column mapping mode = $mode with partition filter (single partition col)",
"3.2") {
test(s"column mapping mode = $mode with partition filter (single partition col)") {
withTable("delta_cm_part") {
spark.sql(s"""
|create table delta_cm_part (id int, name string) using delta
Expand Down Expand Up @@ -153,9 +150,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion(
s"column mapping mode = $mode with partition filter (multi partition col)",
"3.2") {
test(s"column mapping mode = $mode with partition filter (multi partition col)") {
withTable("delta_cm_part_multi") {
spark.sql(s"""
|create table delta_cm_part_multi
Expand Down Expand Up @@ -185,9 +180,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion(
s"column mapping mode = $mode with partition + data filter",
"3.2") {
test(s"column mapping mode = $mode with partition + data filter") {
withTable("delta_cm_part_data") {
spark.sql(s"""
|create table delta_cm_part_data (id int, name string, age int)
Expand Down Expand Up @@ -220,9 +213,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion(
s"column mapping mode = $mode with IS [NOT] NULL on partition col",
"3.2") {
test(s"column mapping mode = $mode with IS [NOT] NULL on partition col") {
withTable("delta_cm_part_null") {
spark.sql(s"""
|create table delta_cm_part_null (id int, name string)
Expand All @@ -247,9 +238,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion(
s"column mapping mode = $mode partition filter survives column rename",
"3.2") {
test(s"column mapping mode = $mode partition filter survives column rename") {
withTable("delta_cm_part_rename") {
spark.sql(s"""
|create table delta_cm_part_rename (id int, name string)
Expand All @@ -272,9 +261,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion(
s"column mapping mode = $mode data column rename + filter (file skipping)",
"3.2") {
test(s"column mapping mode = $mode data column rename + filter (file skipping)") {
withTable("delta_cm_data_rename") {
spark.sql(s"""
|create table delta_cm_data_rename (id int, age int, name string)
Expand Down Expand Up @@ -324,7 +311,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion("delta: partition filters", "3.2") {
test("delta: partition filters") {
withTable("delta_pf") {
spark.sql(s"""
|create table delta_pf (id int, name string) using delta partitioned by (name)
Expand All @@ -343,7 +330,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion("basic test with stats.skipping disabled", "3.2") {
test("basic test with stats.skipping disabled") {
withTable("delta_test2") {
withSQLConf("spark.databricks.delta.stats.skipping" -> "false") {
spark.sql(s"""
Expand All @@ -363,7 +350,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion("column mapping with complex type", "3.2") {
test("column mapping with complex type") {
withTable("t1") {
val simpleNestedSchema = new StructType()
.add("a", StringType, true)
Expand Down Expand Up @@ -440,7 +427,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion("delta: push down input_file_name expression", "3.2") {
test("delta: push down input_file_name expression") {
withTable("source_table") {
withTable("target_table") {
spark.sql(s"""
Expand Down Expand Up @@ -478,7 +465,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion("delta: need to validate delta expression before execution", "3.2") {
test("delta: need to validate delta expression before execution") {
withTable("source_table") {
withTable("target_table") {
spark.sql(s"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ abstract class HudiSuite extends WholeStageTransformerSuite {
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
}

testWithMinSparkVersion("hudi: time travel", "3.2") {
test("hudi: time travel") {
withTable("hudi_tm") {
spark.sql(s"""
|create table hudi_tm (id int, name string) using hudi
Expand All @@ -65,7 +65,7 @@ abstract class HudiSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion("hudi: soft delete", "3.2") {
test("hudi: soft delete") {
withTable("hudi_pf") {
spark.sql(s"""
|create table hudi_pf (id int, name string) using hudi
Expand Down Expand Up @@ -111,7 +111,7 @@ abstract class HudiSuite extends WholeStageTransformerSuite {
}
}

testWithMinSparkVersion("hudi: partition filters", "3.2") {
test("hudi: partition filters") {
withTable("hudi_pf") {
spark.sql(s"""
|create table hudi_pf (id int, name string) using hudi partitioned by (name)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class GlutenExpressionMappingSuite
conf
}

testWithMinSparkVersion("test expression blacklist", "3.2") {
test("test expression blacklist") {
val names = ExpressionMappings.expressionsMap.values.toSet
assert(names.contains("regexp_replace"))
assert(names.contains("regexp_extract"))
Expand All @@ -68,7 +68,7 @@ class GlutenExpressionMappingSuite
}
}

testWithMinSparkVersion("test blacklisting regexp expressions", "3.2") {
test("test blacklisting regexp expressions") {
val names = ExpressionMappings.expressionsMap.values.toSet
assert(names.contains("rlike"))
assert(names.contains("regexp_replace"))
Expand Down
Loading