diff --git a/.github/workflows/velox_backend_enhanced.yml b/.github/workflows/velox_backend_enhanced.yml index a9e31d41f54..c51e9b777c0 100644 --- a/.github/workflows/velox_backend_enhanced.yml +++ b/.github/workflows/velox_backend_enhanced.yml @@ -200,6 +200,10 @@ jobs: spark-test-spark40: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3] env: SPARK_TESTING: true container: apache/gluten:centos-8-jdk17 @@ -221,7 +225,7 @@ jobs: run: | rm -rf /opt/shims/spark40 bash .github/workflows/util/install-spark-resources.sh 4.0 - - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other tests) + - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 @@ -229,20 +233,31 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta \ - -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta \ + -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta \ + -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta \ + -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark40-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark40-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 9b13dbd069a..d6fa89edf14 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -472,6 +472,7 @@ jobs: fail-fast: false matrix: spark: [ "spark-3.3" ] + shard: [ 1, 2, 3 ] runs-on: ubuntu-22.04 steps: - name: Maximize build disk space @@ -501,17 +502,17 @@ jobs: echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - name: Build for Spark ${{ matrix.spark }} run: | - cd $GITHUB_WORKSPACE/ + cd $GITHUB_WORKSPACE/ $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests cd $GITHUB_WORKSPACE/tools/gluten-it $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - - name: TPC-DS SF30.0 Parquet local spark3.3 random kill tasks + - name: TPC-DS SF30.0 Parquet local spark3.3 random kill tasks (shard ${{ matrix.shard }}/3) run: | cd tools/gluten-it \ && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries \ --local --preset=velox --benchmark-type=ds --error-on-memleak -s=30.0 --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \ - --data-gen=skip --random-kill-tasks --no-session-reuse + --data-gen=skip --random-kill-tasks --no-session-reuse --shard=${{ matrix.shard }}/3 tpc-test-centos8-uniffle: needs: build-native-lib-centos-7 @@ -737,6 +738,10 @@ jobs: spark-test-spark34: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3] container: apache/gluten:centos-8-jdk8 steps: - uses: actions/checkout@v4 @@ -752,7 +757,7 @@ jobs: pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.4.4 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.4.4 (other tests) + - name: Build and Run unit test for Spark 3.4.4 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 @@ -761,21 +766,31 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ - ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ - -DargLine="-Dspark.test.home=$SPARK_HOME" + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToExclude="$TAGS_EXCLUDE" -DargLine="-Dspark.test.home=$SPARK_HOME" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToExclude="$TAGS_EXCLUDE" -DargLine="-Dspark.test.home=$SPARK_HOME" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToExclude="$TAGS_EXCLUDE" -DargLine="-Dspark.test.home=$SPARK_HOME" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark34-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark34-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -784,12 +799,16 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-golden-files + name: spark-test-spark34-group${{ matrix.group }}-golden-files path: /tmp/tpch-approved-plan/** spark-test-spark34-slow: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + suite: [extended, slow-hive] env: #TODO remove after image update LANG: C.UTF-8 LC_ALL: C.UTF-8 @@ -805,7 +824,7 @@ jobs: run: | rm -rf /opt/shims/spark34 bash .github/workflows/util/install-spark-resources.sh 3.4 - - name: Build and Run unit test for Spark 3.4.4 (slow tests) + - name: Build and Run unit test for Spark 3.4.4 (slow tests, ${{ matrix.suite }}) run: | cd $GITHUB_WORKSPACE/ yum install -y java-17-openjdk-devel @@ -813,24 +832,26 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ - ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ - -DargLine="-Dspark.test.home=$SPARK_HOME" - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest \ - -DargLine="-Dspark.test.home=$SPARK_HOME" + if [ "${{ matrix.suite }}" = "extended" ]; then + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ + -DargLine="-Dspark.test.home=$SPARK_HOME" + else + $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DtagsToInclude=org.apache.spark.tags.SlowHiveTest \ + -DargLine="-Dspark.test.home=$SPARK_HOME" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark34-slow-${{ matrix.suite }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark34-slow-${{ matrix.suite }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -839,6 +860,10 @@ jobs: spark-test-spark35: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3] env: SPARK_TESTING: true container: apache/gluten:centos-8-jdk8 @@ -856,7 +881,7 @@ jobs: pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.5.5 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.5.5 (other tests) + - name: Build and Run unit test for Spark 3.5.5 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 @@ -864,20 +889,31 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark35-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark35-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -886,12 +922,16 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-golden-files + name: spark-test-spark35-group${{ matrix.group }}-golden-files path: /tmp/tpch-approved-plan/** spark-test-spark35-scala213: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3] env: SPARK_TESTING: true container: apache/gluten:centos-8-jdk8 @@ -909,7 +949,7 @@ jobs: pip3 install setuptools==77.0.3 && \ pip3 install pyspark==3.5.5 cython && \ pip3 install pandas==2.2.3 pyarrow==20.0.0 - - name: Build and Run unit test for Spark 3.5.5 with scala-2.13 (other tests) + - name: Build and Run unit test for Spark 3.5.5 with scala-2.13 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 @@ -917,20 +957,31 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg \ - -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark35-scala213-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark35-scala213-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -939,6 +990,10 @@ jobs: spark-test-spark35-slow: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + suite: [extended, slow-hive] env: SPARK_TESTING: true LANG: C.UTF-8 #TODO remove after image update @@ -955,30 +1010,33 @@ jobs: run: | rm -rf /opt/shims/spark35 bash .github/workflows/util/install-spark-resources.sh 3.5 - - name: Build and Run unit test for Spark 3.5.5 (slow tests) + - name: Build and Run unit test for Spark 3.5.5 (slow tests, ${{ matrix.suite }}) run: | cd $GITHUB_WORKSPACE/ yum install -y java-17-openjdk-devel export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + if [ "${{ matrix.suite }}" = "extended" ]; then + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + else + $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark35-slow-${{ matrix.suite }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark35-slow-${{ matrix.suite }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -1130,6 +1188,10 @@ jobs: spark-test-spark40: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3] env: SPARK_TESTING: true container: apache/gluten:centos-8-jdk17 @@ -1151,7 +1213,7 @@ jobs: run: | rm -rf /opt/shims/spark40 bash .github/workflows/util/install-spark-resources.sh 4.0 - - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other tests) + - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 @@ -1159,20 +1221,31 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon \ - -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark40-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark40-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -1181,6 +1254,10 @@ jobs: spark-test-spark40-slow: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + suite: [extended, slow-hive] env: SPARK_TESTING: true LANG: C.UTF-8 #TODO remove after image update @@ -1197,30 +1274,33 @@ jobs: run: | rm -rf /opt/shims/spark40 bash .github/workflows/util/install-spark-resources.sh 4.0 - - name: Build and Run unit test for Spark 4.0 (slow tests) + - name: Build and Run unit test for Spark 4.0 (slow tests, ${{ matrix.suite }}) run: | cd $GITHUB_WORKSPACE/ yum install -y java-17-openjdk-devel export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + if [ "${{ matrix.suite }}" = "extended" ]; then + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Ppaimon -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + else + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pdelta -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark40-slow-${{ matrix.suite }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark40-slow-${{ matrix.suite }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -1229,6 +1309,14 @@ jobs: spark-test-spark41: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + # Split tests into 3 groups to run in parallel and cut the ~2h wall-clock time. + # group1 – streaming tests + # group2 – execution / catalyst / errors / extension tests + # group3 – top-level sql, connector, sources, hive and remaining tests + group: [1, 2, 3] env: SPARK_TESTING: true container: apache/gluten:centos-9-jdk17 @@ -1252,7 +1340,7 @@ jobs: run: | rm -rf /opt/shims/spark41 bash .github/workflows/util/install-spark-resources.sh 4.1 - - name: Build and Run unit test for Spark 4.1.0 with scala-2.13 (other tests) + - name: Build and Run unit test for Spark 4.1.0 with scala-2.13 (other tests, group ${{ matrix.group }}) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 @@ -1260,20 +1348,34 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ - -Pspark-ut -Pdelta -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ - -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest + TAGS_EXCLUDE="org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowHiveTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest" + if [ "${{ matrix.group }}" = "1" ]; then + # Group 1: streaming + gluten utils + top-level spark tests (~55 classes) + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut -Pdelta \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.streaming,org.apache.spark.GlutenSortShuffleSuite,org.apache.gluten" + elif [ "${{ matrix.group }}" = "2" ]; then + # Group 2: execution + catalyst + errors + extension tests (~140 classes) + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut -Pdelta \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.execution,org.apache.spark.sql.catalyst,org.apache.spark.sql.errors,org.apache.spark.sql.extension" + else + # Group 3: top-level sql, connector, sources, hive and remaining tests (~205 classes) + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut -Pdelta \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" -DtagsToExclude="$TAGS_EXCLUDE" \ + -DwildcardSuites="org.apache.spark.sql.GlutenSQL,org.apache.spark.sql.Gluten,org.apache.spark.sql.connector,org.apache.spark.sql.sources,org.apache.spark.sql.hive,org.apache.spark.sql.gluten,org.apache.spark.sql.shim" + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark41-group${{ matrix.group }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark41-group${{ matrix.group }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log @@ -1282,6 +1384,11 @@ jobs: spark-test-spark41-slow: needs: build-native-lib-centos-7 runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + # Run ExtendedSQLTest and SlowHiveTest in parallel instead of sequentially. + suite: [extended, slow-hive] env: SPARK_TESTING: true LANG: C.UTF-8 #TODO remove after image update @@ -1298,30 +1405,33 @@ jobs: run: | rm -rf /opt/shims/spark41 bash .github/workflows/util/install-spark-resources.sh 4.1 - - name: Build and Run unit test for Spark 4.1 (slow tests) + - name: Build and Run unit test for Spark 4.1 (slow tests, ${{ matrix.suite }}) run: | cd $GITHUB_WORKSPACE/ yum install -y java-17-openjdk-devel export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ - -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ - -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + if [ "${{ matrix.suite }}" = "extended" ]; then + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest + else + $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ + -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ + -DtagsToInclude=org.apache.spark.tags.SlowHiveTest + fi - name: Upload test report if: always() uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-report + name: spark-test-spark41-slow-${{ matrix.suite }}-report path: '**/surefire-reports/TEST-*.xml' - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: - name: ${{ github.job }}-test-log + name: spark-test-spark41-slow-${{ matrix.suite }}-test-log path: | **/target/*.log **/gluten-ut/**/hs_err_*.log diff --git a/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala b/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala index ab73aa40fa0..96749057fe8 100644 --- a/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala +++ b/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala @@ -35,7 +35,7 @@ class GlobalOffHeapMemoryTarget private[memory] with Logging { private val targetName = MemoryTargetUtil.toUniqueName("GlobalOffHeap") private val recorder: MemoryUsageRecorder = new SimpleMemoryUsageRecorder() - private val mode: MemoryMode = { + private def mode: MemoryMode = { val enabled = Option(SparkEnv.get) .map(_.conf.getBoolean(GlutenCoreConfig.DYNAMIC_OFFHEAP_SIZING_ENABLED.key, false)) .getOrElse(false)