Skip to content

Commit 774874e

Browse files
wangzhigang1999wangzhigang1999
authored
Merge branch 'apache:master' into branch-spark-engine-terminator
2 parents f89fc23 + fba1f94 commit 774874e

File tree

168 files changed

+9905
-1336
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

168 files changed

+9905
-1336
lines changed

‎.github/workflows/master.yml‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,14 @@ on:
2222
branches:
2323
- master
2424
- branch-*
25+
paths-ignore:
26+
- 'docs/**'
2527
pull_request:
2628
branches:
2729
- master
2830
- branch-*
31+
paths-ignore:
32+
- 'docs/**'
2933

3034
concurrency:
3135
group: test-${{ github.head_ref || github.run_id }}
@@ -62,22 +66,17 @@ jobs:
6266
comment: 'normal'
6367
- java: 8
6468
spark: '3.5'
65-
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
66-
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
67-
comment: 'verify-on-spark-3.2-binary'
68-
- java: 8
69-
spark: '3.5'
70-
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
69+
spark-archive: '-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
7170
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
7271
comment: 'verify-on-spark-3.3-binary'
7372
- java: 8
7473
spark: '3.5'
75-
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
74+
spark-archive: '-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
7675
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
7776
comment: 'verify-on-spark-3.4-binary'
7877
- java: 17
7978
spark: '3.5'
80-
spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.0 -Dspark.archive.name=spark-4.0.0-bin-hadoop3.tgz'
79+
spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.0.1 -Dspark.archive.name=spark-4.0.1-bin-hadoop3.tgz'
8180
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
8281
comment: 'verify-on-spark-4.0-binary'
8382
env:
@@ -245,24 +244,22 @@ jobs:
245244
java:
246245
- 8
247246
flink:
248-
- '1.17'
249-
- '1.18'
250247
- '1.19'
251248
- '1.20'
252249
flink-archive: [ "" ]
253250
comment: [ "normal" ]
254251
include:
255252
- java: 8
256253
flink: '1.20'
257-
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.17.2 -Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
254+
flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.17.2 -Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
258255
comment: 'verify-on-flink-1.17-binary'
259256
- java: 8
260257
flink: '1.20'
261-
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.18.1 -Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
258+
flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.18.1 -Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
262259
comment: 'verify-on-flink-1.18-binary'
263260
- java: 8
264261
flink: '1.20'
265-
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
262+
flink-archive: '-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
266263
comment: 'verify-on-flink-1.19-binary'
267264
steps:
268265
- uses: actions/checkout@v4
@@ -317,10 +314,10 @@ jobs:
317314
comment: [ "normal" ]
318315
include:
319316
- java: 8
320-
hive-archive: '-Dhive.archive.mirror=https://archive.apache.org/dist/hive/hive-2.3.10 -Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
317+
hive-archive: '-Dhive.archive.mirror=https://www.apache.org/dyn/closer.lua/hive/hive-2.3.10 -Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
321318
comment: 'verify-on-hive-2.3-binary'
322319
- java: 8
323-
hive-archive: '-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release -Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz'
320+
hive-archive: '-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release -Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz -Dhive.archive.query='
324321
comment: 'verify-on-hive-2.1-cdh6-binary'
325322
steps:
326323
- uses: actions/checkout@v4
@@ -422,17 +419,21 @@ jobs:
422419
uses: docker/setup-buildx-action@v3
423420
with:
424421
cache-binary: false
422+
- name: Pull Spark image
423+
run: |
424+
docker pull apache/spark:3.5.5
425425
- name: Build Kyuubi Docker Image
426426
uses: docker/build-push-action@v6
427427
with:
428428
# passthrough CI into build container
429429
build-args: |
430430
CI=${CI}
431-
MVN_ARG=--flink-provided --hive-provided
431+
BASE_IMAGE=apache/spark:3.5.5
432+
MVN_ARG=--spark-provided --flink-provided --hive-provided
432433
context: .
433434
file: build/Dockerfile.CI
434435
load: true
435-
tags: apache/kyuubi:latest
436+
tags: apache/kyuubi:ci
436437
- name: Cache Engine Archives
437438
uses: ./.github/actions/cache-engine-archives
438439
- name: Setup Minikube
@@ -442,9 +443,8 @@ jobs:
442443
sudo install minikube-linux-amd64 /usr/local/bin/minikube
443444
minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force
444445
# https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
445-
minikube image load apache/kyuubi:latest
446+
minikube image load apache/kyuubi:ci
446447
# pre-install spark into minikube
447-
docker pull apache/spark:3.5.5
448448
minikube image load apache/spark:3.5.5
449449
- name: kubectl pre-check
450450
run: |

‎.github/workflows/python.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,14 @@ on:
2222
branches:
2323
- master
2424
- branch-*
25+
paths-ignore:
26+
- 'docs/**'
2527
pull_request:
2628
branches:
2729
- master
2830
- branch-*
31+
paths-ignore:
32+
- 'docs/**'
2933

3034
concurrency:
3135
group: python-${{ github.head_ref || github.run_id }}

‎.github/workflows/style.yml‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ jobs:
6868
build/mvn clean install -pl extensions/spark/kyuubi-extension-spark-3-3 -Pspark-3.3
6969
build/mvn clean install -pl extensions/spark/kyuubi-extension-spark-3-4 -Pspark-3.4
7070
build/mvn clean install -pl extensions/spark/kyuubi-extension-spark-3-5,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.5
71+
build/mvn clean install -pl extensions/spark/kyuubi-extension-spark-4-0 -Pspark-4.0 -Pscala-2.13
7172
7273
- name: Scalastyle with maven
7374
id: scalastyle-check
@@ -132,9 +133,3 @@ jobs:
132133
with:
133134
name: super-linter-log
134135
path: super-linter.log
135-
- name: check bin directory
136-
uses: ludeeus/[email protected]
137-
with:
138-
# TODO: enable for all folders step by step
139-
scandir: './bin'
140-
severity: error

0 commit comments

Comments
 (0)