[Improvement-16635] Using Docker Hub in Schema Check CI (#16636)

apache · Sep 22, 2024 · abe1185 · abe1185
1 parent c989973
commit abe1185
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 68 deletions.
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -124,39 +124,9 @@ jobs:
       - name: Running cluster test
         run: |
           /bin/bash ${{ matrix.case.script }}
-  schema-check-prepare:
-    name: schema-check-prepare-${{ matrix.version }}
-    needs: paths-filter
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        version: [ "3.1.9", "3.2.0" ]
-    steps:
-      - uses: actions/cache/restore@v4
-        name: Download Binary Package Cache
-        with:
-          path: /tmp/${{ matrix.version }}
-          key: apache-dolphinscheduler-${{ matrix.version }}-bin
-          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
-      - name: Download Binary Package if not cached
-        run: |
-          if [ ! -f /tmp/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz ]; then
-            echo "Binary package not found in cache, downloading..."
-            mkdir -p /tmp/${{ matrix.version }}
-            wget https://mirrors.tuna.tsinghua.edu.cn/apache/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P /tmp/${{ matrix.version }}
-          else
-            echo "Binary package found in cache, skipping download."
-          fi
-      - uses: actions/cache/save@v4
-        name: Upload Binary Package Cache
-        with:
-          path: /tmp/${{ matrix.version }}
-          key: apache-dolphinscheduler-${{ matrix.version }}-bin
   schema-check:
     name: ${{ matrix.case.name }}-${{ matrix.version }}
-    needs: [ build, schema-check-prepare]
+    needs: [ build ]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     strategy:
@@ -181,12 +151,6 @@ jobs:
         with:
           name: binary-package-8
           path: ds_schema_check_test/dev
-      - uses: actions/cache/restore@v4
-        name: Download Binary Package Cache
-        with:
-          path: /tmp/${{ matrix.version }}
-          key: apache-dolphinscheduler-${{ matrix.version }}-bin
-          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
       - name: Running Schema Check
         run: |
           /bin/bash ${{ matrix.case.script }} ${{ matrix.version }}

diff --git a/.github/workflows/schema-check/mysql/docker-compose-base.yaml b/.github/workflows/schema-check/mysql/docker-compose-base.yaml
@@ -28,8 +28,14 @@ services:
       MYSQL_DATABASE: dolphinscheduler_dev
     ports:
       - "3306:3306"
+    networks:
+      - schema-test
     healthcheck:
       test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
       interval: 5s
       timeout: 60s
       retries: 120
+
+networks:
+  schema-test:
+    name: schema-test
diff --git a/.github/workflows/schema-check/mysql/running-test.sh b/.github/workflows/schema-check/mysql/running-test.sh
@@ -29,8 +29,15 @@ export SPRING_DATASOURCE_PASSWORD="mysql"
 bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 # Install the target version schema and upgrade it
+docker run -v "./ds_schema_check_test/mysql-connector-java-8.0.16.jar:/opt/dolphinscheduler/tools/libs/mysql-connector-java-8.0.16.jar" \
+--network schema-test apache/dolphinscheduler-tools:${DS_VERSION} -c \
+'export DATABASE="mysql"; \
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"; \
+export SPRING_DATASOURCE_USERNAME="root"; \
+export SPRING_DATASOURCE_PASSWORD="mysql"; \
+export SPRING_DATASOURCE_URL='jdbc:mysql://mysql:3306/dolphinscheduler_${DATABASE_VERSION}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false'; \
+bash tools/bin/upgrade-schema.sh'
 export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${DATABASE_VERSION}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
-bash ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
 bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 # Compare the schema

diff --git a/.github/workflows/schema-check/mysql/start-job.sh b/.github/workflows/schema-check/mysql/start-job.sh
@@ -21,35 +21,22 @@ DS_VERSION=$1
 DATABASE_VERSION=${DS_VERSION//\./}
 
 # Install Atlas and Create Dir
-mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
+mkdir -p ds_schema_check_test/dev
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
-  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
-else
-  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
-fi
-tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
+docker pull apache/dolphinscheduler-tools:${DS_VERSION}
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 
-if [[ $DATABASE_VERSION -lt 300 ]]; then
-  chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/script/create-dolphinscheduler.sh
-else
-  chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
-fi
+chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
 MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
 wget ${MYSQL_JDBC_URL} -O ds_schema_check_test/${MYSQL_JDBC_JAR}
-for base_dir in ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}; do
-  if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
-    cp ds_schema_check_test/${MYSQL_JDBC_JAR} ${base_dir}/lib
-  else
+for base_dir in ds_schema_check_test/dev; do
     for d in alert-server api-server master-server worker-server tools; do
       cp ds_schema_check_test/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
     done
-  fi
 done
 docker compose -f .github/workflows/schema-check/mysql/docker-compose-base.yaml up -d --wait
 docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${DATABASE_VERSION}";

diff --git a/.github/workflows/schema-check/postgresql/docker-compose-base.yaml b/.github/workflows/schema-check/postgresql/docker-compose-base.yaml
@@ -27,8 +27,14 @@ services:
       POSTGRES_DB: dolphinscheduler_dev
     ports:
       - "5432:5432"
+    networks:
+      - schema-test
     healthcheck:
       test: [ "CMD-SHELL", "pg_isready -U postgres" ]
       interval: 5s
       timeout: 60s
       retries: 120
+
+networks:
+  schema-test:
+    name: schema-test
diff --git a/.github/workflows/schema-check/postgresql/running-test.sh b/.github/workflows/schema-check/postgresql/running-test.sh
@@ -29,8 +29,14 @@ export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_
 bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 # Install the target version schema and upgrade it
+docker run --network schema-test apache/dolphinscheduler-tools:${DS_VERSION} -c \
+'export DATABASE="postgresql"; \
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"; \
+export SPRING_DATASOURCE_USERNAME="postgres"; \
+export SPRING_DATASOURCE_PASSWORD="postgres"; \
+export SPRING_DATASOURCE_URL='jdbc:postgresql://postgres:5432/dolphinscheduler_${DATABASE_VERSION}'; \
+bash tools/bin/upgrade-schema.sh'
 export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${DATABASE_VERSION}"
-bash ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
 bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 # Compare the schema

diff --git a/.github/workflows/schema-check/postgresql/start-job.sh b/.github/workflows/schema-check/postgresql/start-job.sh
@@ -21,23 +21,13 @@ DS_VERSION=$1
 DATABASE_VERSION=${DS_VERSION//\./}
 
 # Install Atlas and Create Dir
-mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
+mkdir -p ds_schema_check_test/dev
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
-  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
-else
-  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
-fi
-tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 
-if [[ $DATABASE_VERSION -lt 300 ]]; then
-  chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/script/create-dolphinscheduler.sh
-else
-  chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
-fi
+chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 docker compose -f .github/workflows/schema-check/postgresql/docker-compose-base.yaml up -d --wait
 docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${DATABASE_VERSION}";