From abe1185d2895edf474591ca2c638bbfbebccd4e7 Mon Sep 17 00:00:00 2001
From: xiangzihao <460888207@qq.com>
Date: Sun, 22 Sep 2024 10:45:05 +0800
Subject: [PATCH] [Improvement-16635] Using Docker Hub in Schema Check CI
 (#16636)

---
 .github/workflows/backend.yml                 | 38 +------------------
 .../mysql/docker-compose-base.yaml            |  6 +++
 .../schema-check/mysql/running-test.sh        |  9 ++++-
 .../workflows/schema-check/mysql/start-job.sh | 21 ++--------
 .../postgresql/docker-compose-base.yaml       |  6 +++
 .../schema-check/postgresql/running-test.sh   |  8 +++-
 .../schema-check/postgresql/start-job.sh      | 14 +------
 7 files changed, 34 insertions(+), 68 deletions(-)

diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index ab336eefea77..e7b282a29c68 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -124,39 +124,9 @@ jobs:
       - name: Running cluster test
         run: |
           /bin/bash ${{ matrix.case.script }}
-  schema-check-prepare:
-    name: schema-check-prepare-${{ matrix.version }}
-    needs: paths-filter
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        version: [ "3.1.9", "3.2.0" ]
-    steps:
-      - uses: actions/cache/restore@v4
-        name: Download Binary Package Cache
-        with:
-          path: /tmp/${{ matrix.version }}
-          key: apache-dolphinscheduler-${{ matrix.version }}-bin
-          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
-      - name: Download Binary Package if not cached
-        run: |
-          if [ ! -f /tmp/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz ]; then
-            echo "Binary package not found in cache, downloading..."
-            mkdir -p /tmp/${{ matrix.version }}
-            wget https://mirrors.tuna.tsinghua.edu.cn/apache/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P /tmp/${{ matrix.version }}
-          else
-            echo "Binary package found in cache, skipping download."
-          fi
-      - uses: actions/cache/save@v4
-        name: Upload Binary Package Cache
-        with:
-          path: /tmp/${{ matrix.version }}
-          key: apache-dolphinscheduler-${{ matrix.version }}-bin
   schema-check:
     name: ${{ matrix.case.name }}-${{ matrix.version }}
-    needs: [ build, schema-check-prepare]
+    needs: [ build ]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     strategy:
@@ -181,12 +151,6 @@ jobs:
         with:
           name: binary-package-8
           path: ds_schema_check_test/dev
-      - uses: actions/cache/restore@v4
-        name: Download Binary Package Cache
-        with:
-          path: /tmp/${{ matrix.version }}
-          key: apache-dolphinscheduler-${{ matrix.version }}-bin
-          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
       - name: Running Schema Check
         run: |
           /bin/bash ${{ matrix.case.script }} ${{ matrix.version }}
diff --git a/.github/workflows/schema-check/mysql/docker-compose-base.yaml b/.github/workflows/schema-check/mysql/docker-compose-base.yaml
index d515a917621b..4223f99fdaa9 100644
--- a/.github/workflows/schema-check/mysql/docker-compose-base.yaml
+++ b/.github/workflows/schema-check/mysql/docker-compose-base.yaml
@@ -28,8 +28,14 @@ services:
       MYSQL_DATABASE: dolphinscheduler_dev
     ports:
       - "3306:3306"
+    networks:
+      - schema-test
     healthcheck:
       test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
       interval: 5s
       timeout: 60s
       retries: 120
+
+networks:
+  schema-test:
+    name: schema-test
diff --git a/.github/workflows/schema-check/mysql/running-test.sh b/.github/workflows/schema-check/mysql/running-test.sh
index 72e2fb3b6ac9..cc0197b9f049 100644
--- a/.github/workflows/schema-check/mysql/running-test.sh
+++ b/.github/workflows/schema-check/mysql/running-test.sh
@@ -29,8 +29,15 @@ export SPRING_DATASOURCE_PASSWORD="mysql"
 bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 # Install the target version schema and upgrade it
+docker run -v "./ds_schema_check_test/mysql-connector-java-8.0.16.jar:/opt/dolphinscheduler/tools/libs/mysql-connector-java-8.0.16.jar" \
+--network schema-test apache/dolphinscheduler-tools:${DS_VERSION} -c \
+'export DATABASE="mysql"; \
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"; \
+export SPRING_DATASOURCE_USERNAME="root"; \
+export SPRING_DATASOURCE_PASSWORD="mysql"; \
+export SPRING_DATASOURCE_URL='jdbc:mysql://mysql:3306/dolphinscheduler_${DATABASE_VERSION}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false'; \
+bash tools/bin/upgrade-schema.sh'
 export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${DATABASE_VERSION}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
-bash ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
 bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 # Compare the schema
diff --git a/.github/workflows/schema-check/mysql/start-job.sh b/.github/workflows/schema-check/mysql/start-job.sh
index 852c7f8a0e72..79f2b28c8336 100644
--- a/.github/workflows/schema-check/mysql/start-job.sh
+++ b/.github/workflows/schema-check/mysql/start-job.sh
@@ -21,35 +21,22 @@ DS_VERSION=$1
 DATABASE_VERSION=${DS_VERSION//\./}
 
 # Install Atlas and Create Dir
-mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
+mkdir -p ds_schema_check_test/dev
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
-  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
-else
-  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
-fi
-tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
+docker pull apache/dolphinscheduler-tools:${DS_VERSION}
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 
-if [[ $DATABASE_VERSION -lt 300 ]]; then
-  chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/script/create-dolphinscheduler.sh
-else
-  chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
-fi
+chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
 MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
 wget ${MYSQL_JDBC_URL} -O ds_schema_check_test/${MYSQL_JDBC_JAR}
-for base_dir in ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}; do
-  if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
-    cp ds_schema_check_test/${MYSQL_JDBC_JAR} ${base_dir}/lib
-  else
+for base_dir in ds_schema_check_test/dev; do
     for d in alert-server api-server master-server worker-server tools; do
       cp ds_schema_check_test/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
     done
-  fi
 done
 docker compose -f .github/workflows/schema-check/mysql/docker-compose-base.yaml up -d --wait
 docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${DATABASE_VERSION}";
diff --git a/.github/workflows/schema-check/postgresql/docker-compose-base.yaml b/.github/workflows/schema-check/postgresql/docker-compose-base.yaml
index 9f09f0d32619..1bcae2fbafa9 100644
--- a/.github/workflows/schema-check/postgresql/docker-compose-base.yaml
+++ b/.github/workflows/schema-check/postgresql/docker-compose-base.yaml
@@ -27,8 +27,14 @@ services:
       POSTGRES_DB: dolphinscheduler_dev
     ports:
       - "5432:5432"
+    networks:
+      - schema-test
     healthcheck:
       test: [ "CMD-SHELL", "pg_isready -U postgres" ]
       interval: 5s
       timeout: 60s
       retries: 120
+
+networks:
+  schema-test:
+    name: schema-test
diff --git a/.github/workflows/schema-check/postgresql/running-test.sh b/.github/workflows/schema-check/postgresql/running-test.sh
index 0118ca2477cf..889097f3feed 100644
--- a/.github/workflows/schema-check/postgresql/running-test.sh
+++ b/.github/workflows/schema-check/postgresql/running-test.sh
@@ -29,8 +29,14 @@ export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_
 bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 # Install the target version schema and upgrade it
+docker run --network schema-test apache/dolphinscheduler-tools:${DS_VERSION} -c \
+'export DATABASE="postgresql"; \
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"; \
+export SPRING_DATASOURCE_USERNAME="postgres"; \
+export SPRING_DATASOURCE_PASSWORD="postgres"; \
+export SPRING_DATASOURCE_URL='jdbc:postgresql://postgres:5432/dolphinscheduler_${DATABASE_VERSION}'; \
+bash tools/bin/upgrade-schema.sh'
 export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${DATABASE_VERSION}"
-bash ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
 bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 # Compare the schema
diff --git a/.github/workflows/schema-check/postgresql/start-job.sh b/.github/workflows/schema-check/postgresql/start-job.sh
index 26308923e2ab..cf46dcd6f957 100644
--- a/.github/workflows/schema-check/postgresql/start-job.sh
+++ b/.github/workflows/schema-check/postgresql/start-job.sh
@@ -21,23 +21,13 @@ DS_VERSION=$1
 DATABASE_VERSION=${DS_VERSION//\./}
 
 # Install Atlas and Create Dir
-mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
+mkdir -p ds_schema_check_test/dev
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
-  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
-else
-  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
-fi
-tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 
-if [[ $DATABASE_VERSION -lt 300 ]]; then
-  chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/script/create-dolphinscheduler.sh
-else
-  chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh
-fi
+chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh
 
 docker compose -f .github/workflows/schema-check/postgresql/docker-compose-base.yaml up -d --wait
 docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${DATABASE_VERSION}";