diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index ab336eefea77..e7b282a29c68 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -124,39 +124,9 @@ jobs: - name: Running cluster test run: | /bin/bash ${{ matrix.case.script }} - schema-check-prepare: - name: schema-check-prepare-${{ matrix.version }} - needs: paths-filter - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - version: [ "3.1.9", "3.2.0" ] - steps: - - uses: actions/cache/restore@v4 - name: Download Binary Package Cache - with: - path: /tmp/${{ matrix.version }} - key: apache-dolphinscheduler-${{ matrix.version }}-bin - restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin - - name: Download Binary Package if not cached - run: | - if [ ! -f /tmp/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz ]; then - echo "Binary package not found in cache, downloading..." - mkdir -p /tmp/${{ matrix.version }} - wget https://mirrors.tuna.tsinghua.edu.cn/apache/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P /tmp/${{ matrix.version }} - else - echo "Binary package found in cache, skipping download." - fi - - uses: actions/cache/save@v4 - name: Upload Binary Package Cache - with: - path: /tmp/${{ matrix.version }} - key: apache-dolphinscheduler-${{ matrix.version }}-bin schema-check: name: ${{ matrix.case.name }}-${{ matrix.version }} - needs: [ build, schema-check-prepare] + needs: [ build ] runs-on: ubuntu-latest timeout-minutes: 20 strategy: @@ -181,12 +151,6 @@ jobs: with: name: binary-package-8 path: ds_schema_check_test/dev - - uses: actions/cache/restore@v4 - name: Download Binary Package Cache - with: - path: /tmp/${{ matrix.version }} - key: apache-dolphinscheduler-${{ matrix.version }}-bin - restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin - name: Running Schema Check run: | /bin/bash ${{ matrix.case.script }} ${{ matrix.version }} diff --git a/.github/workflows/schema-check/mysql/docker-compose-base.yaml b/.github/workflows/schema-check/mysql/docker-compose-base.yaml index d515a917621b..4223f99fdaa9 100644 --- a/.github/workflows/schema-check/mysql/docker-compose-base.yaml +++ b/.github/workflows/schema-check/mysql/docker-compose-base.yaml @@ -28,8 +28,14 @@ services: MYSQL_DATABASE: dolphinscheduler_dev ports: - "3306:3306" + networks: + - schema-test healthcheck: test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD interval: 5s timeout: 60s retries: 120 + +networks: + schema-test: + name: schema-test diff --git a/.github/workflows/schema-check/mysql/running-test.sh b/.github/workflows/schema-check/mysql/running-test.sh index 72e2fb3b6ac9..cc0197b9f049 100644 --- a/.github/workflows/schema-check/mysql/running-test.sh +++ b/.github/workflows/schema-check/mysql/running-test.sh @@ -29,8 +29,15 @@ export SPRING_DATASOURCE_PASSWORD="mysql" bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh # Install the target version schema and upgrade it +docker run -v "./ds_schema_check_test/mysql-connector-java-8.0.16.jar:/opt/dolphinscheduler/tools/libs/mysql-connector-java-8.0.16.jar" \ +--network schema-test apache/dolphinscheduler-tools:${DS_VERSION} -c \ +'export DATABASE="mysql"; \ +export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"; \ +export SPRING_DATASOURCE_USERNAME="root"; \ +export SPRING_DATASOURCE_PASSWORD="mysql"; \ +export SPRING_DATASOURCE_URL='jdbc:mysql://mysql:3306/dolphinscheduler_${DATABASE_VERSION}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false'; \ +bash tools/bin/upgrade-schema.sh' export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${DATABASE_VERSION}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false" -bash ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh # Compare the schema diff --git a/.github/workflows/schema-check/mysql/start-job.sh b/.github/workflows/schema-check/mysql/start-job.sh index 852c7f8a0e72..79f2b28c8336 100644 --- a/.github/workflows/schema-check/mysql/start-job.sh +++ b/.github/workflows/schema-check/mysql/start-job.sh @@ -21,35 +21,22 @@ DS_VERSION=$1 DATABASE_VERSION=${DS_VERSION//\./} # Install Atlas and Create Dir -mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION} +mkdir -p ds_schema_check_test/dev curl -sSf https://atlasgo.sh | sh # Preparing the environment -if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then - wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION} -else - mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION} -fi -tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1 +docker pull apache/dolphinscheduler-tools:${DS_VERSION} tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1 -if [[ $DATABASE_VERSION -lt 300 ]]; then - chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/script/create-dolphinscheduler.sh -else - chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh -fi +chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar" wget ${MYSQL_JDBC_URL} -O ds_schema_check_test/${MYSQL_JDBC_JAR} -for base_dir in ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}; do - if [[ $base_dir == *"dolphinscheduler/2"* ]]; then - cp ds_schema_check_test/${MYSQL_JDBC_JAR} ${base_dir}/lib - else +for base_dir in ds_schema_check_test/dev; do for d in alert-server api-server master-server worker-server tools; do cp ds_schema_check_test/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs done - fi done docker compose -f .github/workflows/schema-check/mysql/docker-compose-base.yaml up -d --wait docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${DATABASE_VERSION}"; diff --git a/.github/workflows/schema-check/postgresql/docker-compose-base.yaml b/.github/workflows/schema-check/postgresql/docker-compose-base.yaml index 9f09f0d32619..1bcae2fbafa9 100644 --- a/.github/workflows/schema-check/postgresql/docker-compose-base.yaml +++ b/.github/workflows/schema-check/postgresql/docker-compose-base.yaml @@ -27,8 +27,14 @@ services: POSTGRES_DB: dolphinscheduler_dev ports: - "5432:5432" + networks: + - schema-test healthcheck: test: [ "CMD-SHELL", "pg_isready -U postgres" ] interval: 5s timeout: 60s retries: 120 + +networks: + schema-test: + name: schema-test diff --git a/.github/workflows/schema-check/postgresql/running-test.sh b/.github/workflows/schema-check/postgresql/running-test.sh index 0118ca2477cf..889097f3feed 100644 --- a/.github/workflows/schema-check/postgresql/running-test.sh +++ b/.github/workflows/schema-check/postgresql/running-test.sh @@ -29,8 +29,14 @@ export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_ bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh # Install the target version schema and upgrade it +docker run --network schema-test apache/dolphinscheduler-tools:${DS_VERSION} -c \ +'export DATABASE="postgresql"; \ +export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"; \ +export SPRING_DATASOURCE_USERNAME="postgres"; \ +export SPRING_DATASOURCE_PASSWORD="postgres"; \ +export SPRING_DATASOURCE_URL='jdbc:postgresql://postgres:5432/dolphinscheduler_${DATABASE_VERSION}'; \ +bash tools/bin/upgrade-schema.sh' export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${DATABASE_VERSION}" -bash ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh bash ds_schema_check_test/dev/tools/bin/upgrade-schema.sh # Compare the schema diff --git a/.github/workflows/schema-check/postgresql/start-job.sh b/.github/workflows/schema-check/postgresql/start-job.sh index 26308923e2ab..cf46dcd6f957 100644 --- a/.github/workflows/schema-check/postgresql/start-job.sh +++ b/.github/workflows/schema-check/postgresql/start-job.sh @@ -21,23 +21,13 @@ DS_VERSION=$1 DATABASE_VERSION=${DS_VERSION//\./} # Install Atlas and Create Dir -mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION} +mkdir -p ds_schema_check_test/dev curl -sSf https://atlasgo.sh | sh # Preparing the environment -if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then - wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION} -else - mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION} -fi -tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1 -if [[ $DATABASE_VERSION -lt 300 ]]; then - chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/script/create-dolphinscheduler.sh -else - chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh ds_schema_check_test/${DS_VERSION}/tools/bin/upgrade-schema.sh -fi +chmod +x ds_schema_check_test/dev/tools/bin/upgrade-schema.sh docker compose -f .github/workflows/schema-check/postgresql/docker-compose-base.yaml up -d --wait docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${DATABASE_VERSION}";