Merge branch 'dev' into adbspark

apache · Jul 8, 2024 · d64f774 · d64f774
2 parents b713221 + 8f6df8d
commit d64f774
Show file tree

Hide file tree

Showing 297 changed files with 7,016 additions and 5,865 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -17,6 +17,7 @@
 
 /.github/ @SbloodyS
 /deploy/ @SbloodyS
+/deploy/kubernetes/ @Gallardot
 /dolphinscheduler-alert/ @SbloodyS
 /dolphinscheduler-api/ @caishunfeng @SbloodyS
 /dolphinscheduler-api-test/ @SbloodyS
@@ -31,15 +32,15 @@
 /dolphinscheduler-e2e/ @SbloodyS
 /dolphinscheduler-extract/ @ruanwenjun
 /dolphinscheduler-master/ @caishunfeng @SbloodyS @ruanwenjun
-/dolphinscheduler-meter/  @ruanwenjun @EricGao888
+/dolphinscheduler-meter/  @ruanwenjun @EricGao888 @Gallardot
 /dolphinscheduler-microbench/ @SbloodyS @ruanwenjun
-/dolphinscheduler-registry/  @ruanwenjun
+/dolphinscheduler-registry/  @ruanwenjun @Gallardot
 /dolphinscheduler-scheduler-plugin/ @ruanwenjun
 /dolphinscheduler-service/ @caishunfeng
 /dolphinscheduler-spi/ @caishunfeng
 /dolphinscheduler-standalone-server/ @SbloodyS
 /dolphinscheduler-storage-plugin/ @ruanwenjun @SbloodyS
-/dolphinscheduler-task-plugin/ @caishunfeng @SbloodyS @zhuangchong
+/dolphinscheduler-task-plugin/ @caishunfeng @SbloodyS @zhuangchong @Gallardot
 /dolphinscheduler-tools/ @caishunfeng @SbloodyS @EricGao888
 /dolphinscheduler-ui/ @songjianet @Amy0104
 /dolphinscheduler-worker/ @caishunfeng @SbloodyS @ruanwenjun

diff --git a/.github/actions/workflow-telemetry-action b/.github/actions/workflow-telemetry-action
diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml
@@ -20,6 +20,8 @@ on:
   push:
     branches:
       - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
 
 name: API-Test
 
@@ -35,7 +37,7 @@ jobs:
     outputs:
       not-ignore: ${{ steps.filter.outputs.not-ignore }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
         id: filter
         with:
@@ -49,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Sanity Check
@@ -73,7 +75,7 @@ jobs:
         run: |
           docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
           && du -sh /tmp/standalone-image.tar
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         name: Upload Docker Images
         with:
           name: standalone-image
@@ -104,7 +106,7 @@ jobs:
     env:
       RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Cache local Maven repository
@@ -113,7 +115,7 @@ jobs:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
           restore-keys: ${{ runner.os }}-maven-
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         name: Download Docker Images
         with:
           name: standalone-image
@@ -125,9 +127,9 @@ jobs:
         run: |
           ./mvnw -B -f dolphinscheduler-api-test/pom.xml -am \
             -DfailIfNoTests=false \
-            -Dspotless.skip=false \
+            -Dspotless.skip=true \
             -Dtest=${{ matrix.case.class }} test
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: always()
         name: Upload Recording
         with:

diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -44,7 +44,7 @@ jobs:
       not-ignore: ${{ steps.filter.outputs.not-ignore }}
       db-schema: ${{ steps.filter.outputs.db-schema }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
         id: filter
         with:
@@ -63,7 +63,7 @@ jobs:
         java: [ '8', '11' ]
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Set up JDK ${{ matrix.java }}
@@ -91,7 +91,7 @@ jobs:
                  -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
       - name: Check dependency license
         run: tools/dependencies/check-LICENSE.sh
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: ${{ matrix.java == '8' }}
         name: Upload Binary Package
         with:
@@ -115,10 +115,10 @@ jobs:
           - name: cluster-test-postgresql-with-postgresql-registry
             script: .github/workflows/cluster-test/postgresql_with_postgresql_registry/start-job.sh
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         name: Download Binary Package
         with:
           # Only run cluster test on jdk8
@@ -165,7 +165,7 @@ jobs:
           mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}
           curl -sSf https://atlasgo.sh | sh
       - name: Download Tarball
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: binary-package-8
           path: dolphinscheduler/dev

diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml
@@ -23,6 +23,8 @@ on:
   pull_request:
     branches:
       - 'dev'
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
 
 concurrency:
   group: codeql-${{ github.event.pull_request.number || github.ref }}

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -30,7 +30,7 @@ jobs:
     timeout-minutes: 10
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Style Check
         run: ./mvnw spotless:check
   img-check:
@@ -40,7 +40,7 @@ jobs:
       run:
         working-directory: docs
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
@@ -54,7 +54,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - run: sudo npm install -g [email protected]
       - run: sudo apt install plocate -y
       # NOTE: Change command from `find . -name "*.md"` to `find . -not -path "*/node_modules/*" -not -path "*/.tox/*" -name "*.md"`
@@ -70,7 +70,7 @@ jobs:
     outputs:
       helm-doc: ${{ steps.filter.outputs.helm-doc }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
         id: filter
         with:
@@ -84,7 +84,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Generating helm-doc

diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml
@@ -20,6 +20,8 @@ on:
   push:
     branches:
       - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
 
 name: E2E-K8S
 
@@ -35,7 +37,7 @@ jobs:
     outputs:
       not-ignore: ${{ steps.filter.outputs.not-ignore }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
         id: filter
         with:
@@ -49,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Build Image

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -20,13 +20,17 @@ on:
   push:
     branches:
       - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
 
 name: E2E
 
 concurrency:
   group: e2e-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  pull-requests: write
 
 jobs:
   paths-filter:
@@ -35,7 +39,7 @@ jobs:
     outputs:
       not-ignore: ${{ steps.filter.outputs.not-ignore }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
         id: filter
         with:
@@ -49,7 +53,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Sanity Check
@@ -74,7 +78,7 @@ jobs:
         run: |
           docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
           && du -sh /tmp/standalone-image.tar
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         name: Upload Docker Images
         with:
           name: standalone-image
@@ -120,63 +124,36 @@ jobs:
             class: org.apache.dolphinscheduler.e2e.cases.ClickhouseDataSourceE2ETest
           - name: PostgresDataSource
             class: org.apache.dolphinscheduler.e2e.cases.PostgresDataSourceE2ETest
-    env:
-      RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - name: Cache local Maven repository
-        uses: actions/cache@v3
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
-          restore-keys: ${{ runner.os }}-maven-
-      - uses: actions/download-artifact@v2
-        name: Download Docker Images
-        with:
-          name: standalone-image
-          path: /tmp
-      - name: Load Docker Images
-        run: |
-          docker load -i /tmp/standalone-image.tar
-      - name: Run Test
-        run: |
-          ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
-            -DfailIfNoTests=false \
-            -Dtest=${{ matrix.case.class }} test
-      - uses: actions/upload-artifact@v2
-        if: always()
-        name: Upload Recording
-        with:
-          name: recording-${{ matrix.case.name }}
-          path: ${{ env.RECORDING_PATH }}
-          retention-days: 1
-  e2e-optional:
-    name: ${{ matrix.case.name }}
-    needs: build
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        case:
+          - name: ShellTaskE2ETest
+            class: org.apache.dolphinscheduler.e2e.cases.tasks.ShellTaskE2ETest
+          - name: PythonTaskE2ETest
+            class: org.apache.dolphinscheduler.e2e.cases.tasks.PythonTaskE2ETest
           - name: SqlServerDataSource
             class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest
           - name: HiveDataSource
             class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest
     env:
       RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'adopt'
+      - name: Collect Workflow Telemetry
+        uses: ./.github/actions/workflow-telemetry-action
+        with:
+          comment_on_pr: false
       - name: Cache local Maven repository
         uses: actions/cache@v3
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
           restore-keys: ${{ runner.os }}-maven-
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         name: Download Docker Images
         with:
           name: standalone-image
@@ -189,7 +166,7 @@ jobs:
           ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
             -DfailIfNoTests=false \
             -Dtest=${{ matrix.case.class }} test
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: always()
         name: Upload Recording
         with:

diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
@@ -20,7 +20,8 @@ name: Frontend
 on:
   push:
     branches:
-      - dev
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
     paths:
       - '.github/workflows/frontend.yml'
       - 'dolphinscheduler-ui/**'
@@ -41,7 +42,7 @@ jobs:
     outputs:
       not-ignore: ${{ steps.filter.outputs.not-ignore }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
         id: filter
         with:
@@ -58,7 +59,7 @@ jobs:
       matrix:
         os: [ ubuntu-latest, macos-latest ]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - if: matrix.os == 'ubuntu-latest'
@@ -82,7 +83,7 @@ jobs:
     needs: [ build, paths-filter ]
     if: always()
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Status
         run: |
           if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then