Merge remote-tracking branch 'origin/dev' into dev

.github/workflows/api-test.yml

@@ -62,7 +62,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Build Image
         run: |
@@ -122,7 +122,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-api-test
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - uses: actions/download-artifact@v4
         name: Download Docker Images

.github/workflows/backend.yml

@@ -21,15 +21,8 @@ on:
   push:
     branches:
       - dev
-    paths:
-      - '.github/workflows/backend.yml'
-      - 'package.xml'
-      - 'pom.xml'
-      - 'dolphinscheduler-alert/**'
-      - 'dolphinscheduler-api/**'
-      - 'dolphinscheduler-common/**'
-      - 'dolphinscheduler-dao/**'
-      - 'dolphinscheduler-rpc/**'
+      - '[0-9]+.[0-9]+.[0-9]+-prepare'
+      - '[0-9]+.[0-9]+.[0-9]+-release'
   pull_request:
 
 concurrency:
@@ -131,9 +124,39 @@ jobs:
       - name: Running cluster test
         run: |
           /bin/bash ${{ matrix.case.script }}
+  schema-check-prepare:
+    name: schema-check-prepare-${{ matrix.version }}
+    needs: paths-filter
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [ "3.1.9", "3.2.0" ]
+    steps:
+      - uses: actions/cache/restore@v4
+        name: Download Binary Package Cache
+        with:
+          path: /tmp/${{ matrix.version }}
+          key: apache-dolphinscheduler-${{ matrix.version }}-bin
+          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
+      - name: Download Binary Package if not cached
+        run: |
+          if [ ! -f /tmp/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz ]; then
+            echo "Binary package not found in cache, downloading..."
+            mkdir -p /tmp/${{ matrix.version }}
+            wget https://mirrors.tuna.tsinghua.edu.cn/apache/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P /tmp/${{ matrix.version }}
+          else
+            echo "Binary package found in cache, skipping download."
+          fi
+      - uses: actions/cache/save@v4
+        name: Upload Binary Package Cache
+        with:
+          path: /tmp/${{ matrix.version }}
+          key: apache-dolphinscheduler-${{ matrix.version }}-bin
   schema-check:
     name: ${{ matrix.case.name }}-${{ matrix.version }}
-    needs: build
+    needs: [ build, schema-check-prepare]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     strategy:
@@ -158,6 +181,12 @@ jobs:
         with:
           name: binary-package-8
           path: ds_schema_check_test/dev
+      - uses: actions/cache/restore@v4
+        name: Download Binary Package Cache
+        with:
+          path: /tmp/${{ matrix.version }}
+          key: apache-dolphinscheduler-${{ matrix.version }}-bin
+          restore-keys: apache-dolphinscheduler-${{ matrix.version }}-bin
       - name: Running Schema Check
         run: |
           /bin/bash ${{ matrix.case.script }} ${{ matrix.version }}

.github/workflows/e2e.yml

@@ -64,7 +64,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Build Image
         run: |
@@ -151,7 +151,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - uses: actions/download-artifact@v4
         name: Download Docker Images

.github/workflows/publish-docker.yaml

@@ -51,7 +51,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-docker
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Set environment variables
         run: |

.github/workflows/schema-check/mysql/start-job.sh

@@ -25,7 +25,11 @@ mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
+  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+else
+  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
+fi
 tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 

.github/workflows/schema-check/postgresql/start-job.sh

@@ -25,7 +25,11 @@ mkdir -p ds_schema_check_test/dev ds_schema_check_test/${DS_VERSION}
 curl -sSf https://atlasgo.sh | sh
 
 # Preparing the environment
-wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+if [ ! -f /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ]; then
+  wget https://archive.apache.org/dist/dolphinscheduler/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -P ds_schema_check_test/${DS_VERSION}
+else
+  mv /tmp/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz ds_schema_check_test/${DS_VERSION}
+fi
 tar -xzf ds_schema_check_test/${DS_VERSION}/apache-dolphinscheduler-${DS_VERSION}-bin.tar.gz -C ds_schema_check_test/${DS_VERSION} --strip-components 1
 tar -xzf ds_schema_check_test/dev/apache-dolphinscheduler-*-bin.tar.gz -C ds_schema_check_test/dev --strip-components 1
 

.github/workflows/unit-test.yml

@@ -74,7 +74,7 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-unit-test
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
           restore-keys: ${{ runner.os }}-maven-
       - name: Run Unit tests
         run: ./mvnw clean verify -B -Dmaven.test.skip=false -Dspotless.skip=true -DskipUT=false

docs/docs/en/guide/installation/pseudo-cluster.md

@@ -33,7 +33,7 @@ echo "dolphinscheduler" | passwd --stdin dolphinscheduler
 
 # Configure sudo without password
 sed -i '$adolphinscheduler  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
-sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers
+sed -i 's/Defaults    requiretty/#Defaults    requiretty/g' /etc/sudoers
 
 # Modify directory permissions and grant permissions for user you created above
 chown -R dolphinscheduler:dolphinscheduler apache-dolphinscheduler-*-bin
@@ -43,7 +43,11 @@ chmod -R 755 apache-dolphinscheduler-*-bin
 > **_NOTICE:_**
 >
 > - Due to DolphinScheduler's multi-tenant task switch user using command `sudo -u {linux-user} -i`, the deployment user needs to have `sudo` privileges and be password-free. If novice learners don’t understand, you can ignore this point for now.
-> - If you find the line "Defaults requirett" in the `/etc/sudoers` file, please comment the content.
+> - If you find the line "Defaults requiretty" in the `/etc/sudoers` file, please comment the content.
+
+### Prepare Zookeeper
+
+If you use Zookeeper as the registry center, you need to install Zookeeper and start it first.
 
 ## Modify Configuration
 

docs/docs/en/guide/installation/standalone.md

@@ -14,6 +14,31 @@ If you want to deploy DolphinScheduler in production, we recommend you follow [c
 - JDK：download [JDK][jdk] (1.8 or 11), install and configure environment variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH` variable. You can skip this step if it already exists in your environment.
 - Binary package: download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download/<version>).  <!-- markdown-link-check-disable-line -->
 
+### Configure User Exemption and Permissions
+
+Create a deployment user, and make sure to configure `sudo` without password. Here make an example to create user `dolphinscheduler`:
+
+```shell
+# To create a user, login as root
+useradd dolphinscheduler
+
+# Add password
+echo "dolphinscheduler" | passwd --stdin dolphinscheduler
+
+# Configure sudo without password
+sed -i '$adolphinscheduler  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
+sed -i 's/Defaults    requiretty/#Defaults    requiretty/g' /etc/sudoers
+
+# Modify directory permissions and grant permissions for user you created above
+chown -R dolphinscheduler:dolphinscheduler apache-dolphinscheduler-*-bin
+chmod -R 755 apache-dolphinscheduler-*-bin
+```
+
+> **_NOTICE:_**
+>
+> - Due to DolphinScheduler's multi-tenant task switch user using command `sudo -u {linux-user} -i`, the deployment user needs to have `sudo` privileges and be password-free. If novice learners don’t understand, you can ignore this point for now.
+> - If you find the line "Defaults requiretty" in the `/etc/sudoers` file, please comment the content.
+
 ## Start DolphinScheduler Standalone Server
 
 ### Extract and Start DolphinScheduler

docs/docs/zh/guide/installation/pseudo-cluster.md

@@ -33,7 +33,7 @@ echo "dolphinscheduler" | passwd --stdin dolphinscheduler
 
 # 配置 sudo 免密
 sed -i '$adolphinscheduler  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
-sed -i 's/Defaults    requirett/#Defaults    requirett/g' /etc/sudoers
+sed -i 's/Defaults    requiretty/#Defaults    requiretty/g' /etc/sudoers
 
 # 修改目录权限，使得部署用户对二进制包解压后的 apache-dolphinscheduler-*-bin 目录有操作权限
 chown -R dolphinscheduler:dolphinscheduler apache-dolphinscheduler-*-bin
@@ -43,16 +43,11 @@ chmod -R 755 apache-dolphinscheduler-*-bin
 > **_注意:_**
 >
 > - 因为任务执行服务是以 `sudo -u {linux-user} -i` 切换不同 linux 用户的方式来实现多租户运行作业，所以部署用户需要有 sudo 权限，而且是免密的。初学习者不理解的话，完全可以暂时忽略这一点
-> - 如果发现 `/etc/sudoers` 文件中有 "Defaults requirett" 这行，也请注释掉
+> - 如果发现 `/etc/sudoers` 文件中有 "Defaults requiretty" 这行，也请注释掉
 
-### 启动 zookeeper
+### 准备 zookeeper
 
-进入 zookeeper 的安装目录，将 `zoo_sample.cfg` 配置文件复制到 `conf/zoo.cfg`，并将 `conf/zoo.cfg` 中 dataDir 中的值改成 `dataDir=./tmp/zookeeper`
-
-```shell
-# 启动 zookeeper
-./bin/zkServer.sh start
-```
+如果使用 Zookeeper 作为注册中心，需要先安装 Zookeeper 并启动。
 
 ## 修改相关配置
 

docs/docs/zh/guide/installation/standalone.md

@@ -12,6 +12,31 @@ Standalone 仅适用于 DolphinScheduler 的快速体验.
 - JDK：下载[JDK][jdk] (1.8 or 11)，安装并配置 `JAVA_HOME` 环境变量，并将其下的 `bin` 目录追加到 `PATH` 环境变量中。如果你的环境中已存在，可以跳过这步。
 - 二进制包：在[下载页面](https://dolphinscheduler.apache.org/en-us/download/<version>)下载 DolphinScheduler 二进制包  <!-- markdown-link-check-disable-line -->
 
+## 配置用户免密及权限
+
+创建部署用户，并且一定要配置 `sudo` 免密。以创建 dolphinscheduler 用户为例
+
+```shell
+# 创建用户需使用 root 登录
+useradd dolphinscheduler
+
+# 添加密码
+echo "dolphinscheduler" | passwd --stdin dolphinscheduler
+
+# 配置 sudo 免密
+sed -i '$adolphinscheduler  ALL=(ALL)  NOPASSWD: NOPASSWD: ALL' /etc/sudoers
+sed -i 's/Defaults    requiretty/#Defaults    requiretty/g' /etc/sudoers
+
+# 修改目录权限，使得部署用户对二进制包解压后的 apache-dolphinscheduler-*-bin 目录有操作权限
+chown -R dolphinscheduler:dolphinscheduler apache-dolphinscheduler-*-bin
+chmod -R 755 apache-dolphinscheduler-*-bin
+```
+
+> **_注意:_**
+>
+> - 因为任务执行服务是以 `sudo -u {linux-user} -i` 切换不同 linux 用户的方式来实现多租户运行作业，所以部署用户需要有 sudo 权限，而且是免密的。初学习者不理解的话，完全可以暂时忽略这一点
+> - 如果发现 `/etc/sudoers` 文件中有 "Defaults requiretty" 这行，也请注释掉
+
 ## 启动 DolphinScheduler Standalone Server
 
 ### 解压并启动 DolphinScheduler