Merge branch 'master' into Add-Inverted-Checker-Case

.asf.yaml

@@ -152,13 +152,13 @@ github:
     - LemonLiTree
     - Yukang-Lian
     - TangSiyang2001
-    - Lchangliang
     - freemandealer
     - shuke987
     - wm1581066
     - KassieZ
     - yujun777
     - doris-robot
+    - LiBinfeng-01
 
 notifications:
     pullrequests_status:  [email protected]

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,39 @@
-## Proposed changes
+### What problem does this PR solve?
 
 Issue Number: close #xxx
 
-<!--Describe your changes.-->
+Related PR: #xxx
+
+Problem Summary:
+
+### Release note
+
+None
+
+### Check List (For Author)
+
+- Test <!-- At least one of them must be included. -->
+    - [ ] Regression test
+    - [ ] Unit Test
+    - [ ] Manual test (add detailed scripts or steps below)
+    - [ ] No need to test or manual test. Explain why:
+        - [ ] This is a refactor/code format and no logic has been changed.
+        - [ ] Previous test can cover this change.
+        - [ ] No code files have been changed.
+        - [ ] Other reason <!-- Add your reason?  -->
+
+- Behavior changed:
+    - [ ] No.
+    - [ ] Yes. <!-- Explain the behavior change -->
+
+- Does this need documentation?
+    - [ ] No.
+    - [ ] Yes. <!-- Add document PR link here. eg: https://github.com/apache/doris-website/pull/1214 -->
+
+### Check List (For Reviewer who merge this PR)
+
+- [ ] Confirm the release note
+- [ ] Confirm test cases
+- [ ] Confirm document
+- [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
 

.github/workflows/auto-cherry-pick.yml

@@ -30,7 +30,7 @@ permissions:
 jobs:
   auto_cherry_pick:
     runs-on: ubuntu-latest
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') && github.event.pull_request.merged == true }}
+    if: ${{ (contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/2.1.x')) && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -45,18 +45,27 @@ jobs:
           pip install PyGithub
       - name: Check SHA
         run: |
-            expected_sha="80b7c6087f2a3e4f4c7f035a52e8e7b05ce00f27aa5c1bd52179df685c912447f94a96145fd3204a3958d8ed9777de5a5183b120e99e0e95bbca0366d69b0ac0"
+            expected_sha="4e4c0d7689b765c7f0677d75d23222555afa9286af46cf77ced66fa247a298d9f8a8c86830d0ce55f70e5f09532b54fbafee040c0343833077cbc7e214d486d2"
             calculated_sha=$(sha512sum tools/auto-pick-script.py | awk '{ print $1 }')
             if [ "$calculated_sha" != "$expected_sha" ]; then
               echo "SHA mismatch! Expected: $expected_sha, but got: $calculated_sha"
               exit 1
             else
               echo "SHA matches: $calculated_sha"
             fi
-      - name: Auto cherry-pick
+      - name: Auto cherry-pick to branch-3.0
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO_NAME: ${{ github.repository }}
           CONFLICT_LABEL: cherry-pick-conflict-in-3.0
         run: |
           python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-3.0  
+      - name: Auto cherry-pick to branch-2.1
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/2.1.x') }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO_NAME: ${{ github.repository }}
+          CONFLICT_LABEL: cherry-pick-conflict-in-2.1.x
+        run: |
+            python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-2.1         

.github/workflows/build-extension.yml

@@ -20,7 +20,9 @@ name: Build Extensions
 
 on:
   pull_request:
-
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]
 concurrency:
   group: ${{ github.ref }} (Build Extensions)
   cancel-in-progress: true
@@ -29,6 +31,12 @@ jobs:
   changes:
     name: Detect Changes
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]')
     outputs:
       broker_changes: ${{ steps.filter.outputs.broker_changes }}
       docs_changes: ${{ steps.filter.outputs.docs_changes }}

.github/workflows/build-thirdparty.yml

@@ -19,6 +19,9 @@ name: Build Third Party Libraries
 
 on:
   pull_request:
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]  
 
 concurrency:
   group: ${{ github.ref }} (Build Third Party Libraries)
@@ -28,6 +31,12 @@ jobs:
   changes:
     name: Detect Changes
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]') 
     outputs:
       thirdparty_changes: ${{ steps.filter.outputs.thirdparty_changes }}
     steps:

.github/workflows/checkstyle.yaml

@@ -20,11 +20,20 @@ name: FE Code Style Checker
 
 on:
   pull_request:
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]
 
 jobs:
   java-checkstyle:
     name: "CheckStyle"
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]')
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/clang-format.yml

@@ -19,12 +19,22 @@
 ---
 name: Code Formatter
 
-on: [push, pull_request_target]
-
+on:
+  pull_request:
+  pull_request_target:
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]
 jobs:
   clang-format:
     name: "Clang Formatter"
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request') || (github.event_name == 'pull_request_target') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]')
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         if: ${{ github.event_name != 'pull_request_target' }}

.github/workflows/license-eyes.yml

@@ -22,10 +22,21 @@ on:
   push:
     branches:
       - master
+  workflow_dispatch:
+  issue_comment:
+    types: [ created ]
+
 jobs:
   license-check:
     name: "License Check"
     runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'pull_request_target') ||
+      (github.event_name == 'push' && github.ref == 'refs/heads/master') ||
+      (github.event_name == 'issue_comment' &&
+       github.event.comment.body == 'run buildall' &&
+       github.actor == 'doris-robot' &&
+       github.event.issue.user.login == 'github-actions[bot]')
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         if: ${{ github.event_name != 'pull_request_target' }}

be/src/agent/agent_server.cpp

@@ -33,6 +33,7 @@
 #include "agent/utils.h"
 #include "agent/workload_group_listener.h"
 #include "agent/workload_sched_policy_listener.h"
+#include "cloud/config.h"
 #include "common/config.h"
 #include "common/logging.h"
 #include "common/status.h"
@@ -48,9 +49,9 @@ using std::vector;
 
 namespace doris {
 
-AgentServer::AgentServer(ExecEnv* exec_env, const TMasterInfo& master_info)
-        : _master_info(master_info), _topic_subscriber(new TopicSubscriber()) {
-    MasterServerClient::create(master_info);
+AgentServer::AgentServer(ExecEnv* exec_env, const ClusterInfo* cluster_info)
+        : _cluster_info(cluster_info), _topic_subscriber(new TopicSubscriber()) {
+    MasterServerClient::create(cluster_info);
 
 #if !defined(BE_TEST) && !defined(__APPLE__)
     // Add subscriber here and register listeners
@@ -169,7 +170,7 @@ void AgentServer::start_workers(StorageEngine& engine, ExecEnv* exec_env) {
             "ALTER_TABLE", config::alter_tablet_worker_count, [&engine](auto&& task) { return alter_tablet_callback(engine, task); });
 
     _workers[TTaskType::CLONE] = std::make_unique<TaskWorkerPool>(
-            "CLONE", config::clone_worker_count, [&engine, &master_info = _master_info](auto&& task) { return clone_callback(engine, master_info, task); });
+            "CLONE", config::clone_worker_count, [&engine, &cluster_info = _cluster_info](auto&& task) { return clone_callback(engine, cluster_info, task); });
 
     _workers[TTaskType::STORAGE_MEDIUM_MIGRATE] = std::make_unique<TaskWorkerPool>(
             "STORAGE_MEDIUM_MIGRATE", config::storage_medium_migrate_count, [&engine](auto&& task) { return storage_medium_migrate_callback(engine, task); });
@@ -187,13 +188,13 @@ void AgentServer::start_workers(StorageEngine& engine, ExecEnv* exec_env) {
             "UPDATE_VISIBLE_VERSION", 1, [&engine](auto&& task) { return visible_version_callback(engine, task); });
 
     _report_workers.push_back(std::make_unique<ReportWorker>(
-            "REPORT_TASK", _master_info, config::report_task_interval_seconds, [&master_info = _master_info] { report_task_callback(master_info); }));
+            "REPORT_TASK", _cluster_info, config::report_task_interval_seconds, [&cluster_info = _cluster_info] { report_task_callback(cluster_info); }));
 
     _report_workers.push_back(std::make_unique<ReportWorker>(
-            "REPORT_DISK_STATE", _master_info, config::report_disk_state_interval_seconds, [&engine, &master_info = _master_info] { report_disk_callback(engine, master_info); }));
+            "REPORT_DISK_STATE", _cluster_info, config::report_disk_state_interval_seconds, [&engine, &cluster_info = _cluster_info] { report_disk_callback(engine, cluster_info); }));
 
     _report_workers.push_back(std::make_unique<ReportWorker>(
-            "REPORT_OLAP_TABLE", _master_info, config::report_tablet_interval_seconds,[&engine, &master_info = _master_info] { report_tablet_callback(engine, master_info); }));
+            "REPORT_OLAP_TABLET", _cluster_info, config::report_tablet_interval_seconds,[&engine, &cluster_info = _cluster_info] { report_tablet_callback(engine, cluster_info); }));
     // clang-format on
 }
 
@@ -211,13 +212,27 @@ void AgentServer::cloud_start_workers(CloudStorageEngine& engine, ExecEnv* exec_
             "CALC_DBM_TASK", config::calc_delete_bitmap_worker_count,
             [&engine](auto&& task) { return calc_delete_bitmap_callback(engine, task); });
 
+    // cloud, drop tablet just clean clear_cache, so just one thread do it
+    _workers[TTaskType::DROP] = std::make_unique<TaskWorkerPool>(
+            "DROP_TABLE", 1, [&engine](auto&& task) { return drop_tablet_callback(engine, task); });
+
     _report_workers.push_back(std::make_unique<ReportWorker>(
-            "REPORT_TASK", _master_info, config::report_task_interval_seconds,
-            [&master_info = _master_info] { report_task_callback(master_info); }));
+            "REPORT_TASK", _cluster_info, config::report_task_interval_seconds,
+            [&cluster_info = _cluster_info] { report_task_callback(cluster_info); }));
 
     _report_workers.push_back(std::make_unique<ReportWorker>(
-            "REPORT_DISK_STATE", _master_info, config::report_disk_state_interval_seconds,
-            [&engine, &master_info = _master_info] { report_disk_callback(engine, master_info); }));
+            "REPORT_DISK_STATE", _cluster_info, config::report_disk_state_interval_seconds,
+            [&engine, &cluster_info = _cluster_info] {
+                report_disk_callback(engine, cluster_info);
+            }));
+
+    if (config::enable_cloud_tablet_report) {
+        _report_workers.push_back(std::make_unique<ReportWorker>(
+                "REPORT_OLAP_TABLET", _cluster_info, config::report_tablet_interval_seconds,
+                [&engine, &cluster_info = _cluster_info] {
+                    report_tablet_callback(engine, cluster_info);
+                }));
+    }
 }
 
 // TODO(lingbin): each task in the batch may have it own status or FE must check and
@@ -226,8 +241,8 @@ void AgentServer::submit_tasks(TAgentResult& agent_result,
                                const std::vector<TAgentTaskRequest>& tasks) {
     Status ret_st;
 
-    // TODO check master_info here if it is the same with that of heartbeat rpc
-    if (_master_info.network_address.hostname.empty() || _master_info.network_address.port == 0) {
+    // TODO check cluster_info here if it is the same with that of heartbeat rpc
+    if (_cluster_info->master_fe_addr.hostname.empty() || _cluster_info->master_fe_addr.port == 0) {
         Status ret_st = Status::Cancelled("Have not get FE Master heartbeat yet");
         ret_st.to_thrift(&agent_result.status);
         return;

be/src/agent/agent_server.h

@@ -35,15 +35,15 @@ class ExecEnv;
 class TAgentPublishRequest;
 class TAgentResult;
 class TAgentTaskRequest;
-class TMasterInfo;
+class ClusterInfo;
 class TSnapshotRequest;
 class StorageEngine;
 class CloudStorageEngine;
 
 // Each method corresponds to one RPC from FE Master, see BackendService.
 class AgentServer {
 public:
-    explicit AgentServer(ExecEnv* exec_env, const TMasterInfo& master_info);
+    explicit AgentServer(ExecEnv* exec_env, const ClusterInfo* cluster_info);
     ~AgentServer();
 
     void start_workers(StorageEngine& engine, ExecEnv* exec_env);
@@ -63,8 +63,8 @@ class AgentServer {
     void stop_report_workers();
 
 private:
-    // Reference to the ExecEnv::_master_info
-    const TMasterInfo& _master_info;
+    // Reference to the ExecEnv::_cluster_info
+    const ClusterInfo* _cluster_info;
 
     std::unordered_map<int64_t /* TTaskType */, std::unique_ptr<TaskWorkerPoolIf>> _workers;