Merge branch 'main' into refactor-optimize-exprbloomFilter

Author: dantengsky

.config/version-schema.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "dependencies": {
+      "type": "object",
+      "properties": {
+        "databend-common-version": false
+      }
+    }
+  }
+}

.github/actions/build_linux/action.yml

@@ -30,7 +30,7 @@ runs:
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}
-        bypass_env_vars: RUSTFLAGS,RUST_LOG,DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,DATABEND_RELEASE_VERSION,DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY,DATABEND_ENTERPRISE_LICENSE_EMBEDDED,DATABEND_TELEMETRY_ENDPOINT,DATABEND_TELEMETRY_API_KEY
 
     - name: Cross setup
       if: startsWith(inputs.target, 'aarch64-')

.github/actions/pack_binaries/action.yml

@@ -5,7 +5,7 @@ inputs:
     description: "Release target"
     required: true
   category:
-    description: "Release default/hdfs/udf/testsuite/dbg"
+    description: "Release default/hdfs/udf/testsuite"
     required: false
     default: default
   version:
@@ -15,17 +15,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Download artifact for dbg
-      if: inputs.category == 'dbg'
-      uses: ./.github/actions/artifact_download
-      with:
-        sha: ${{ github.sha }}
-        target: ${{ inputs.target }}
-        category: default
-        path: distro/bin
-        artifacts: metactl,meta,query,query.debug,bendsave
     - name: Download artifact for others
-      if: inputs.category != 'dbg'
       uses: ./.github/actions/artifact_download
       with:
         sha: ${{ github.sha }}

.github/actions/publish_binary/action.yml

@@ -23,7 +23,7 @@ runs:
           default)
             publish_name="databend-${{ inputs.version }}-${{ inputs.target }}"
             ;;
-          hdfs|udf|testsuite|dbg)
+          hdfs|udf|testsuite)
             publish_name="databend-${{ inputs.category }}-${{ inputs.version }}-${{ inputs.target }}"
             ;;
           *)
@@ -41,6 +41,7 @@ runs:
 
     - name: Sync normal release to R2
       shell: bash
+      continue-on-error: true
       if: inputs.category == 'default'
       run: |
         aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress --checksum-algorithm=CRC32

.github/actions/publish_deb/action.yml

@@ -56,6 +56,7 @@ runs:
 
     - name: Publish to S3
       shell: bash
+      continue-on-error: true
       working-directory: scripts/distribution/deb
       run: |
         aws s3 sync dists s3://repo/deb/dists --delete --no-progress --checksum-algorithm=CRC32

.github/actions/setup_build_tool/action.yml

@@ -34,7 +34,7 @@ runs:
         export TARGET=${{ inputs.target }}
         export CARGO_INCREMENTAL=0
         export CARGO_TERM_COLOR=always
-        export BYPASS_ENV_VARS=${{ inputs.bypass_env_vars }},CARGO_INCREMENTAL,CARGO_TERM_COLOR,BUILD_PROFILE,JEMALLOC_SYS_WITH_LG_PAGE,DATABEND_RELEASE_VERSION
+        export BYPASS_ENV_VARS=${{ inputs.bypass_env_vars }},CARGO_INCREMENTAL,CARGO_TERM_COLOR,BUILD_PROFILE,JEMALLOC_SYS_WITH_LG_PAGE
         EOF
 
         RUNNER_PROVIDER="${RUNNER_PROVIDER:-github}"

.github/actions/setup_test/action.yml

@@ -5,6 +5,10 @@ inputs:
     description: "default to ./target/${BUILD_PROFILE}/"
     required: false
     default: ""
+  category:
+    description: "Category to download, only works with s3/gcs"
+    required: false
+    default: "full"
   artifacts:
     description: "Artifacts to download, only works with s3/gcs"
     required: false
@@ -25,7 +29,7 @@ runs:
       with:
         sha: ${{ github.sha }}
         target: ${{ steps.target.outputs.target }}
-        category: full
+        category: ${{ inputs.category }}
         artifacts: ${{ inputs.artifacts }}
         path: ${{ inputs.path }}
 

.github/actions/test_compat_fuse/action.yml

@@ -13,12 +13,11 @@ runs:
       run: |
           bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  1.2.46   --reader-version  current  --meta-versions  1.2.527  1.2.677  --logictest-path  base
           bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  1.2.241  --reader-version  current  --meta-versions  1.2.527  1.2.677  --logictest-path  revoke
-          bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  1.2.306  --reader-version  current  --meta-versions  1.2.527  1.2.677  --logictest-path  rbac
-          bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  1.2.307  --reader-version  current  --meta-versions  1.2.527  1.2.677  --logictest-path  rbac
+          bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  1.2.311  --reader-version  current  --meta-versions  1.2.527  1.2.677  --logictest-path  rbac
           bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  1.2.318  --reader-version  current  --meta-versions  1.2.527  1.2.677  --logictest-path  rbac
           bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  1.2.680  --reader-version  current  --meta-versions  1.2.527  1.2.680  --logictest-path  udf
 
-          bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  current  --reader-version  1.2.307  --meta-versions  1.2.677           --logictest-path  rbac
+          bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  current  --reader-version  1.2.311  --meta-versions  1.2.677           --logictest-path  rbac
           bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  current  --reader-version  1.2.318  --meta-versions  1.2.677           --logictest-path  rbac
           bash ./tests/compat_fuse/test_compat_fuse.sh  --writer-version  current  --reader-version  1.2.680  --meta-versions  1.2.680           --logictest-path  udf
     - name: Upload failure

.github/actions/test_compat_meta_meta/action.yml

@@ -13,8 +13,8 @@ runs:
       # test-*.sh <leader-meta-ver> <follower-meta-ver>
       # 1.2.288 is the first version after 1.2.212 that has built a binary.
       run: |
-        bash ./tests/compat/meta_meta/test_meta_meta.sh         1.2.288  current
-        bash ./tests/compat/meta_meta/test_meta_meta.sh         current  1.2.288
+        bash ./tests/compat/meta_meta/test_meta_meta.sh         1.2.547  current
+        bash ./tests/compat/meta_meta/test_meta_meta.sh         current  1.2.547
 
     - name: Upload failure
       if: failure()

.github/actions/test_logs/action.yml

@@ -5,11 +5,41 @@ runs:
   steps:
     - uses: ./.github/actions/setup_test
 
+    - name: Install lsof
+      shell: bash
+      run: sudo apt-get update -yq && sudo apt-get install -yq lsof
+
+    - name: Minio Setup for (ubuntu-latest only)
+      shell: bash
+      run: |
+        docker run -d --network host --name minio \
+                -e "MINIO_ACCESS_KEY=minioadmin" \
+                -e "MINIO_SECRET_KEY=minioadmin" \
+                -e "MINIO_ADDRESS=:9900" \
+                -v /tmp/data:/data \
+                -v /tmp/config:/root/.minio \
+                minio/minio server /data
+
+        export AWS_ACCESS_KEY_ID=minioadmin
+        export AWS_SECRET_ACCESS_KEY=minioadmin
+        export AWS_EC2_METADATA_DISABLED=true
+
+        aws --endpoint-url http://127.0.0.1:9900/ s3 mb s3://testbucket
+        aws --endpoint-url http://127.0.0.1:9900/ s3 cp tests/data s3://testbucket/data  --recursive --no-progress
+
     - name: Run logging Tests
       shell: bash
       run: |
         bash ./tests/logging/test-logs.sh
 
+    - name: Run History Tables Tests
+      env:
+        LOG_HISTORY_STORAGE_S3_ACCESS_KEY_ID: 'minioadmin'
+        LOG_HISTORY_STORAGE_S3_SECRET_ACCESS_KEY: 'minioadmin'
+      shell: bash
+      run: |
+        bash ./tests/logging/test-history-tables.sh
+
     - name: Upload failure
       if: failure()
       uses: ./.github/actions/artifact_failure