Merge branch 'main' into 787-figure-out-queue-block

Author: BohuTANG

.github/actions/build_bindings_python/action.yml

@@ -16,16 +16,16 @@ runs:
       shell: bash
       run: |
         raw_version="${{ inputs.version }}"
-        # Remove v prefix: v1.2.3-nightly -> 1.2.3-nightly
-        version_no_v=${raw_version/v/}
-        # Convert to Python-compatible version: 1.2.3-nightly -> 1.2.3.dev0, 1.2.3-p1 -> 1.2.3.post1
-        VERSION=$(echo "$version_no_v" | sed 's/-nightly/.dev0/' | sed 's/-p\([0-9]*\)/.post\1/')
+        # Remove v prefix and suffixes: v1.2.809-nightly -> 1.2.809
+        VERSION=$(echo "$raw_version" | sed 's/^v//' | sed 's/-nightly//' | sed 's/-p[0-9]*//')
+
         echo "building version: $raw_version -> $VERSION"
-        sed "s#version = \"0.1.0\"#version = \"$VERSION\"#g" Cargo.toml > Cargo.toml.bak
-        sed "s#version = \"0.1.0\"#version = \"$VERSION\"#g" pyproject.toml > pyproject.toml.bak
+        # Replace any existing version in Cargo.toml and pyproject.toml
+        sed "s/^version = \".*\"/version = \"$VERSION\"/" Cargo.toml > Cargo.toml.tmp
+        sed "s/^version = \".*\"/version = \"$VERSION\"/" pyproject.toml > pyproject.toml.tmp
 
-        mv Cargo.toml.bak Cargo.toml
-        mv pyproject.toml.bak pyproject.toml
+        mv Cargo.toml.tmp Cargo.toml
+        mv pyproject.toml.tmp pyproject.toml
 
         echo "version in Cargo.toml: $(grep 'version' Cargo.toml)"
         echo "version in pyproject.toml: $(grep 'version' pyproject.toml)"
@@ -80,7 +80,8 @@ runs:
           # Install cargo-zigbuild manually to avoid musl target issues
           cargo install cargo-zigbuild --target ${{ inputs.target }}
           ../../scripts/setup/dev_setup.sh -yb
-          uv venv --python=python3.12
+          # Create virtual environment for build dependencies (overwrite if exists)
+          uv venv .venv --python=python3.12 --clear
           uv sync --all-groups --all-extras
 
     - name: Setup Rust for development builds
@@ -97,8 +98,8 @@ runs:
       run: |
         echo "Building development wheel for testing..."
 
-        # Create and activate virtual environment
-        uv venv --python python3.12
+        # Create virtual environment (overwrite if exists)
+        uv venv .venv --python python3.12 --clear
         source .venv/bin/activate
 
         # Install development dependencies
@@ -116,22 +117,3 @@ runs:
         python -m pytest tests/ -v --tb=short
 
         echo "All Python binding tests passed!"
-
-    - name: Test built wheels
-      if: inputs.version != ''
-      shell: bash
-      working-directory: src/bendpy
-      run: |
-        echo "Testing built wheels..."
-
-        # Install from built wheel for release testing
-        uv venv --python python3.12
-        source .venv/bin/activate
-        uv pip install dist/*.whl
-        uv pip install pytest pandas polars pyarrow
-
-        # Run pytest tests
-        echo "Executing pytest tests..."
-        python -m pytest tests/ -v --tb=short
-
-        echo "All Python binding tests passed!"

.github/actions/test_private_tasks/action.yml

@@ -17,7 +17,7 @@ runs:
       shell: bash
       run: |
         bash ./tests/task/test-private-task.sh
-        bash ./tests/task/test-private-task-warehouse.sh
+#        bash ./tests/task/test-private-task-warehouse.sh
 
     - name: Upload failure
       if: failure()

.github/workflows/bindings.python.yml

@@ -1,20 +1,7 @@
-name: Bindings Python
+name: Python Package Release
 
 on:
-  ## uncomment it when bendpy is enabled
   workflow_dispatch:
-    inputs:
-      version:
-        description: Version to release (optional for testing)
-        required: false
-        type: string
-        default: "test-build"
-  pull_request:
-    branches:
-      - main
-    paths:
-      - "src/**"
-      - ".github/workflows/bindings.python.yml"
   workflow_call:
     inputs:
       version:
@@ -33,9 +20,43 @@ permissions:
   packages: write
 
 jobs:
+  get-version:
+    name: Determine Version
+    if: (github.event_name == 'workflow_call' && inputs.version != '') || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Get and validate version
+        id: version
+        shell: bash
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.1.0")
+            echo "Using latest tag for manual trigger: $VERSION"
+          elif [[ "${{ github.event_name }}" == "workflow_call" && -n "${{ inputs.version }}" ]]; then
+            VERSION="${{ inputs.version }}"
+            echo "Using provided version from workflow_call: $VERSION"
+          else
+            echo "Error: No version provided for workflow_call"
+            exit 1
+          fi
+
+          # Validate version format
+          if [[ ! $VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then
+            echo "Warning: Version $VERSION may not follow semantic versioning (vX.Y.Z)"
+          fi
+
+          echo "Final version: $VERSION"
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
   test:
-    # Run tests on all PRs and workflow dispatches
-    if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+    name: Run Tests
+    if: (github.event_name == 'workflow_call' && inputs.version != '') || github.event_name == 'workflow_dispatch'
+    needs: [get-version]
     runs-on:
       - self-hosted
       - X64
@@ -52,79 +73,103 @@ jobs:
           # No version means development build with tests
 
   linux:
-    # Only run for version builds (releases)
-    if: inputs.version
+    name: Build Linux Wheels
+    if: (github.event_name == 'workflow_call' && inputs.version != '') || github.event_name == 'workflow_dispatch'
+    needs: [get-version, test]
     runs-on:
       - self-hosted
       - "${{ matrix.runner }}"
       - Linux
       - 4c16g
       - aws
     strategy:
+      fail-fast: false
       matrix:
         include:
           - { arch: x86_64, runner: X64 }
-          # - { arch: aarch64, runner: ARM64 }
+          - { arch: aarch64, runner: ARM64 }
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: ./.github/actions/build_bindings_python
         with:
           target: ${{ matrix.arch }}-unknown-linux-gnu
-          version: ${{ inputs.version }}
-      - name: upload
-        if: inputs.version
+          version: ${{ needs.get-version.outputs.version }}
+      - name: Upload Linux wheel
         uses: actions/upload-artifact@v4
         with:
           name: python-linux-${{ matrix.arch }}
           path: src/bendpy/dist/*.whl
+          retention-days: 7
 
-  # macos:
-  #   if: inputs.version
-  #   runs-on: macos-latest
-  #   strategy:
-  #     matrix:
-  #       arch:
-  #         - aarch64
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #       with:
-  #         fetch-depth: 0
-  #     - uses: ./.github/actions/build_bindings_python
-  #       with:
-  #         target: ${{ matrix.arch }}-apple-darwin
-  #         version: ${{ inputs.version }}
-  #     - name: upload
-  #       if: inputs.version
-  #       uses: actions/upload-artifact@v4
-  #       with:
-  #         name: python-macos-${{ matrix.arch }}
-  #         path: src/bendpy/dist/*.whl
+  macos:
+    name: Build macOS Wheels
+    if: (github.event_name == 'workflow_call' && inputs.version != '') || github.event_name == 'workflow_dispatch'
+    needs: [get-version, test]
+    runs-on: macos-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install dependencies
+        run: |
+          # Install OpenSSL and necessary tools
+          brew install openssl@3
+
+          # Use vendored OpenSSL to avoid cross-compilation issues
+          echo "OPENSSL_STATIC=1" >> $GITHUB_ENV
+          echo "OPENSSL_VENDORED=1" >> $GITHUB_ENV
+          echo "PKG_CONFIG_ALLOW_CROSS=1" >> $GITHUB_ENV
+
+      - uses: ./.github/actions/build_bindings_python
+        with:
+          target: ${{ matrix.target }}
+          version: ${{ needs.get-version.outputs.version }}
+      - name: Upload macOS wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-macos-${{ matrix.target }}
+          path: src/bendpy/dist/*.whl
+          retention-days: 7
 
   publish:
-    if: inputs.version
-    name: Publish
-    needs: [linux]
+    name: Publish to PyPI
+    if: (github.event_name == 'workflow_call' && inputs.version != '') || github.event_name == 'workflow_dispatch'
+    needs: [get-version, test, linux]
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      pull-requests: write
-      contents: read
-      packages: write
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
         with:
           pattern: python-*
           merge-multiple: true
           path: src/bendpy/dist
+        continue-on-error: true
+
+      - name: Show packages to publish
+        run: |
+          echo "Publishing packages for version: ${{ needs.get-version.outputs.version }}"
+          echo "Packages found:"
+          ls -la src/bendpy/dist/ || echo "No packages found"
+          echo "Total packages: $(ls src/bendpy/dist/*.whl 2>/dev/null | wc -l)"
 
       - name: Publish to PyPI
         timeout-minutes: 10
         run: |
           pip install twine
-          twine upload --skip-existing --verbose src/bendpy/dist/*.whl
+          echo "Publishing to PyPI..."
+          if [ -n "$(find src/bendpy/dist -name "*.whl" 2>/dev/null)" ]; then
+            twine upload --skip-existing --verbose src/bendpy/dist/*.whl
+          else
+            echo "No wheel files found to publish"
+            exit 1
+          fi
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

Cargo.lock

@@ -10,4 +10,4 @@ echo "🔧 Installing databend_test_helper..."
 pip install -e "$PROJECT_DIR/databend_test_helper"
 
 echo "🚀 Running test cluster..."
-python "$SCRIPT_DIR/test_cluster.py"
+python "$SCRIPT_DIR/test_cluster.py"

scripts/test-bend-tests/main.sh

@@ -25,13 +25,16 @@ databend-common-exception = { workspace = true }
 databend-common-expression = { workspace = true }
 databend-common-license = { workspace = true }
 databend-common-meta-app = { workspace = true }
+databend-common-telemetry = { workspace = true }
 databend-common-tracing = { workspace = true }
 databend-common-version = { workspace = true }
 databend-query = { workspace = true, features = [
     "simd",
     "disable_initial_exec_tls",
 ] }
 pyo3 = { version = "0.24", features = ["generate-import-lib", "abi3-py312"] }
+serde_json = { workspace = true }
+sysinfo = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync"] }
 tokio-stream = { workspace = true }