Use boost.interprocess to refactor mmap
#14219
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| - '*.*.*' | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| # The only difference between pull_request and pull_request_target is the context in which the workflow runs: | |
| # — pull_request_target workflows use the workflow files from the default branch, and secrets are available. | |
| # — pull_request workflows use the workflow files from the pull request branch, and secrets are unavailable. | |
| pull_request: | |
| types: [ synchronize, ready_for_review ] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| permissions: | |
| contents: read | |
| actions: read | |
| checks: write | |
| statuses: write | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| code_static_check: | |
| name: code_static_check | |
| if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci')) }} | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| steps: | |
| - name: Ensure workspace ownership | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "Workflow triggered by ${GITHUB_EVENT_NAME}" | |
| if [[ ${GITHUB_EVENT_NAME} == 'pull_request' ]]; then | |
| echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
| fi | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} | |
| fetch-depth: 0 | |
| fetch-tags: false | |
| submodules: recursive | |
| - name: Backup code | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| if [[ ${GITHUB_EVENT_NAME} != 'pull_request' ]]; then | |
| HEAD=$(git rev-parse HEAD) | |
| # Find a PR that introduced a given commit | |
| gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}" | |
| PR_NUMBER=$(gh pr list --search ${HEAD} --state merged --json number --jq .[0].number) | |
| echo "HEAD=${HEAD}" | |
| echo "PR_NUMBER=${PR_NUMBER}" | |
| if [[ -n ${PR_NUMBER} ]]; then | |
| PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER} | |
| if [[ -f ${PR_SHA_FP} ]]; then | |
| read -r PR_SHA PR_RUN_ID < "${PR_SHA_FP}" | |
| # Calculate the hash of the current workspace content | |
| HEAD_SHA=$(git rev-parse HEAD^{tree}) | |
| if [[ ${HEAD_SHA} == ${PR_SHA} ]]; then | |
| echo "The workspace content hash is the same with PR #${PR_NUMBER} merged. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${PR_RUN_ID} for details." | |
| echo "Upload coverage report of PR #${PR_NUMBER}" | |
| export ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${PR_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" | |
| export CODECOV_COMMIT_SHA=${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} | |
| echo "CODECOV_COMMIT_SHA=${CODECOV_COMMIT_SHA}" | |
| ./scripts/upload_codecov.sh | |
| echo "Cancel myself" | |
| gh run cancel ${GITHUB_RUN_ID} | |
| while true; do | |
| status=$(gh run view ${GITHUB_RUN_ID} --json status -q .status) | |
| [ "$status" = "completed" ] && break | |
| sleep 5 | |
| done | |
| exit 1 | |
| fi | |
| fi | |
| fi | |
| else | |
| PR_NUMBER=${{ github.event.pull_request.number }} | |
| PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER} | |
| # Calculate the hash of the current workspace content | |
| PR_SHA=$(git rev-parse HEAD^{tree}) | |
| echo "PR #${PR_NUMBER} workspace content hash: ${PR_SHA}" | |
| mkdir -p ${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY} | |
| echo "${PR_SHA} ${GITHUB_RUN_ID}" > ${PR_SHA_FP} | |
| fi | |
| tar --create --gzip --file=infinity.tgz .git $(git ls-files --recurse-submodules) | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| rm -rf ${ARTIFACTS_DIR} && mkdir -p ${ARTIFACTS_DIR} && cp infinity.tgz ${ARTIFACTS_DIR}/ | |
| # https://github.com/astral-sh/ruff-action | |
| - name: Static check with Ruff | |
| uses: astral-sh/ruff-action@v3 | |
| with: | |
| version: ">=0.14.x" | |
| args: "check --exclude=**/remote_thrift/infinity_thrift_rpc/**" | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| CHECK_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| echo "CHECK_CONTAINER=${CHECK_CONTAINER}" >> ${GITHUB_ENV} | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| sudo docker run --privileged --cap-add=NET_ADMIN -d --name ${CHECK_CONTAINER} -e TZ=${TZ} -e UV_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu20_clang20 | |
| - name: Check format of changed C/C++ files | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| if [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
| CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \ | |
| | grep -E '\.(cpp|h|hpp|cppm)$' \ | |
| | grep -v 'third_party/' \ | |
| | grep -v 'network/' \ | |
| | grep -v 'remote_thrift/infinity_thrift_rpc/' \ | |
| | grep -v 'parser/' || true) | |
| if [ -n "$CHANGED_FILES" ]; then | |
| echo "Check format of changed C/C++ files with clang-format-20" | |
| readarray -t files <<< "$CHANGED_FILES" | |
| HAS_ERROR=0 | |
| for file in "${files[@]}"; do | |
| if [ -f "$file" ]; then | |
| if sudo docker exec ${CHECK_CONTAINER} bash -c "cd /infinity; clang-format-20 $file > ${file}.formatted; diff $file ${file}.formatted 2>/dev/null"; then | |
| echo "✅ $file" | |
| else | |
| echo "❌ $file" | |
| HAS_ERROR=1 | |
| fi | |
| rm -f "${file}.formatted" | |
| fi | |
| done | |
| if [ $HAS_ERROR -ne 0 ]; then | |
| exit 1 | |
| fi | |
| else | |
| echo "No C++ files changed" | |
| fi | |
| fi | |
| - name: Remove container | |
| if: always() | |
| run: | | |
| if [[ -n "${CHECK_CONTAINER}" ]]; then | |
| sudo docker rm -f -v "${CHECK_CONTAINER}" | |
| fi | |
| debug_build: | |
| name: debug_build | |
| runs-on: [ "self-hosted", "infinity-build" ] | |
| needs: code_static_check | |
| steps: | |
| - name: Prepare code | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| tar --extract --gzip --file=${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> ${GITHUB_ENV} | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| sudo docker run --privileged -d --name ${BUILDER_CONTAINER} -e TZ=${TZ} -e UV_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu20_clang20 | |
| - name: Build debug version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCODE_COVERAGE=ON -S /infinity -B /infinity/cmake-build-debug && cmake --build /infinity/cmake-build-debug -j --target infinity test_main" | |
| - name: Upload artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: tar --create --file=- cmake-build-debug/src/{infinity,test_main} | tar --extract --directory=${ARTIFACTS_DIR} --file=- | |
| - name: Remove build container | |
| if: always() | |
| run: | | |
| if [[ -n "${BUILDER_CONTAINER}" ]]; then | |
| sudo docker rm -f -v "${BUILDER_CONTAINER}" | |
| fi | |
| debug_unit_test: | |
| name: debug_unit_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar --directory=${ARTIFACTS_DIR} --create --file=- cmake-build-debug/src/test_main | tar --extract --file=- | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Start minio container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| MINIO_VOLUME_PATH=${PWD}/${MINIO_CONTAINER} | |
| echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> ${GITHUB_ENV} | |
| echo "MINIO_VOLUME_PATH=${MINIO_VOLUME_PATH}" >> ${GITHUB_ENV} | |
| rm -rf ${MINIO_VOLUME_PATH} && mkdir -p ${MINIO_VOLUME_PATH} | |
| sudo docker run -d --name ${MINIO_CONTAINER} --network container:$(hostname) -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_VOLUME_PATH}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
| sleep 5s | |
| if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
| echo "Minio container is not running" | |
| exit 1 | |
| fi | |
| - name: Unit test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| export LLVM_PROFILE_FILE="infinity-%p.profraw" | |
| ASAN_OPTIONS=detect_leaks=0 cmake-build-debug/src/test_main --gtest_also_run_disabled_tests --gtest_filter=-*SLOW_* > debug_unit_test_stdout.log 2> debug_unit_test_stderr.log | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| echo "debug unit test log" && tail -n 1000 debug_unit_test*.log | |
| mkdir -p ${ARTIFACTS_DIR}/debug_unit_test/ && cp debug_unit_test*.log ${ARTIFACTS_DIR}/debug_unit_test/ | |
| - name: Generate LCOV report | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| llvm-profdata-20 merge -sparse infinity-*.profraw -o coverage.profdata && llvm-cov-20 export cmake-build-debug/src/test_main -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*unit_test.*|.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov | |
| head -10 coverage.lcov | |
| echo "LCOV file lines: $(wc -l < coverage.lcov)" | |
| mkdir -p ${ARTIFACTS_DIR}/coverage/debug_unit_test && cp coverage.lcov ${ARTIFACTS_DIR}/coverage/debug_unit_test | |
| - name: Remove minio container | |
| if: always() | |
| run: | | |
| if [[ -n "${MINIO_CONTAINER}" ]]; then sudo docker rm -f -v "${MINIO_CONTAINER}"; fi | |
| if [[ -n "${MINIO_VOLUME_PATH}" ]]; then sudo rm -rf "${MINIO_VOLUME_PATH}"; fi | |
| debug_sqllogical_test: | |
| name: debug_sqllogical_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar --extract --gzip --file=${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar --directory=${ARTIFACTS_DIR} --create --file=- cmake-build-debug/src/infinity | tar --extract --file=- | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| export LLVM_PROFILE_FILE="infinity-%p.profraw" | |
| export LLVM_PROFILE_DUMP_ON_EXIT=1 | |
| nohup cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > /dev/null 2> debug_sqllogictest_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run sqllogictest | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/sqllogictest.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/debug_sqllogic_test/ && cp debug_sqllogictest*.log ${ARTIFACTS_DIR}/debug_sqllogic_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/debug_sqllogic_test/ | |
| echo "debug sqllogictest log" && tail -n 1000 debug_sqllogictest*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| - name: Generate LCOV report | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| timeout 30s bash -c ' | |
| until find . -name "infinity*.profraw" -size +0 | grep -q .; do | |
| sleep 1s | |
| done | |
| ' || echo "Warning: infinity*.profraw is still empty after 30s" | |
| ls -lh infinity*.profraw | |
| llvm-profdata-20 merge -sparse infinity-*.profraw -o coverage.profdata && llvm-cov-20 export cmake-build-debug/src/infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov | |
| head -10 coverage.lcov | |
| echo "LCOV file lines: $(wc -l < coverage.lcov)" | |
| mkdir -p ${ARTIFACTS_DIR}/coverage/debug_sqllogical_test && cp coverage.lcov ${ARTIFACTS_DIR}/coverage/debug_sqllogical_test | |
| debug_restart_test: | |
| name: debug_restart_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-debug/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Run restart test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv run python3 tools/run_restart_test.py --infinity_path=cmake-build-debug/src/infinity | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/debug_restart_test && cp restart_test*.log* ${ARTIFACTS_DIR}/debug_restart_test/ | |
| echo "debug_restart_test log(last 1000 lines):" && tail -n 1000 restart_test*.log* | |
| debug_pysdk_test: | |
| name: debug_pysdk_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-debug/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| export LLVM_PROFILE_FILE="infinity-%p.profraw" | |
| export LLVM_PROFILE_DUMP_ON_EXIT=1 | |
| nohup cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > /dev/null 2> debug_pysdk_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run pysdk test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_pysdk_remote_infinity_test.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/debug_pysdk_test && cp debug_pysdk*.log ${ARTIFACTS_DIR}/debug_pysdk_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/debug_pysdk_test/ | |
| echo "debug pysdk log" && tail -n 1000 debug_pysdk*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| - name: Generate LCOV report | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| timeout 30s bash -c ' | |
| until find . -name "infinity*.profraw" -size +0 | grep -q .; do | |
| sleep 1s | |
| done | |
| ' || echo "Warning: infinity*.profraw is still empty after 30s" | |
| ls -lh infinity*.profraw | |
| llvm-profdata-20 merge -sparse infinity-*.profraw -o coverage.profdata && llvm-cov-20 export cmake-build-debug/src/infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov | |
| head -10 coverage.lcov | |
| echo "LCOV file lines: $(wc -l < coverage.lcov)" | |
| mkdir -p ${ARTIFACTS_DIR}/coverage/debug_pysdk_test && cp coverage.lcov ${ARTIFACTS_DIR}/coverage/debug_pysdk_test | |
| debug_http_test: | |
| name: debug_http_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-debug/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| export LLVM_PROFILE_FILE="infinity-%p.profraw" | |
| export LLVM_PROFILE_DUMP_ON_EXIT=1 | |
| nohup cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > /dev/null 2> debug_http_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run http test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_http_api.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/debug_http_test && cp debug_http*.log ${ARTIFACTS_DIR}/debug_http_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/debug_http_test/ | |
| echo "debug http log" && tail -n 1000 debug_http*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| - name: Generate LCOV report | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| timeout 30s bash -c ' | |
| until find . -name "infinity*.profraw" -size +0 | grep -q .; do | |
| sleep 1s | |
| done | |
| ' || echo "Warning: infinity*.profraw is still empty after 30s" | |
| ls -lh infinity*.profraw | |
| llvm-profdata-20 merge -sparse infinity-*.profraw -o coverage.profdata && llvm-cov-20 export cmake-build-debug/src/infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov | |
| head -10 coverage.lcov | |
| echo "LCOV file lines: $(wc -l < coverage.lcov)" | |
| mkdir -p ${ARTIFACTS_DIR}/coverage/debug_http_test && cp coverage.lcov ${ARTIFACTS_DIR}/coverage/debug_http_test | |
| debug_pysdk_parallel_test: | |
| name: debug_pysdk_parallel_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: | |
| - debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-debug/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| export LLVM_PROFILE_FILE="infinity-%p.profraw" | |
| export LLVM_PROFILE_DUMP_ON_EXIT=1 | |
| nohup cmake-build-debug/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > /dev/null 2> debug_parallel_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run pysdk test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_parallel_test.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/debug_parallel_test && cp debug_parallel_stderr.log ${ARTIFACTS_DIR}/debug_parallel_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/debug_parallel_test/ | |
| echo "debug parallel log" && tail -n 1000 debug_parallel*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| - name: Generate LCOV report | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| timeout 30s bash -c ' | |
| until find . -name "infinity*.profraw" -size +0 | grep -q .; do | |
| sleep 1s | |
| done | |
| ' || echo "Warning: infinity*.profraw is still empty after 30s" | |
| ls -lh infinity*.profraw | |
| llvm-profdata-20 merge -sparse infinity-*.profraw -o coverage.profdata && llvm-cov-20 export cmake-build-debug/src/infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov | |
| head -10 coverage.lcov | |
| echo "LCOV file lines: $(wc -l < coverage.lcov)" | |
| mkdir -p ${ARTIFACTS_DIR}/coverage/debug_parallel_test && cp coverage.lcov ${ARTIFACTS_DIR}/coverage/debug_parallel_test | |
| debug_upload_codecov: | |
| name: debug_upload_codecov | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: [debug_unit_test, debug_sqllogical_test, debug_pysdk_test, debug_http_test,debug_pysdk_parallel_test] | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-debug/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Upload coverage reports to Codecov | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| export CODECOV_COMMIT_SHA=${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} | |
| echo "CODECOV_COMMIT_SHA=${CODECOV_COMMIT_SHA}" | |
| export GITHUB_REF_NAME=${{ github.ref_name }} | |
| echo "GITHUB_REF_NAME=${GITHUB_REF_NAME}" | |
| export PR_NUMBER=${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.number || '' }} | |
| echo "PR_NUMBER=${PR_NUMBER}" | |
| ./scripts/upload_codecov.sh | |
| reldeb_build: | |
| name: reldeb_build | |
| runs-on: [ "self-hosted", "infinity-build" ] | |
| needs: code_static_check | |
| steps: | |
| - name: Prepare code | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> ${GITHUB_ENV} | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| sudo docker run --privileged -d --name ${BUILDER_CONTAINER} -e TZ=${TZ} -e UV_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu20_clang20 | |
| - name: Build reldeb version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-reldeb && mkdir -p cmake-build-reldeb && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S /infinity -B /infinity/cmake-build-reldeb && cmake --build /infinity/cmake-build-reldeb -j --target infinity test_main knn_import_benchmark knn_query_benchmark" | |
| - name: Upload artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: tar -cf - cmake-build-reldeb/src/{infinity,test_main} cmake-build-reldeb/benchmark/local_infinity/{knn_import_benchmark,knn_query_benchmark} | tar -C ${ARTIFACTS_DIR} -xf - | |
| - name: Remove build container | |
| if: always() | |
| run: | | |
| if [[ -n "${BUILDER_CONTAINER}" ]]; then | |
| sudo docker rm -f -v "${BUILDER_CONTAINER}" | |
| fi | |
| reldeb_unit_test: | |
| name: reldeb_unit_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/src/test_main | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Start minio container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| MINIO_VOLUME_PATH=${PWD}/${MINIO_CONTAINER} | |
| echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> ${GITHUB_ENV} | |
| echo "MINIO_VOLUME_PATH=${MINIO_VOLUME_PATH}" >> ${GITHUB_ENV} | |
| rm -rf ${MINIO_VOLUME_PATH} && mkdir -p ${MINIO_VOLUME_PATH} | |
| sudo docker run -d --name ${MINIO_CONTAINER} --network container:$(hostname) -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_VOLUME_PATH}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
| sleep 5s | |
| if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
| echo "Minio container is not running" | |
| exit 1 | |
| fi | |
| - name: Unit test reldeb version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| cmake-build-reldeb/src/test_main --gtest_also_run_disabled_tests --gtest_filter=-*SLOW_* > reldeb_unit_test_stdout.log 2> reldeb_unit_test_stderr.log | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_unit_test && cp reldeb_unit_test*.log ${ARTIFACTS_DIR}/reldeb_unit_test/ | |
| echo "reldeb unit test log" && tail -n 1000 reldeb_unit_test*.log | |
| - name: Remove minio container | |
| if: always() | |
| run: | | |
| if [[ -n "${MINIO_CONTAINER}" ]]; then sudo docker rm -f -v "${MINIO_CONTAINER}"; fi | |
| if [[ -n "${MINIO_VOLUME_PATH}" ]]; then sudo rm -rf "${MINIO_VOLUME_PATH}"; fi | |
| reldeb_sqllogical_test: | |
| name: reldeb_sqllogical_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| nohup cmake-build-reldeb/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > /dev/null 2> reldeb_sqllogic_test_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run sqllogictest | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/sqllogictest.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_sqllogic_test && cp reldeb_sqllogic_test*.log ${ARTIFACTS_DIR}/reldeb_sqllogic_test/ | |
| cp reldeb_sqllogic_test* ${ARTIFACTS_DIR}/reldeb_sqllogic_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/reldeb_sqllogic_test/ | |
| echo "reldeb sqllogic test log" && tail -n 1000 reldeb_sqllogic_test*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| reldeb_pysdk_test: | |
| name: reldeb_pysdk_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| nohup cmake-build-reldeb/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > /dev/null 2> reldeb_pysdk_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run pysdk test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_pysdk_remote_infinity_test.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_pysdk_test && cp reldeb_pysdk*.log ${ARTIFACTS_DIR}/reldeb_pysdk_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/reldeb_pysdk_test/ | |
| echo "reldeb pysdk log" && tail -n 1000 reldeb_pysdk*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| reldeb_http_test: | |
| name: reldeb_http_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| nohup cmake-build-reldeb/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > /dev/null 2> reldeb_http_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run http api test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_http_api.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_http_test && cp reldeb_http*.log ${ARTIFACTS_DIR}/reldeb_http_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/reldeb_http_test/ | |
| echo "reldeb http log" && tail -n 1000 reldeb_http*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| reldeb_pysdk_parallel_test: | |
| name: reldeb_pysdk_parallel_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| nohup cmake-build-reldeb/src/infinity --config=conf/pytest_parallel_infinity_conf.toml > /dev/null 2> reldeb_pysdk_parallel_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run parallel test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_parallel_test.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_pysdk_parallel_test && cp reldeb_pysdk_parallel*.log ${ARTIFACTS_DIR}/reldeb_pysdk_parallel_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/reldeb_pysdk_parallel_test/ | |
| echo "reldeb parallel log" && tail -n 1000 reldeb_pysdk_parallel*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| reldeb_minio_pysdk_test: | |
| name: reldeb_minio_pysdk_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start minio container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| MINIO_VOLUME_PATH=${PWD}/${MINIO_CONTAINER} | |
| echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> ${GITHUB_ENV} | |
| echo "MINIO_VOLUME_PATH=${MINIO_VOLUME_PATH}" >> ${GITHUB_ENV} | |
| rm -rf ${MINIO_VOLUME_PATH} && mkdir -p ${MINIO_VOLUME_PATH} | |
| sudo docker run -d --name ${MINIO_CONTAINER} --network container:$(hostname) -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_VOLUME_PATH}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
| sleep 5s | |
| if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
| echo "Minio container is not running" | |
| exit 1 | |
| fi | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| nohup cmake-build-reldeb/src/infinity --config=conf/pytest_parallel_infinity_minio.toml > /dev/null 2> reldeb_minio_pysdk_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run minio pysdk test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_pysdk_remote_infinity_test.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_minio_pysdk_test && cp reldeb_minio_pysdk*.log ${ARTIFACTS_DIR}/reldeb_minio_pysdk_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/reldeb_minio_pysdk_test/ | |
| echo "reldeb minio pysdk log" && tail -n 1000 reldeb_minio_pysdk*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| - name: Remove minio container | |
| if: always() | |
| run: | | |
| if [[ -n "${MINIO_CONTAINER}" ]]; then sudo docker rm -f -v "${MINIO_CONTAINER}"; fi | |
| if [[ -n "${MINIO_VOLUME_PATH}" ]]; then sudo rm -rf "${MINIO_VOLUME_PATH}"; fi | |
| reldeb_minio_parallel_test: | |
| name: reldeb_minio_parallel_test | |
| runs-on: [ "self-hosted", "infinity-test" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start minio container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| MINIO_VOLUME_PATH=${PWD}/${MINIO_CONTAINER} | |
| echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> ${GITHUB_ENV} | |
| echo "MINIO_VOLUME_PATH=${MINIO_VOLUME_PATH}" >> ${GITHUB_ENV} | |
| rm -rf ${MINIO_VOLUME_PATH} && mkdir -p ${MINIO_VOLUME_PATH} | |
| sudo docker run -d --name ${MINIO_CONTAINER} --network container:$(hostname) -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_VOLUME_PATH}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
| sleep 5s | |
| if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
| echo "Minio container is not running" | |
| exit 1 | |
| fi | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| nohup cmake-build-reldeb/src/infinity --config=conf/pytest_parallel_infinity_minio.toml > /dev/null 2> reldeb_minio_parallel_stderr.log & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run minio parallel test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_parallel_test.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_minio_parallel_test && cp reldeb_minio_parallel*.log ${ARTIFACTS_DIR}/reldeb_minio_parallel_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/reldeb_minio_parallel_test/ | |
| echo "reldeb minio parallel log" && tail -n 1000 reldeb_minio_parallel*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| - name: Remove minio container | |
| if: always() | |
| run: | | |
| if [[ -n "${MINIO_CONTAINER}" ]]; then sudo docker rm -f -v "${MINIO_CONTAINER}"; fi | |
| if [[ -n "${MINIO_VOLUME_PATH}" ]]; then sudo rm -rf "${MINIO_VOLUME_PATH}"; fi | |
| reldeb_benchmark: | |
| name: reldeb_benchmark | |
| runs-on: [ "self-hosted", "benchmark" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/benchmark/local_infinity/{knn_import_benchmark,knn_query_benchmark} | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Prepare sift dataset | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}} | |
| echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> ${GITHUB_ENV} | |
| chmod +x ./tools/ci_tools/check_benchmark_result.py | |
| rm -rf test/data/benchmark && ln -s ${RUNNER_WORKSPACE_PREFIX}/benchmark test/data/benchmark | |
| - name: Golden benchmark sift | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| if [[ -f "${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log" ]]; then | |
| echo "golden benchmark result already exists!" | |
| exit 0 | |
| fi | |
| rm -rf ${PWD}/db_tmp && mkdir -p ${PWD}/db_tmp && cat ${RUNNER_WORKSPACE_PREFIX}/benchmark/infinity_conf.toml | sed -e "s|/var/infinity|${PWD}/db_tmp|g" > ${PWD}/db_tmp/infinity_conf.toml | |
| ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_import_benchmark --dataset=sift --dataset_dir=${PWD}/test/data --infinity_dir=${PWD}/db_tmp | |
| echo "1 50" | ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_query_benchmark sift 200 false ${PWD}/test/data ${PWD}/db_tmp | sudo tee ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log | |
| echo "8 50" | ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_query_benchmark sift 200 false ${PWD}/test/data ${PWD}/db_tmp | sudo tee ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_8_threads.log | |
| - name: Latest benchmark sift | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| rm -rf ${PWD}/db_tmp && mkdir -p ${PWD}/db_tmp && cat conf/infinity_conf.toml | sed -e "s|/var/infinity|${PWD}/db_tmp|g" > ${PWD}/db_tmp/infinity_conf.toml | |
| ./cmake-build-reldeb/benchmark/local_infinity/knn_import_benchmark --dataset=sift --dataset_dir=${PWD}/test/data --infinity_dir=${PWD}/db_tmp --config_path=db_tmp/infinity_conf.toml | |
| echo "1 50" | ./cmake-build-reldeb/benchmark/local_infinity/knn_query_benchmark sift 200 false ${PWD}/test/data ${PWD}/db_tmp db_tmp/infinity_conf.toml | tee benchmark_sift_1_thread.log | |
| echo "8 50" | ./cmake-build-reldeb/benchmark/local_infinity/knn_query_benchmark sift 200 false ${PWD}/test/data ${PWD}/db_tmp db_tmp/infinity_conf.toml | tee benchmark_sift_8_threads.log | |
| ./tools/ci_tools/check_benchmark_result.py ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log benchmark_sift_1_thread.log | |
| ./tools/ci_tools/check_benchmark_result.py ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_8_threads.log benchmark_sift_8_threads.log | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| sudo rm -rf ${PWD}/db_tmp |