[Feature] Config eviction_duration #49
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CE Compile Job | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - develop | |
| - 'release/*' | |
| permissions: read-all | |
| concurrency: | |
| group: CE-Job-${{ github.ref }}-${{ github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| ce_job_pre_check: | |
| runs-on: ubuntu-latest | |
| env: | |
| COMPILE_BRANCH: ${{ vars.COMPILE_BRANCH }} | |
| CE_COMPILE_SELECTION: ${{ vars.CE_COMPILE_SELECTION }} | |
| COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ vars.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }} | |
| outputs: | |
| branch_match: ${{ steps.set_output.outputs.branch_match }} | |
| compile_use_paddle_whl_url: ${{ steps.set_output.outputs.compile_use_paddle_whl_url }} | |
| sm8689_match: ${{ steps.set_output.outputs.sm8689_match }} | |
| sm8090_match: ${{ steps.set_output.outputs.sm8090_match }} | |
| steps: | |
| - name: Set Version | |
| id: set_output | |
| env: | |
| COMPILE_BRANCH: ${{ env.COMPILE_BRANCH }} | |
| CE_COMPILE_SELECTION: ${{ env.CE_COMPILE_SELECTION }} | |
| COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ env.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }} | |
| GITHUB_REF_NAME: ${{ github.ref_name }} | |
| run: | | |
| # 选择要触发编译任务的分支 done | |
| # 选择指定分支要编译的任务 8090或者8689 | |
| # 指定分支编译要使用的Paddle的安装包,默认使用nightly最新的 | |
| IFS=',' read -ra BRANCHES <<< "$COMPILE_BRANCH" | |
| MATCH=false | |
| for b in "${BRANCHES[@]}"; do | |
| if [[ "$b" == "${GITHUB_REF_NAME}" ]]; then | |
| MATCH=true | |
| break | |
| fi | |
| done | |
| echo "branch_match=$MATCH" >> $GITHUB_OUTPUT | |
| # 通过变量CE_COMPILE_SELECTION中的映射关系,决定分支是编译sm8090还是sm8689 | |
| for pair in $(echo "$CE_COMPILE_SELECTION" | tr ';' ' '); do | |
| branch=$(echo "$pair" | cut -d',' -f1) | |
| compile_task_list=$(echo "$pair" | cut -d',' -f2) | |
| if [[ "$branch" == "$GITHUB_REF_NAME" ]]; then | |
| # 判断里面是否包含 sm8090 或 sm8689 | |
| if [[ "$compile_task_list" == *"sm8090"* ]]; then | |
| echo "sm8090_match=true" >> $GITHUB_OUTPUT | |
| fi | |
| if [[ "$compile_task_list" == *"sm8689"* ]]; then | |
| echo "sm8689_match=true" >> $GITHUB_OUTPUT | |
| fi | |
| break | |
| fi | |
| done | |
| # 通过变量COMPILE_USE_PADDLE_WHL_URL_MAPPINGS中的映射关系,决定是否是安装指定版本的Paddle还是直接安装URL | |
| for pair in $(echo $COMPILE_USE_PADDLE_WHL_URL_MAPPINGS | tr ';' ' '); do | |
| branch=$(echo "$pair" | cut -d',' -f1) | |
| paddle_whl_url=$(echo "$pair" | cut -d',' -f2) | |
| if [[ "$branch" == "${{ github.ref_name }}" ]]; then | |
| FOUND_PADDLE_URL="$paddle_whl_url" | |
| echo "compile_use_paddle_whl_url=${FOUND_PADDLE_URL}" >> $GITHUB_OUTPUT | |
| break | |
| fi | |
| done | |
| print_ce_job_pre_check_outputs: | |
| runs-on: ubuntu-latest | |
| needs: ce_job_pre_check | |
| steps: | |
| - name: Print outputs as JSON | |
| run: | | |
| echo '${{ toJSON(needs.ce_job_pre_check.outputs) }}' | |
| clone: | |
| environment: CodeSync | |
| name: FD-Clone-Linux | |
| runs-on: ubuntu-latest | |
| needs: ce_job_pre_check | |
| if: ${{ needs.ce_job_pre_check.outputs.branch_match == 'true' }} | |
| outputs: | |
| repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }} | |
| steps: | |
| - name: Clone FastDeploy | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' | |
| && github.event.pull_request.base.ref | |
| || github.ref_name }} | |
| submodules: 'recursive' | |
| fetch-depth: 1000 | |
| - name: Python Setup | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.10' | |
| - name: Code Info Show and Upload | |
| id: set_output | |
| env: | |
| AK: ${{ secrets.BOS_AK }} | |
| SK: ${{ secrets.BOS_SK }} | |
| run: | | |
| git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'" | |
| echo "Current HEAD Log:" | |
| git log --oneline -n 5 | |
| ls | |
| cd .. | |
| tar -zcf FastDeploy.tar.gz FastDeploy | |
| commit_id=${{ github.sha }} | |
| branch_name=${{ github.ref_name }} | |
| wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py | |
| push_file=$(realpath bos_tools.py) | |
| python -m pip install bce-python-sdk==0.9.29 | |
| filename="FastDeploy.tar.gz" | |
| target_paths=( | |
| "paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id}" | |
| "paddle-qa/BRANCH/FastDeploy/${branch_name}/latest" | |
| ) | |
| for target_path in "${target_paths[@]}"; do | |
| echo "Uploading ${filename} to ${target_path}" | |
| python "${push_file}" "${filename}" "${target_path}" | |
| done | |
| base_prefix="paddle-qa/" | |
| commit_path_stripped="${target_paths[0]#${base_prefix}}" | |
| latest_path_stripped="${target_paths[1]#${base_prefix}}" | |
| REPO_ARCHIVE_URL="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}" | |
| LATEST_REPO_ARCHIVE_URL="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}" | |
| echo "commit archive url is ${REPO_ARCHIVE_URL}" | |
| echo "latest archive url is ${LATEST_REPO_ARCHIVE_URL}" | |
| echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT | |
| resultshow: | |
| name: Show Code Archive Output | |
| needs: clone | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Print repo_archive_url path | |
| run: | | |
| echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}" | |
| build_sm8090: | |
| name: BUILD_SM8090 | |
| needs: [clone, ce_job_pre_check] | |
| if: ${{ needs.ce_job_pre_check.outputs.sm8090_match == 'true' }} | |
| uses: ./.github/workflows/_build_linux.yml | |
| with: | |
| DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda126-manylinux | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| COMPILE_ARCH: "80,90" | |
| WITH_NIGHTLY_BUILD: OFF | |
| FD_VERSION: 0.0.0 | |
| PADDLE_WHL_URL: ${{ needs.ce_job_pre_check.outputs.compile_use_paddle_whl_url }} | |
| secrets: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| build_sm8090_rl: | |
| name: BUILD_SM8090_RL | |
| needs: [clone, ce_job_pre_check] | |
| if: ${{ needs.ce_job_pre_check.outputs.sm8090_match == 'true' }} | |
| uses: ./.github/workflows/_build_linux_rl.yml | |
| with: | |
| DOCKER_IMAGE: iregistry.baidu-int.com/new_rl_infra/base-images:paddlecloud-ubuntu24.04-gcc13.3-cuda12.9-cudnn9.9-bccl1.4.1.4-nccl2.26.5-openmpi4.1.5-FleetY13.0.0-v2.4.0-rc1 | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| COMPILE_ARCH: "80,90" | |
| WITH_NIGHTLY_BUILD: OFF | |
| FD_VERSION: 0.0.0 | |
| PADDLE_WHL_URL: https://paddle-qa.bj.bcebos.com/paddle-pipeline/Paddle-RL-Compile/develop/latest/paddlepaddle_gpu-3.3.0.dev-cp310-cp310-linux_x86_64.whl | |
| build_sm8689: | |
| name: BUILD_SM8689 | |
| needs: [clone, ce_job_pre_check] | |
| if: ${{ needs.ce_job_pre_check.outputs.sm8689_match == 'true' }} | |
| uses: ./.github/workflows/_build_linux.yml | |
| with: | |
| DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-build-cuda126-manylinux | |
| FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} | |
| COMPILE_ARCH: "86,89" | |
| WITH_NIGHTLY_BUILD: OFF | |
| FD_VERSION: 0.0.0 | |
| PADDLE_WHL_URL: ${{ needs.ce_job_pre_check.outputs.compile_use_paddle_whl_url }} | |
| secrets: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| ce_upload_sm8090: | |
| environment: CodeSync | |
| name: CE_UPLOAD | |
| needs: build_sm8090 | |
| runs-on: ubuntu-latest | |
| env: | |
| AK: ${{ secrets.BOS_AK }} | |
| SK: ${{ secrets.BOS_SK }} | |
| FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} | |
| COMPILE_ARCH: "80,90" | |
| steps: | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.10' | |
| - name: Wheel Info Show and Upload | |
| run: | | |
| echo "The wheel is located at: ${{ needs.build_sm8090.outputs.wheel_path }}" | |
| wget -q --no-check-certificate ${{ needs.build_sm8090.outputs.wheel_path }} | |
| filename=$(basename ${{ needs.build_sm8090.outputs.wheel_path }}) | |
| commit_id=${{ github.sha }} | |
| branch_name=${{ github.ref_name }} | |
| wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py | |
| push_file=$(realpath bos_tools.py) | |
| python -m pip install bce-python-sdk==0.9.29 | |
| target_paths=( | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/${commit_id}" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_80/${branch_name}/${commit_id}" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_90/${branch_name}/${commit_id}" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/latest" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_80/${branch_name}/latest" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_90/${branch_name}/latest" | |
| ) | |
| for target_path in "${target_paths[@]}"; do | |
| echo "Uploading ${filename} to ${target_path}" | |
| python "${push_file}" "${filename}" "${target_path}" | |
| done | |
| base_prefix="paddle-qa/" | |
| commit_path_stripped="${target_paths[0]#${base_prefix}}" | |
| latest_path_stripped="${target_paths[3]#${base_prefix}}" | |
| WHEEL_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}" | |
| WHEEL_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}" | |
| echo "commit wheel url is ${WHEEL_PATH}" | |
| echo "latest wheel url is ${WHEEL_PATH_LATEST}" | |
| ce_upload_sm8090_rl: | |
| environment: CodeSync | |
| name: CE_UPLOAD_RL | |
| needs: build_sm8090_rl | |
| runs-on: ubuntu-latest | |
| env: | |
| AK: ${{ secrets.BOS_AK }} | |
| SK: ${{ secrets.BOS_SK }} | |
| FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090_rl.outputs.wheel_path_rl }} | |
| COMPILE_ARCH: "80,90" | |
| steps: | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.10' | |
| - name: Wheel Info Show and Upload | |
| run: | | |
| echo "The wheel is located at: ${{ needs.build_sm8090_rl.outputs.wheel_path_rl }}" | |
| wget -q --no-check-certificate ${{ needs.build_sm8090_rl.outputs.wheel_path_rl }} | |
| filename=$(basename ${{ needs.build_sm8090_rl.outputs.wheel_path_rl }}) | |
| commit_id=${{ github.sha }} | |
| branch_name=${{ github.ref_name }} | |
| wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py | |
| push_file=$(realpath bos_tools.py) | |
| python -m pip install bce-python-sdk==0.9.29 | |
| target_paths=( | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE_RL/cu129/SM_8090/${branch_name}/${commit_id}" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE_RL/cu129/SM_8090/${branch_name}/latest" | |
| ) | |
| for target_path in "${target_paths[@]}"; do | |
| echo "Uploading ${filename} to ${target_path}" | |
| python "${push_file}" "${filename}" "${target_path}" | |
| done | |
| base_prefix="paddle-qa/" | |
| commit_path_stripped="${target_paths[0]#${base_prefix}}" | |
| latest_path_stripped="${target_paths[1]#${base_prefix}}" | |
| WHEEL_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}" | |
| WHEEL_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}" | |
| echo "commit wheel url is ${WHEEL_PATH}" | |
| echo "latest wheel url is ${WHEEL_PATH_LATEST}" | |
| ce_upload_sm8689: | |
| environment: CodeSync | |
| name: CE_UPLOAD | |
| needs: build_sm8689 | |
| runs-on: ubuntu-latest | |
| env: | |
| AK: ${{ secrets.BOS_AK }} | |
| SK: ${{ secrets.BOS_SK }} | |
| FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8689.outputs.wheel_path }} | |
| COMPILE_ARCH: "86,89" | |
| steps: | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.10' | |
| - name: Wheel Info Show and Upload | |
| run: | | |
| echo "The wheel is located at: ${{ needs.build_sm8689.outputs.wheel_path }}" | |
| wget -q --no-check-certificate ${{ needs.build_sm8689.outputs.wheel_path }} | |
| filename=$(basename ${{ needs.build_sm8689.outputs.wheel_path }}) | |
| commit_id=${{ github.sha }} | |
| branch_name=${{ github.ref_name }} | |
| wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py | |
| push_file=$(realpath bos_tools.py) | |
| python -m pip install bce-python-sdk==0.9.29 | |
| target_paths=( | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/${commit_id}" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_86/${branch_name}/${commit_id}" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_89/${branch_name}/${commit_id}" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/SM${COMPILE_ARCH//,/_}/${branch_name}/latest" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_86/${branch_name}/latest" | |
| "paddle-qa/paddle-pipeline/FastDeploy_ActionCE/cu126/SM_89/${branch_name}/latest" | |
| ) | |
| for target_path in "${target_paths[@]}"; do | |
| echo "Uploading ${filename} to ${target_path}" | |
| python "${push_file}" "${filename}" "${target_path}" | |
| done | |
| base_prefix="paddle-qa/" | |
| commit_path_stripped="${target_paths[0]#${base_prefix}}" | |
| latest_path_stripped="${target_paths[3]#${base_prefix}}" | |
| WHEEL_PATH="https://paddle-qa.bj.bcebos.com/${commit_path_stripped}/${filename}" | |
| WHEEL_PATH_LATEST="https://paddle-qa.bj.bcebos.com/${latest_path_stripped}/${filename}" | |
| echo "commit wheel url is ${WHEEL_PATH}" | |
| echo "latest wheel url is ${WHEEL_PATH_LATEST}" |