Skip to content

Commit 985b126

Browse files
XieYunshenXieYunshen
authored
CE 编译任务(合入触发) (#3491)
* add ce compile job * fix * update
1 parent 31f639f commit 985b126

File tree

1 file changed

+244
-0
lines changed

1 file changed

+244
-0
lines changed

‎.github/workflows/ce_job.yml‎

Lines changed: 244 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
name: CE Compile Job
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- develop
8+
- 'release/*'
9+
permissions: read-all
10+
11+
concurrency:
12+
group: ${{ github.ref }}-${{ github.sha }}
13+
cancel-in-progress: true
14+
15+
jobs:
16+
ce_job_pre_check:
17+
runs-on: ubuntu-latest
18+
env:
19+
COMPILE_BRANCH: ${{ vars.COMPILE_BRANCH }}
20+
CE_COMPILE_SELECTION: ${{ vars.CE_COMPILE_SELECTION }}
21+
COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ vars.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }}
22+
outputs:
23+
branch_match: ${{ steps.set_output.outputs.branch_match }}
24+
compile_use_paddle_whl_url: ${{ steps.set_output.outputs.compile_use_paddle_whl_url }}
25+
sm8689_match: ${{ steps.set_output.outputs.sm8689_match }}
26+
sm8090_match: ${{ steps.set_output.outputs.sm8090_match }}
27+
28+
steps:
29+
- name: Set Version
30+
id: set_output
31+
env:
32+
COMPILE_BRANCH: ${{ env.COMPILE_BRANCH }}
33+
CE_COMPILE_SELECTION: ${{ env.CE_COMPILE_SELECTION }}
34+
COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ env.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }}
35+
GITHUB_REF_NAME: ${{ github.ref_name }}
36+
run: |
37+
# 选择要触发编译任务的分支 done
38+
# 选择指定分支要编译的任务 8090或者8689
39+
# 指定分支编译要使用的Paddle的安装包,默认使用nightly最新的
40+
41+
IFS=',' read -ra BRANCHES <<< "$COMPILE_BRANCH"
42+
MATCH=false
43+
for b in "${BRANCHES[@]}"; do
44+
if [[ "$b" == "${GITHUB_REF_NAME}" ]]; then
45+
MATCH=true
46+
break
47+
fi
48+
done
49+
echo "branch_match=$MATCH" >> $GITHUB_OUTPUT
50+
51+
# 通过变量CE_COMPILE_SELECTION中的映射关系,决定分支是编译sm8090还是sm8689
52+
for pair in $(echo "$CE_COMPILE_SELECTION" | tr ';' ' '); do
53+
branch=$(echo "$pair" | cut -d',' -f1)
54+
compile_task_list=$(echo "$pair" | cut -d',' -f2)
55+
56+
if [[ "$branch" == "$GITHUB_REF_NAME" ]]; then
57+
58+
# 判断里面是否包含 sm8090 或 sm8689
59+
if [[ "$compile_task_list" == *"sm8090"* ]]; then
60+
echo "sm8090_match=true" >> $GITHUB_OUTPUT
61+
fi
62+
if [[ "$compile_task_list" == *"sm8689"* ]]; then
63+
echo "sm8689_match=true" >> $GITHUB_OUTPUT
64+
fi
65+
break
66+
fi
67+
done
68+
69+
# 通过变量COMPILE_USE_PADDLE_WHL_URL_MAPPINGS中的映射关系,决定是否是安装指定版本的Paddle还是直接安装URL
70+
for pair in $(echo $COMPILE_USE_PADDLE_WHL_URL_MAPPINGS | tr ';' ' '); do
71+
branch=$(echo "$pair" | cut -d',' -f1)
72+
paddle_whl_url=$(echo "$pair" | cut -d',' -f2)
73+
if [[ "$branch" == "${{ github.ref_name }}" ]]; then
74+
FOUND_PADDLE_URL="$paddle_whl_url"
75+
echo "compile_use_paddle_whl_url=${FOUND_PADDLE_URL}" >> $GITHUB_OUTPUT
76+
break
77+
fi
78+
done
79+
80+
print_ce_job_pre_check_outputs:
81+
runs-on: ubuntu-latest
82+
needs: ce_job_pre_check
83+
steps:
84+
- name: Print outputs as JSON
85+
run: |
86+
echo '${{ toJSON(needs.ce_job_pre_check.outputs) }}'
87+
88+
89+
clone:
90+
environment: CodeSync
91+
name: FD-Clone-Linux
92+
runs-on: ubuntu-latest
93+
needs: ce_job_pre_check
94+
if: ${{ needs.ce_job_pre_check.outputs.branch_match == 'true' }}
95+
outputs:
96+
repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }}
97+
steps:
98+
- name: Clone FastDeploy
99+
uses: actions/checkout@v4
100+
with:
101+
ref: ${{ github.event_name == 'pull_request'
102+
&& github.event.pull_request.base.ref
103+
|| github.ref_name }}
104+
submodules: 'recursive'
105+
fetch-depth: 1000
106+
107+
- name: Python Setup
108+
uses: actions/setup-python@v5
109+
with:
110+
python-version: '3.10'
111+
- name: Code Info Show and Upload
112+
id: set_output
113+
env:
114+
AK: ${{ secrets.BOS_AK }}
115+
SK: ${{ secrets.BOS_SK }}
116+
run: |
117+
git config --unset http.https://github.com/.extraheader
118+
git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
119+
git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
120+
echo "Current HEAD Log:"
121+
git log --oneline -n 5
122+
ls
123+
cd ..
124+
tar -zcf FastDeploy.tar.gz FastDeploy
125+
commit_id=${{ github.sha }}
126+
branch_name=${{ github.ref_name }}
127+
target_path=paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id}
128+
wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
129+
push_file=$(realpath bos_tools.py)
130+
python -m pip install bce-python-sdk==0.9.29
131+
ls
132+
python ${push_file} FastDeploy.tar.gz ${target_path}
133+
target_path_stripped="${target_path#paddle-qa/}"
134+
REPO_ARCHIVE_URL=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/FastDeploy.tar.gz
135+
echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT
136+
137+
resultshow:
138+
name: Show Code Archive Output
139+
needs: clone
140+
runs-on: ubuntu-latest
141+
steps:
142+
- name: Print wheel path
143+
run: |
144+
echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}"
145+
146+
build_sm8090:
147+
name: BUILD_SM8090
148+
needs: [clone, ce_job_pre_check]
149+
if: ${{ needs.ce_job_pre_check.outputs.sm8090_match == 'true' }}
150+
uses: ./.github/workflows/_build_linux.yml
151+
with:
152+
DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
153+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
154+
COMPILE_ARCH: "80,90"
155+
WITH_NIGHTLY_BUILD: OFF
156+
FD_VERSION: 0.0.0
157+
158+
build_sm8689:
159+
name: BUILD_SM8689
160+
needs: [clone, ce_job_pre_check]
161+
if: ${{ needs.ce_job_pre_check.outputs.sm8689_match == 'true' }}
162+
uses: ./.github/workflows/_build_linux.yml
163+
with:
164+
DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate
165+
FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }}
166+
COMPILE_ARCH: "86,89"
167+
WITH_NIGHTLY_BUILD: OFF
168+
FD_VERSION: 0.0.0
169+
170+
ce_upload_sm8090:
171+
environment: CodeSync
172+
name: CE_UPLOAD
173+
needs: build_sm8090
174+
runs-on: ubuntu-latest
175+
env:
176+
AK: ${{ secrets.BOS_AK }}
177+
SK: ${{ secrets.BOS_SK }}
178+
FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }}
179+
COMPILE_ARCH: "80,90"
180+
steps:
181+
- uses: actions/setup-python@v5
182+
with:
183+
python-version: '3.10'
184+
- name: Wheel Info Show and Upload
185+
if: github.ref_name == 'develop' || github.ref_type == 'tag'
186+
run: |
187+
echo "The wheel is located at: ${{ needs.build_sm8090.outputs.wheel_path }}"
188+
wget -q --no-check-certificate ${{ needs.build_sm8090.outputs.wheel_path }}
189+
filename=$(basename ${{ needs.build_sm8090.outputs.wheel_path }})
190+
191+
commit_id=${{ github.sha }}
192+
branch_name=${{ github.ref_name }}
193+
target_path=paddle-qa/paddle-pipeline/FastDeploy_ActionCE${COMPILE_ARCH//,/_}/${branch_name}/${commit_id}
194+
195+
wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
196+
push_file=$(realpath bos_tools.py)
197+
python -m pip install bce-python-sdk==0.9.29
198+
ls
199+
python ${push_file} ${filename} ${target_path}
200+
target_path_stripped="${target_path#paddle-qa/}"
201+
WHEEL_PATH=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/${fd_wheel_name}
202+
203+
target_path_latest=paddle-qa/paddle-pipeline/FastDeploy_ActionCE${COMPILE_ARCH//,/_}/${branch_name}/latest
204+
python ${push_file} ${filename} ${target_path_latest}
205+
target_path_stripped_latest="${target_path_latest#paddle-qa/}"
206+
WHEEL_PATH_LATEST=https://paddle-qa.bj.bcebos.com/${target_path_stripped_latest}/${fd_wheel_name}
207+
208+
ce_upload_sm8689:
209+
environment: CodeSync
210+
name: CE_UPLOAD
211+
needs: build_sm8689
212+
runs-on: ubuntu-latest
213+
env:
214+
AK: ${{ secrets.BOS_AK }}
215+
SK: ${{ secrets.BOS_SK }}
216+
FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }}
217+
COMPILE_ARCH: "86,89"
218+
steps:
219+
- uses: actions/setup-python@v5
220+
with:
221+
python-version: '3.10'
222+
- name: Wheel Info Show and Upload
223+
if: github.ref_name == 'develop' || github.ref_type == 'tag'
224+
run: |
225+
echo "The wheel is located at: ${{ needs.build_sm8090.outputs.wheel_path }}"
226+
wget -q --no-check-certificate ${{ needs.build_sm8090.outputs.wheel_path }}
227+
filename=$(basename ${{ needs.build_sm8090.outputs.wheel_path }})
228+
229+
commit_id=${{ github.sha }}
230+
branch_name=${{ github.ref_name }}
231+
target_path=paddle-qa/paddle-pipeline/FastDeploy_ActionCE${COMPILE_ARCH//,/_}/${branch_name}/${commit_id}
232+
233+
wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
234+
push_file=$(realpath bos_tools.py)
235+
python -m pip install bce-python-sdk==0.9.29
236+
ls
237+
python ${push_file} ${filename} ${target_path}
238+
target_path_stripped="${target_path#paddle-qa/}"
239+
WHEEL_PATH=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/${fd_wheel_name}
240+
241+
target_path_latest=paddle-qa/paddle-pipeline/FastDeploy_ActionCE${COMPILE_ARCH//,/_}/${branch_name}/latest
242+
python ${push_file} ${filename} ${target_path_latest}
243+
target_path_stripped_latest="${target_path_latest#paddle-qa/}"
244+
WHEEL_PATH_LATEST=https://paddle-qa.bj.bcebos.com/${target_path_stripped_latest}/${fd_wheel_name}

0 commit comments

Comments
 (0)