diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 32650756..ce86947a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -99,4 +99,4 @@ jobs: chmod -R 777 $workdir if [ -d "${{ env.CI_PATH }}" ]; then rm -rf ${{ env.CI_PATH }} - fi \ No newline at end of file + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..ae24cfc4 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,147 @@ +name: Release on Tag + +on: + push: + tags: + - "v*.*.*" # 只对形如 v1.2.3 的 tag 触发 + +env: + RELEASE_PATH: "${{ vars.CI_BASE_PATH }}/GitHub/Release/${{ github.repository }}/${GITHUB_RUN_NUMBER}" + THIRD_PARTY_PATH: "${{ vars.CI_BASE_PATH }}/data/DLCompiler/third_party" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + release: + if: github.repository == 'DeepLink-org/DLCompiler' + runs-on: DLC-60 + + steps: + - name: Checkout code via SSH + uses: actions/checkout@v4 + with: + ssh-key: ${{ secrets.SSH_PRIVATE_KEY_DLC_CI }} + + - name: Create custom directory + run: | + set -ex + echo ${{ env.RELEASE_PATH }} + mkdir -p ${{ env.RELEASE_PATH }} + + - name: Clean custom directory + run: | + set -ex + if [ -d "${{ env.RELEASE_PATH }}" ]; then + rm -rf ${{ env.RELEASE_PATH }}/* + rm -rf ${{ env.RELEASE_PATH }}/.github + fi + + - name: Move code to custom directory + run: | + set -ex + mv $GITHUB_WORKSPACE/* ${{ env.RELEASE_PATH }}/ + mv $GITHUB_WORKSPACE/.github ${{ env.RELEASE_PATH }}/ + mv $GITHUB_WORKSPACE/.git ${{ env.RELEASE_PATH }}/ + + - name: Setup Git Safe Directory + run: | + git config --global --add safe.directory '*' + + - name: Install build dependencies + run: | + pip install build twine -i https://mirrors.huaweicloud.com/repository/pypi/simple + pip install pybind11 -i https://mirrors.huaweicloud.com/repository/pypi/simple + + - name: Build DLCompiler Package + run: | + set -ex + source /home/dlc_ci/.bashrc + source activate dlcompiler + source /usr/local/Ascend/ascend-toolkit/set_env.sh + cd ${{ env.RELEASE_PATH }} + export JSON_PATH34=${{ vars.CI_BASE_PATH }}/data/v34/include.zip + export GOOGLETEST_DIR34=${{ vars.CI_BASE_PATH }}/data/v34/googletest + export LLVM_TGZ_PATH34=${{ vars.CI_BASE_PATH }}/data/v34/llvm-064f02da-ubuntu-arm64.tar.gz + + rm -rf ./third_party/* + git clone --no-hardlinks ${{ env.THIRD_PARTY_PATH }}/ascendnpu-ir ./third_party/ascendnpu-ir + git clone --no-hardlinks ${{ env.THIRD_PARTY_PATH }}/json ./third_party/json + git clone --no-hardlinks ${{ env.THIRD_PARTY_PATH }}/triton_shared ./third_party/triton_shared + git clone --no-hardlinks ${{ env.THIRD_PARTY_PATH }}/triton ./third_party/triton + echo "whoami? $(whoami)" + echo "which python? $(which python)" + git submodule update --init + echo "git submodule update done." + pip install nanobind -i https://mirrors.huaweicloud.com/repository/pypi/simple + if [[ "${{ github.ref }}" == refs/tags/* ]]; then + bash compile_shared.sh apply_patch=true build_package=true + fi + find ${{ env.RELEASE_PATH }} -name "*.whl" + + - name: Check build output files + run: | + set -ex + echo "=== 当前工作目录 ===" + pwd + + echo -e "\n=== RELEASE_PATH 目录结构 ===" + ls -la ${{ env.RELEASE_PATH }} + + echo -e "\n=== 编译输出目录 (triton/dist) 内容 ===" + TRITON_DIST_DIR="${{ env.RELEASE_PATH }}/third_party/triton/dist" + if [ -d "$TRITON_DIST_DIR" ]; then + ls -la "$TRITON_DIST_DIR" + echo -e "\n=== 生成的 .whl 文件列表 ===" + ls -la "$TRITON_DIST_DIR"/*.whl || echo "未找到 .whl 文件" + else + echo "错误:编译输出目录 $TRITON_DIST_DIR 不存在" + fi + + - name: Copy WHL to workspace + run: | + set -ex + WHL_SRC="${{ env.RELEASE_PATH }}/third_party/triton/dist/*.whl" + WHL_DST="./dist" + mkdir -p "$WHL_DST" + cp $WHL_SRC "$WHL_DST/" + echo "Copied wheel(s) to $WHL_DST" + ls -la "$WHL_DST" + realpath "$WHL_DST" + + - name: Create Release and Upload Assets + uses: softprops/action-gh-release@v2 + with: + files: | + ./dist/*.whl + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Debug failed release assets + if: failure() + run: | + set -ex + TRITON_DIST_DIR="${{ env.RELEASE_PATH }}/third_party/triton/dist" + echo "=== 发布失败 - 检查资产目录 ===" + echo "目标目录: $TRITON_DIST_DIR" + if [ -d "$TRITON_DIST_DIR" ]; then + echo "目录存在,内容如下:" + ls -la "$TRITON_DIST_DIR" + echo -e "\nWHL文件列表:" + ls -la "$TRITON_DIST_DIR"/*.whl || echo "未找到WHL文件" + else + echo "错误: 目标目录不存在" + fi + + - name: Clear workfile + if: always() + run: | + export workdir=$(pwd) + cd .. + rm -rf $workdir + mkdir $workdir + chmod -R 777 $workdir + if [ -d "${{ env.RELEASE_PATH }}" ]; then + rm -rf ${{ env.RELEASE_PATH }} + fi diff --git a/compile_shared.sh b/compile_shared.sh index 57f48f42..d5fdd058 100644 --- a/compile_shared.sh +++ b/compile_shared.sh @@ -4,45 +4,25 @@ export LC_ALL="zh_CN.UTF-8" home_path=$(pwd) # compile triton shared library with patch -compile_triton_shared=false apply_patch=false +build_package=false # 遍历所有参数 for arg in "$@"; do - if [ "$arg" = "compile_triton_shared=true" ]; then - echo "检测到 compile_triton_shared=true" - compile_triton_shared=true - elif [ "$arg" = "apply_patch=true" ]; then + if [ "$arg" = "apply_patch=true" ]; then echo "检测到 apply_patch=true" apply_patch=true + elif [ "$arg" = "build_package=true" ]; then + echo "检测到 build_package=true" + build_package=true fi done echo "start compile ========================================" -echo compile_triton_shared: $compile_triton_shared echo apply_patch: $apply_patch echo "======================================================" -if [[ $compile_triton_shared == true ]]; then - echo "start compile triton_shared" - cd third_party - mkdir -p build && rm -rf build/* - git clone --no-hardlinks triton_shared build/triton_shared - git clone --no-hardlinks triton build/triton - cd build - export TRITON_PLUGIN_DIRS=$(pwd)/triton_shared - cd triton_shared && git clean -xdf && git checkout . && git checkout 2b728ad97bc02af821a0805b09075838911d4c19 && ls ../../../patch/v3_4/triton_shared.patch | xargs -n1 git apply && cd ../ - cd triton && git clean -xdf && git checkout . && cd ../ - cd triton && git checkout $(cat ../triton_shared/triton-hash.txt) && ls ../../../patch/v3_4/triton.patch | xargs -n1 git apply - TRITON_BUILD_WITH_CLANG_LLD=true TRITON_BUILD_WITH_CCACHE=true python3 -m pip install --no-build-isolation -vvv '.[tests]' - if [ $? -ne 0 ]; then - echo "Error: triton_shared compile failed." >&2 - exit 1 - fi - echo "triton_shared compile success!" -fi - echo "start apply ascendnpu-ir patch" cd $home_path/third_party/ascendnpu-ir git checkout . @@ -97,22 +77,31 @@ notify_apply_patch() { fi } -pip uninstall triton dlcompiler -y +if [[ $build_package == true ]]; then + echo "build_package is true, skip uninstall triton dlcompiler" + cd $TRITON_PLUGIN_DIRS/third_party/triton/ +else + pip uninstall triton dlcompiler -y -cd $TRITON_PLUGIN_DIRS/third_party/triton/ -rm -rf build/ + cd $TRITON_PLUGIN_DIRS/third_party/triton/ + rm -rf build/ +fi if [ -z "$LLVM_BUILD_DIR" ]; then - TRITON_BUILD_WITH_CLANG_LLD=true TRITON_BUILD_WITH_CCACHE=true \ - python3 -m pip install --no-build-isolation -vvv .[tests] -i https://mirrors.huaweicloud.com/repository/pypi/simple - if [ $? -ne 0 ]; then - notify_apply_patch - echo "Error: DLCompiler compile failed." - exit 1 + # 使用build_package参数,控制是否进行编译 + if [[ $build_package == true ]]; then + echo "LLVM_BUILD_DIR is not set, using system LLVM or downloading prebuilt LLVM build package." + TRITON_BUILD_WITH_CLANG_LLD=true TRITON_BUILD_WITH_CCACHE=true \ + python3 -m pip wheel --no-deps --no-build-isolation -w dist/ . + else + TRITON_BUILD_WITH_CLANG_LLD=true TRITON_BUILD_WITH_CCACHE=true \ + python3 -m pip install --no-build-isolation -vvv .[tests] -i https://mirrors.huaweicloud.com/repository/pypi/simple + if [ $? -ne 0 ]; then + notify_apply_patch + echo "Error: DLCompiler compile failed." + exit 1 + fi fi - # echo "LLVM_BUILD_DIR is not set, using system LLVM or downloading prebuilt LLVM." - # TRITON_BUILD_WITH_CLANG_LLD=true TRITON_BUILD_WITH_CCACHE=true \ - # python3 -m pip wheel --no-deps --no-build-isolation -w dist/ . else echo "LLVM_BUILD_DIR is set to $LLVM_BUILD_DIR" LLVM_INCLUDE_DIRS=$LLVM_BUILD_DIR/include \ @@ -127,4 +116,4 @@ else exit 1 fi fi -notify_apply_patch \ No newline at end of file +notify_apply_patch