Skip to content

Commit 78e998a

Browse files
nmreadelflmangani
authored and
auxten
committedJun 7, 2024
Add arm64 linux build (#70)
* Add arm build * Update build_arm_whells.yml * fix build wheel * Cleanup Job Cleanup Job for Self-Hosted runners to reclaim disk space after successful or failed builds. --------- Co-authored-by: Lorenzo Mangani <[email protected]>
·
v3.1.1v2.0.0b0
1 parent 79a7e49 commit 78e998a

File tree

5 files changed

+220
-16
lines changed

5 files changed

+220
-16
lines changed
 

‎.github/workflows/build_arm_wheels.yml

+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
name: Build arm64
2+
3+
on:
4+
# for test action
5+
pull_request:
6+
branches:
7+
- main
8+
- feat/add-arm-wheel-build
9+
# end for test action
10+
workflow_dispatch:
11+
inputs:
12+
TAG_NAME:
13+
description: 'Release Version Tag'
14+
required: true
15+
release:
16+
types: [created]
17+
18+
defaults:
19+
run:
20+
shell: bash -leo pipefail {0}
21+
22+
jobs:
23+
build_wheels_linux_arm64:
24+
name: ${{ matrix.os }} py${{ matrix.python-version }}
25+
runs-on:
26+
- self-hosted
27+
- ARM64
28+
strategy:
29+
fail-fast: false
30+
matrix:
31+
os: [ ubuntu-20.04 ]
32+
env:
33+
RUNNER_OS: ${{ matrix.os }}
34+
PYTHON_VERSIONS: "3.8 3.9 3.10 3.11"
35+
steps:
36+
- name: Install clang++ for Ubuntu
37+
if: matrix.os == 'ubuntu-20.04'
38+
run: |
39+
pwd
40+
uname -a
41+
wget https://apt.llvm.org/llvm.sh
42+
chmod +x llvm.sh
43+
sudo ./llvm.sh 15
44+
which clang++-15
45+
clang++-15 --version
46+
sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget
47+
ccache -s
48+
- name: Update git
49+
run: |
50+
sudo add-apt-repository ppa:git-core/ppa -y
51+
sudo apt-get update
52+
sudo apt-get install -y git
53+
git --version
54+
shell: bash -l {0}
55+
- uses: actions/checkout@v3
56+
with:
57+
fetch-depth: 0
58+
- name: Restore submodules cache
59+
uses: actions/cache/restore@v3
60+
id: cache
61+
with:
62+
path: |
63+
contrib/**
64+
key: |
65+
submodule-${{ hashFiles('.gitmodules') }}
66+
- name: Update submodules if cache miss
67+
if: steps.cache.outputs.cache-hit != 'true'
68+
run: |
69+
git submodule update --init --recursive --jobs 4
70+
- name: Save submodules cache
71+
if: steps.cache.outputs.cache-hit != 'true'
72+
uses: actions/cache/save@v3
73+
with:
74+
path: |
75+
contrib/**
76+
key: |
77+
submodule-${{ hashFiles('.gitmodules') }}
78+
- name: ccache
79+
uses: hendrikmuhs/ccache-action@v1.2
80+
with:
81+
key: ${{ matrix.os }}
82+
max-size: 5G
83+
append-timestamp: true
84+
- name: remove old clang and link clang-15 to clang
85+
if: matrix.os == 'ubuntu-20.04'
86+
run: |
87+
sudo rm /usr/bin/clang
88+
sudo ln -s /usr/bin/clang-15 /usr/bin/clang
89+
sudo rm /usr/bin/clang++
90+
sudo ln -s /usr/bin/clang++-15 /usr/bin/clang++
91+
which clang++
92+
clang++ --version
93+
- name: Make linux-arm64
94+
run: |
95+
#make linux-arm64
96+
bash -l -e chdb/build_linux_arm64.sh
97+
continue-on-error: false
98+
- name: Check ccache statistics
99+
run: |
100+
ccache -s
101+
ls -lh chdb
102+
df -h
103+
- name: Install patchelf from github
104+
run: |
105+
pyenv local "${{ matrix.python-version }}"
106+
wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz
107+
tar -xvf patchelf.tar.gz
108+
sudo cp bin/patchelf /usr/bin/
109+
sudo chmod +x /usr/bin/patchelf
110+
patchelf --version
111+
- name: Audit wheels
112+
run: |
113+
pyenv local "${{ matrix.python-version }}"
114+
python3 -m pip install auditwheel
115+
auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl
116+
continue-on-error: false
117+
- name: Show files
118+
run: |
119+
# e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
120+
sudo rm -f dist/*linux_aarch64.whl
121+
ls -lh dist
122+
shell: bash
123+
continue-on-error: false
124+
- uses: actions/upload-artifact@v3
125+
with:
126+
path: ./dist/*.whl
127+
- name: Upload pypi
128+
if: startsWith(github.ref, 'refs/tags/v')
129+
run: |
130+
pyenv local "${{ matrix.python-version }}"
131+
python3 -m pip install twine
132+
python3 -m twine upload wheelhouse/*.whl
133+
env:
134+
TWINE_USERNAME: __token__
135+
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
136+
137+
# #build for macos12 arm64(Apple Silicon)
138+
# build_wheels_macos_arm64:
139+
140+
runner_cleanup_arm64:
141+
name: Wipe Caches on Self-Hosted Runners
142+
needs: [ build_wheels_linux_arm64 ]
143+
if: always()
144+
runs-on:
145+
- self-hosted
146+
- ARM64
147+
steps:
148+
- name: Actions Cache Cleanup
149+
run: if [ "$(which action_cleanup)" != "" ]; then action_cleanup; else echo bypass; fi

‎.github/workflows/build_wheels.yml

+7-15
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,13 @@
1-
name: Build
1+
name: Build X86
22

33
on:
4-
push:
5-
tags:
6-
- 'v*'
7-
branches:
8-
- main
9-
paths-ignore:
10-
- '**.md'
11-
- '**.yml'
12-
- '.github/**'
13-
- 'examples/**'
14-
- 'tests/**'
15-
pull_request:
16-
branches:
17-
- main
184
workflow_dispatch:
5+
inputs:
6+
TAG_NAME:
7+
description: 'Release Version Tag'
8+
required: true
9+
release:
10+
types: [created]
1911

2012
jobs:
2113
build_wheels_linux:

‎Makefile

+6-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,12 @@ clean:
2626

2727
mac-arm64:
2828
@echo "Make macOS arm64 whl"
29-
packages/build_mac_arm64.sh
29+
chdb/build_mac_arm64.sh
30+
@echo "Done."
31+
32+
linux-arm64:
33+
@echo "Make linux arm64 whl"
34+
chdb/build_linux_arm64.sh
3035
@echo "Done."
3136

3237
build: clean buildlib wheel

‎chdb/build_linux_arm64.sh

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash -e
2+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
3+
PROJ_DIR=$(dirname ${DIR})
4+
5+
[ -z "${PYTHON_VERSIONS}" ] && { echo "please provide PYTHON_VERSIONS env, e.g: PYTHON_VERSIONS='3.8 3.9'"; exit 1; }
6+
7+
# echo ${PROJ_DIR}
8+
9+
for PY_VER in ${PYTHON_VERSIONS}; do
10+
cd ${PROJ_DIR}
11+
pyenv local "${PY_VER}"
12+
python3 --version
13+
python3 -m pip install pybind11
14+
export CC=/usr/bin/clang
15+
export CXX=/usr/bin/clang++
16+
# Install universal2 pkg
17+
python3 -VV
18+
python3-config --includes
19+
# if python3 -VV does not contain ${PY_VER}, then exit
20+
if ! python3 -VV 2>&1 | grep -q "${PY_VER}"; then
21+
echo "Error: Required version of Python (${PY_VER}) not found. Aborting."
22+
exit 1
23+
fi
24+
25+
python3 -m pip install -U pybind11 wheel build tox
26+
rm -rf ./buildlib
27+
28+
./chdb/build.sh
29+
cd chdb && python3 -c "import _chdb; res = _chdb.query('select 1112222222,555', 'JSON'); print(res)" && cd -
30+
31+
./gen_manifest.sh
32+
cat ./MANIFEST.in
33+
34+
# try delete
35+
whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./} || echo "notfound.whl")
36+
rm -f ${whl_file} || :
37+
38+
python3 -m build --wheel
39+
40+
python3 -m wheel tags --platform-tag=manylinux_2_17_aarch64
41+
42+
find dist
43+
44+
python3 -m pip install pandas pyarrow psutil
45+
find dist
46+
whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./})
47+
python3 -m pip install --force-reinstall ${whl_file}
48+
49+
python3 -c "import chdb; res = chdb.query('select version()', 'CSV'); print(res)"
50+
51+
python3 -m chdb "SELECT 1, 'ab'" arrowtable
52+
53+
python3 -m chdb "SELECT 1, 'ab'" dataframe
54+
55+
bash -x ./chdb/test_smoke.sh
56+
57+
make test
58+
done

‎packages/build_mac_arm64.sh ‎chdb/build_mac_arm64.sh

File renamed without changes.

0 commit comments

Comments
 (0)
Please sign in to comment.