From 2086092f164d29b7193c6ea5c5d2a34283bb114e Mon Sep 17 00:00:00 2001 From: Eduard Stefes Date: Fri, 21 Feb 2025 10:58:24 +0100 Subject: [PATCH] Added CI worker for native S390X --- .github/workflows/cmake.yml | 29 ++++++++- .github/workflows/configure.yml | 26 +++++++- contrib/README.contrib | 3 + contrib/s390x/README.md | 59 +++++++++++++++++++ .../actions_runner.container | 9 +++ .../self-hosted-builder/actions_runner.sh | 32 ++++++++++ .../actions_runner_conf.sh | 3 + .../actions_runner_ubuntu.Dockerfile | 52 ++++++++++++++++ 8 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 contrib/s390x/README.md create mode 100644 contrib/s390x/self-hosted-builder/actions_runner.container create mode 100755 contrib/s390x/self-hosted-builder/actions_runner.sh create mode 100644 contrib/s390x/self-hosted-builder/actions_runner_conf.sh create mode 100644 contrib/s390x/self-hosted-builder/actions_runner_ubuntu.Dockerfile diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 25a3b8124..4c21521a4 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -9,6 +9,7 @@ jobs: matrix: include: - name: Ubuntu GCC + enabled: true os: ubuntu-latest compiler: gcc cflags: -Wall -Wextra @@ -16,6 +17,7 @@ jobs: cmake-args: -DMINIZIP_ENABLE_BZIP2=ON - name: Ubuntu GCC -O3 + enabled: true os: ubuntu-latest compiler: gcc cflags: -O3 -Wall -Wextra @@ -23,6 +25,7 @@ jobs: cmake-args: -DMINIZIP_ENABLE_BZIP2=ON - name: Ubuntu Clang + enabled: true os: ubuntu-latest compiler: clang cflags: -Wall -Wextra @@ -30,6 +33,7 @@ jobs: cmake-args: -DMINIZIP_ENABLE_BZIP2=ON - name: Ubuntu Clang Debug + enabled: true os: ubuntu-latest compiler: clang cflags: -Wall -Wextra @@ -38,6 +42,7 @@ jobs: cmake-args: -DMINIZIP_ENABLE_BZIP2=ON - name: Windows MSVC Win32 + enabled: true os: windows-latest compiler: cl cflags: /W3 @@ -45,6 +50,7 @@ jobs: pkgtgt: PACKAGE - name: Windows MSVC Win64 + enabled: true os: windows-latest compiler: cl cflags: /W3 @@ -52,6 +58,7 @@ jobs: pkgtgt: PACKAGE - name: Windows GCC + enabled: true os: windows-latest compiler: gcc cflags: -Wall -Wextra @@ -59,6 +66,7 @@ jobs: pkgtgt: package - name: macOS Clang + enabled: true os: macos-latest compiler: clang cflags: -Wall -Wextra @@ -66,44 +74,59 @@ jobs: cmake-args: -DMINIZIP_ENABLE_BZIP2=ON - name: macOS GCC + enabled: true os: macos-latest compiler: gcc-12 cflags: -Wall -Wextra pkgtgt: package cmake-args: -DMINIZIP_ENABLE_BZIP2=ON + - name: S390X Native GCC + enabled: ${{ github.repository == 'fneddy/zlib' && 'true' || false }} + os: ${{ github.repository == 'fneddy/zlib' && 'S390X' || 'ubuntu-latest' }} + compiler: gcc + cflags: -Wall -Wextra + cmake-args: -DMINIZIP_ENABLE_BZIP2=OFF + pkgtgt: package + steps: - name: Checkout repository + if: matrix.enabled uses: actions/checkout@v4 - name: Install packages (Windows) - if: runner.os == 'Windows' + if: runner.os == 'Windows' && matrix.enabled run: | choco install --no-progress ninja - name: Install packages (Linux) - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.os != 'S390X' && matrix.enabled run: | sudo apt install libbz2-dev + - name: Generate project files + if: matrix.enabled run: cmake -S . -B ../build ${{ matrix.cmake-args }} -D CMAKE_BUILD_TYPE=${{ matrix.build-config || 'Release' }} -DZLIB_BUILD_MINIZIP=ON env: CC: ${{ matrix.compiler }} CFLAGS: ${{ matrix.cflags }} - name: Compile source code + if: matrix.enabled run: cmake --build ../build --config ${{ matrix.build-config || 'Release' }} - name: Run test cases + if: matrix.enabled run: ctest -C Release --output-on-failure --max-width 120 working-directory: ../build - name: create packages + if: matrix.enabled run: cmake --build ../build --config ${{ matrix.build-config || 'Release' }} -t ${{ matrix.pkgtgt }} - name: Upload build errors uses: actions/upload-artifact@v4 - if: failure() + if: failure() && matrix.enabled == true with: name: ${{ matrix.name }} (cmake) path: | diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index f19272c72..e9e4df6ba 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -9,12 +9,14 @@ jobs: matrix: include: - name: Ubuntu GCC + enabled: true os: ubuntu-latest compiler: gcc configure-args: --warn # Test out of source builds - name: Ubuntu GCC OSB + enabled: true os: ubuntu-latest compiler: gcc configure-args: --warn @@ -22,6 +24,7 @@ jobs: src-dir: ../zlib - name: Ubuntu GCC ARM SF + enabled: true os: ubuntu-latest compiler: arm-linux-gnueabi-gcc configure-args: --warn @@ -30,6 +33,7 @@ jobs: qemu-run: qemu-arm -L /usr/arm-linux-gnueabi - name: Ubuntu GCC ARM HF + enabled: true os: ubuntu-latest compiler: arm-linux-gnueabihf-gcc configure-args: --warn @@ -38,6 +42,7 @@ jobs: qemu-run: qemu-arm -L /usr/arm-linux-gnueabihf - name: Ubuntu GCC AARCH64 + enabled: true os: ubuntu-latest compiler: aarch64-linux-gnu-gcc configure-args: --warn @@ -46,6 +51,7 @@ jobs: qemu-run: qemu-aarch64 -L /usr/aarch64-linux-gnu - name: Ubuntu GCC PPC + enabled: true os: ubuntu-latest compiler: powerpc-linux-gnu-gcc configure-args: --warn --static @@ -56,6 +62,7 @@ jobs: ldflags: -static - name: Ubuntu GCC PPC64 + enabled: true os: ubuntu-latest compiler: powerpc64-linux-gnu-gcc configure-args: --warn --static @@ -66,6 +73,7 @@ jobs: ldflags: -static - name: Ubuntu GCC PPC64LE + enabled: true os: ubuntu-latest compiler: powerpc64le-linux-gnu-gcc configure-args: --warn @@ -73,7 +81,9 @@ jobs: packages: qemu-system qemu-user gcc-powerpc64le-linux-gnu libc-dev-ppc64el-cross qemu-run: qemu-ppc64le -L /usr/powerpc64le-linux-gnu + # if on base repo use the native workflow runner, else QEMU - name: Ubuntu GCC S390X + enabled: true os: ubuntu-latest compiler: s390x-linux-gnu-gcc configure-args: --warn --static @@ -83,27 +93,37 @@ jobs: cflags: -static ldflags: -static + - name: Native S390X GCC + enabled: ${{ github.repository == 'fneddy/zlib' && 'true' || false }} + os: ${{ github.repository == 'fneddy/zlib' && 'S390X' || 'ubuntu-latest' }} + compiler: gcc + configure-args: --warn + - name: macOS GCC + enabled: true os: macos-latest compiler: gcc-12 configure-args: --warn - name: macOS Clang + enabled: true os: macos-latest compiler: clang configure-args: --warn steps: - name: Checkout repository + if: matrix.enabled uses: actions/checkout@v4 - name: Install packages (Ubuntu) - if: runner.os == 'Linux' && matrix.packages + if: runner.os == 'Linux' && matrix.packages && matrix.enabled run: | sudo apt-get update sudo apt-get install -y ${{ matrix.packages }} - name: Generate project files + if: matrix.enabled run: | [ -d ${{ matrix.build-dir || '.' }} ] || mkdir ${{ matrix.build-dir || '.' }} cd ${{ matrix.build-dir || '.' }} @@ -115,10 +135,12 @@ jobs: CHOST: ${{ matrix.chost }} - name: Compile source code + if: matrix.enabled run: make -j2 working-directory: ${{ matrix.build-dir }} - name: Run test cases + if: matrix.enabled run: | make test make cover @@ -128,7 +150,7 @@ jobs: - name: Upload build errors uses: actions/upload-artifact@v4 - if: failure() + if: failure() && matrix.enabled with: name: ${{ matrix.name }} (configure) path: | diff --git a/contrib/README.contrib b/contrib/README.contrib index 5e5f95054..1b8bc0f03 100644 --- a/contrib/README.contrib +++ b/contrib/README.contrib @@ -55,3 +55,6 @@ untgz/ by Pedro A. Aranda Gutierrez vstudio/ by Gilles Vollant Building a minizip-enhanced zlib with Microsoft Visual Studio Includes vc11 from kreuzerkrieg and vc12 from davispuh + +s390x/ by Eduard Stefes + Scripts to create S390X github action workers. diff --git a/contrib/s390x/README.md b/contrib/s390x/README.md new file mode 100644 index 000000000..9844f2af2 --- /dev/null +++ b/contrib/s390x/README.md @@ -0,0 +1,59 @@ +# General Notes + +> [!NOTE] +> This is a modified and simplified copy of the s390x self-hosted action runner scripts from here: +> - https://github.com/zlib-ng/zlib-ng/tree/a0fa24710c8faa1a746a20cfd5c7c24571e15ca4/arch/s390/self-hosted-builder +> - https://github.com/anup-kodlekere/gaplib +> +> **This code does NOT setup ephemeral workers!!**. It means that the worker context will be reused between CI runs and never be destroyed. +> +> The implications are: +> - one needs to check a PR that does not leak the /etc/actions_runner_conf +> - one needs to check a PR that it does not alter the system in a persistent way +> - one needs to check a PR that it does not access any file outside of the build folder +> +> To create ephemeral workers we need to setup a token request process similar like zlib-ng has done. But such a process implies that the worker maintainer needs, not just commit, but administration rights to the repository. + +## Self Hosted Runner + +Given complexity of Vector and DFLTCC machine instruction, it is not clear whether QEMU TCG will ever support it. At the time of writing, one has to have access to an IBM z15+ VM or LPAR in order to test the support. Since DFLTCC and Vector instructions are non-privileged, neither special VM/LPAR configuration nor root are required. + +The CI uses a self-hosted builder, provided by marist university. There is no official IBM Z GitHub Actions runner, so we build one inspired by anup-kodlekere/gaplib. Future updates to actions-runner might need an updated patch. The .net version number patch has been separated into a separate file to avoid a need for constantly changing the patch. + +### Setup + +1. request an account at https://linuxone.cloud.marist.edu/#/login and create a VM +2. install podman: + +``` +sudo apt install podman +``` + +3. request an action runner authentication token via the github project website: +``` +Settings->Actions->Runners->New self-hosted runner +``` +4. setup the repo and token in the actions_runner_conf.sh file: +``` +REPO="fneddy/zlib" +TOKEN="AAGXXXXXXXXXXXXXXXXXXXXXXXXXX" +LABEL="S390X" +``` +5. build the action runner container: +``` +podman build --squash -f actions_runner_ubuntu.Dockerfile --tag zlib/actions_runner . +``` +6. start the actions runner via either: + +- directly running it from cli +``` +podman run --rm -it zlib/actions_runner +``` +- let systemd quadlet hooks start it automatically +``` +mkdir -p $HOME/.config/containers/systemd/ +cp actions_runner.container $HOME/.config/containers/systemd/ +systemctl --user daemon-reload +systemctl --user start zlib_actions_runner +sudo -E loginctl enable-linger $USER +``` diff --git a/contrib/s390x/self-hosted-builder/actions_runner.container b/contrib/s390x/self-hosted-builder/actions_runner.container new file mode 100644 index 000000000..d81bd9dc8 --- /dev/null +++ b/contrib/s390x/self-hosted-builder/actions_runner.container @@ -0,0 +1,9 @@ +[Unit] +Description=GitHub Actions Worker for zlib + +[Container] +Image=zlib/actions_runner + +[Install] +WantedBy=multi-user.target default.target + diff --git a/contrib/s390x/self-hosted-builder/actions_runner.sh b/contrib/s390x/self-hosted-builder/actions_runner.sh new file mode 100755 index 000000000..222bd1789 --- /dev/null +++ b/contrib/s390x/self-hosted-builder/actions_runner.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# +# action runner startup script. +# +# Expects the following environment variables: +# +# - REPO= +# - TOKEN=<***> +# - LABEL= +# + +set -e -u +source /etc/actions_runner_conf + +# Validate required environment variables +if [ -z "${REPO:-}" ] || [ -z "${TOKEN:-}" ] || [ -z "${LABEL:-}" ]; then + echo "Error: REPO and/or TOKEN and/or LABEL environment variables not found" + exit 1 +fi + +./config.sh \ + --unattended \ + --url "https://github.com/$REPO" \ + --token "$TOKEN" \ + --disableupdate \ + --name "${LABEL}_actions_runner" \ + --replace \ + --labels "$LABEL" + +# Run the actions-runner +./run.sh diff --git a/contrib/s390x/self-hosted-builder/actions_runner_conf.sh b/contrib/s390x/self-hosted-builder/actions_runner_conf.sh new file mode 100644 index 000000000..ed691bc95 --- /dev/null +++ b/contrib/s390x/self-hosted-builder/actions_runner_conf.sh @@ -0,0 +1,3 @@ +REPO="fneddy/zlib" +TOKEN="AAG..." +LABEL="S390X" diff --git a/contrib/s390x/self-hosted-builder/actions_runner_ubuntu.Dockerfile b/contrib/s390x/self-hosted-builder/actions_runner_ubuntu.Dockerfile new file mode 100644 index 000000000..912507098 --- /dev/null +++ b/contrib/s390x/self-hosted-builder/actions_runner_ubuntu.Dockerfile @@ -0,0 +1,52 @@ +FROM ubuntu:24.10 + +ARG RUNNER_PATCH="https://github.com/anup-kodlekere/gaplib/raw/1a5e012e8af1ff96f482f493ba1c27eb115930af/patches/runner-main-sdk8-s390x.patch" +ARG RUNNER_REPO="https://github.com/actions/runner" +ARG ACTIONS_RUNNER_SCRIPT="actions_runner.sh" +ARG ACTIONS_RUNNER_CONFIG="actions_runner_conf.sh" + +ENV DEBIAN_FRONTEND=noninteractive + +# install dependencies +RUN apt-get -qq update -y && \ + apt-get -qq -y install wget git sudo curl dotnet-sdk-8.0 && \ + apt-get autoclean + +# load actions runner +RUN cd /tmp && \ + git clone -q ${RUNNER_REPO} && \ + cd runner && \ + git checkout main -b build && \ + wget ${RUNNER_PATCH} -O runner.patch && \ + git apply runner.patch && \ + sed -i'' -e /version/s/8......\"$/$8.0.100\"/ src/global.json + +# build actions runner +RUN cd /tmp/runner/src && \ + ./dev.sh layout && \ + ./dev.sh package && \ + rm -rf /root/.dotnet /root/.nuget + +# setup actions runner env +RUN useradd -c "Actions Runner" -m actions_runner && \ + usermod -L actions_runner + +RUN tar -xf /tmp/runner/_package/*.tar.gz -C /home/actions_runner && \ + chown -R actions_runner:actions_runner /home/actions_runner + +# cleanup +RUN rm -rf /tmp/runner + +# install build utils for the CI +RUN apt-get -qq update -y && \ + apt-get -qq -y install build-essential cmake ninja-build clang g++ \ + python3 flex bison jq pkg-config rsync && \ + apt-get autoclean + +# setup container config +USER actions_runner +EXPOSE 443 +COPY --chmod=0555 ${ACTIONS_RUNNER_SCRIPT} /usr/bin/actions_runner +COPY --chmod=0444 ${ACTIONS_RUNNER_CONFIG} /etc/actions_runner_conf +WORKDIR /home/actions_runner +ENTRYPOINT ["/usr/bin/actions_runner"]