Skip to content

Update .github/workflows/build_and_publish_test.yml #181

Update .github/workflows/build_and_publish_test.yml

Update .github/workflows/build_and_publish_test.yml #181

name: "TEST Self-Hosted Runners"
on:
push:
branches:
- 'testing'
tags:
- "test_rapidyenc"
release:
types: [published]
workflow_dispatch:
repository_dispatch:
pull_request:
branches:
- main
#- "test*"
permissions:
contents: write
packages: read
jobs:
build_matrix:
name: Build ${{ matrix.name }}-${{ matrix.libc }}
runs-on:
- self-hosted
- ${{ matrix.runner_label }}
strategy:
fail-fast: false
max-parallel: 8
matrix:
include:
# Alpine Linux cross-builds (musl)
# amd64
- runner_label: ubu2204
os: linux
arch: amd64
name: alpine-amd64
libc: musl
# arm64
- runner_label: ubu2204
os: linux
arch: arm64
name: alpine-arm64
libc: musl
# i386
#- runner_label: ubu2204
# os: linux
# arch: 386
# name: alpine-i386
# libc: musl
# armv7
#- runner_label: ubu2204
# os: linux
# arch: arm
# name: alpine-armv7
# libc: musl
# FreeBSD cross-builds
# amd64
#- runner_label: ubu2204
# os: freebsd
# arch: amd64
# name: freebsd-amd64
# arm64
#- runner_label: ubu2204
# os: freebsd
# arch: arm64
# name: freebsd-arm64
# i386
#- runner_label: ubu2204
# os: freebsd
# arch: 386
# name: freebsd-i386
# armv7
#- runner_label: ubu2204
# os: freebsd
# arch: arm
# name: freebsd-armv7
# OpenBSD cross-builds
# amd64
#- runner_label: ubu2204
# os: openbsd
# arch: amd64
# name: openbsd-amd64
## arm64
#- runner_label: ubu2204
# os: openbsd
# arch: arm64
# name: openbsd-arm64
## i386
#- runner_label: ubu2204
# os: openbsd
# arch: 386
# name: openbsd-i386
# NetBSD cross-builds
# amd64
#- runner_label: ubu2204
# os: netbsd
# arch: amd64
# name: netbsd-amd64
## arm64
#- runner_label: ubu2204
# os: netbsd
# arch: arm64
# name: netbsd-arm64
## i386
#- runner_label: ubu2204
# os: netbsd
# arch: 386
# name: netbsd-i386
## armv7
#- runner_label: ubu2204
# os: netbsd
# arch: arm
# name: netbsd-armv7
# macOS cross-builds
# amd64
#- runner_label: ubu2204
# os: darwin
# arch: amd64
# name: macos-amd64
## arm64
#- runner_label: ubu2204
# os: darwin
# arch: arm64
# name: macos-arm64
# Windows cross-builds
# amd64
- runner_label: ubu2204
os: windows
arch: amd64
name: windows-amd64
# arm64
#- runner_label: ubu2204
# os: windows
# arch: arm64
# name: windows-arm64
# i386
#- runner_label: ubu2204
# os: windows
# arch: 386
# name: windows-i386
# Debian 10-12 self-hosted builds amd64
# amd64
- runner_label: deb10
os: linux
arch: amd64
name: debian-10-amd64
libc: glibc
- runner_label: deb11
os: linux
arch: amd64
name: debian-11-amd64
libc: glibc
- runner_label: deb12
os: linux
arch: amd64
name: debian-12-amd64
libc: glibc
# Debian 10-12 self-hosted builds arm64
# arm64
- runner_label: deb10
os: linux
arch: arm64
name: debian-10-arm64
libc: glibc
- runner_label: deb11
os: linux
arch: arm64
name: debian-11-arm64
libc: glibc
- runner_label: deb12
os: linux
arch: arm64
name: debian-12-arm64
libc: glibc
# Debian 10-12 self-hosted builds 32-bit ARM
# armv7
#- runner_label: deb10
# os: linux
# arch: arm
# name: debian-10-armv7
# libc: glibc
#- runner_label: deb11
# os: linux
# arch: arm
# name: debian-11-armv7
# libc: glibc
#- runner_label: deb12
# os: linux
# arch: arm
# name: debian-12-armv7
# libc: glibc
# Debian 10-12 self-hosted builds i386
# i386
#- runner_label: deb10
# os: linux
# arch: 386
# name: debian-10-i386
# libc: glibc
#- runner_label: deb11
# os: linux
# arch: 386
# name: debian-11-i386
# libc: glibc
#- runner_label: deb12
# os: linux
# arch: 386
# name: debian-12-i386
# libc: glibc
# Ubuntu 20.04 - 24.04 self-hosted builds amd64
# amd64
- runner_label: ubu2004
os: linux
arch: amd64
name: ubuntu-20.04-amd64
libc: glibc
- runner_label: ubu2204
os: linux
arch: amd64
name: ubuntu-22.04-amd64
libc: glibc
- runner_label: ubu2404
os: linux
arch: amd64
name: ubuntu-24.04-amd64
libc: glibc
# Ubuntu 20.04 - 24.04 self-hosted builds arm64
# arm64
- runner_label: ubu2004
os: linux
arch: arm64
name: ubuntu-20.04-arm64
libc: glibc
- runner_label: ubu2204
os: linux
arch: arm64
name: ubuntu-22.04-arm64
libc: glibc
- runner_label: ubu2404
os: linux
arch: arm64
name: ubuntu-24.04-arm64
libc: glibc
# Ubuntu 20.04 - 24.04 self-hosted builds 32-bit ARM
# armv7
#- runner_label: ubu2004
# os: linux
# arch: arm
# name: ubuntu-20.04-armv7
# libc: glibc
#- runner_label: ubu2204
# os: linux
# arch: arm
# name: ubuntu-22.04-armv7
# libc: glibc
#- runner_label: ubu2404
# os: linux
# arch: arm
# name: ubuntu-24.04-armv7
# libc: glibc
# Ubuntu 20.04 - 24.04 self-hosted builds i386
# i386
#- runner_label: ubu2004
# os: linux
# arch: 386
# name: ubuntu-20.04-i386
# libc: glibc
#- runner_label: ubu2204
# os: linux
# arch: 386
# name: ubuntu-22.04-i386
# libc: glibc
#- runner_label: ubu2404
# os: linux
# arch: 386
# name: ubuntu-24.04-i386
# libc: glibc
steps:
- name: Show actor
run: echo "Triggered by ${{ github.actor }}"
# Block unallowed push users
- name: Block unallowed push user
if: github.event_name == 'push' && github.actor != 'go-while'
run: |
echo "Push not allowed for user ${{ github.actor }}"
exit 1
# Block unallowed PR users
- name: Block unallowed PR user
if: github.event_name == 'pull_request' && github.event.pull_request.user.login != 'go-while'
run: |
echo "Pull request not allowed for user ${{ github.event.pull_request.user.login }}"
exit 1
- name: "@Start Clean directory"
run: |
echo "deleting workspace"
rm -rf dist build ${{ github.workspace }}/
mkdir -vp ${{ github.workspace }}
- name: "Checkout code"
uses: actions/checkout@v4
# - name: Clean Go module cache
# run: |
# rm -rf ~/.cache/go-build ~/go/pkg/mod
# - name: Restore Go modules cache
# # your cache restore step here
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
cache: false
# if your self-hosted runners are not reset (i.e., they persist their workspace and their Go module cache between jobs),
# using the GitHub Actions dependency cache provides little to no benefit.
# In fact, it can sometimes cause issues (like stale or conflicting cache states)
# and may slow down your pipeline due to unnecessary cache save/restore steps.
# On self-hosted runners where the module cache persists between jobs,
# letting Go’s native cache do its job is almost always faster and more reliable.
# Results with 44 jobs:
# Before (with Actions cache): ~8 minutes
# After (cache: false): ~2m 27s
# - name: Cache Go modules
# uses: actions/cache@v4
# with:
# path: |
# ~/.cache/go-build
# ~/go/pkg/mod
# key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
# restore-keys: |
# ${{ runner.os }}-go-
# - name: "Restore go cache"
# run: runner/go-cache-restore.sh
- name: "clone rapidyenc"
run: |
cd rapidyenc
if [ ! -e rapidyenc ]; then
./clone_rapidyenc.sh
if [ ! -e rapidyenc/.git ]; then
echo "rapidyenc/ src not found, exiting"
exit 1
fi
else
echo "rapidyenc/ src exists, skipping clone"
fi
- name: "compile rapidyenc linux-amd64"
if: matrix.os == 'linux' && matrix.arch == 'amd64'
run: cd rapidyenc && ./build_rapidyenc_linux-amd64.sh
#- name: "compile rapidyenc linux-386"
# if: matrix.os == 'linux' && matrix.arch == '386'
# run: cd rapidyenc && ./build_rapidyenc_linux-386.sh
- name: "compile rapidyenc windows-amd64"
if: matrix.os == 'windows' && matrix.arch == 'amd64'
run: |
cd rapidyenc && ./crossbuild_rapidyenc_windows-amd64.sh
echo "CC=x86_64-w64-mingw32-gcc" >> $GITHUB_ENV
echo "CGO_ENABLED=1" >> $GITHUB_ENV
- name: "compile rapidyenc linux-arm64"
if: matrix.os == 'linux' && matrix.arch == 'arm64'
run: |
cd rapidyenc && ./build_rapidyenc_linux-arm64.sh
echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV
echo "CXX=aarch64-linux-gnu-g++" >> $GITHUB_ENV
echo "CGO_ENABLED=1" >> $GITHUB_ENV
#- name: "Set up Win386 cross-compiler"
# if: matrix.goarch == '386' && matrix.os == 'windows'
# run: |
# cd rapidyenc && ./build_rapidyenc_win_386.sh
# echo "CC=x86_64-w64-mingw32-gcc" >> $GITHUB_ENV
# echo "CXX=x86_64-w64-mingw32-g++" >> $GITHUB_ENV
# echo "CGO_ENABLED=1" >> $GITHUB_ENV
# Run tests with race detector for supported platforms
#- name: "Run race detector tests (amd64/linux only)"
# if: matrix.os == 'linux' && matrix.arch == 'amd64'
# run: go test -race ./...
#- name: "Run normal tests (other platforms)"
# if: ${{ !(matrix.os == 'linux' && matrix.arch == 'amd64') }}
# run: go test ./...
- name: "Set GaRuS Variables"
id: vars
env:
XLIBC: ${{ matrix.libc }}
GITHUB_REF_TYPE: ${{ github.ref_type }}
GITHUB_EVENT_NAME: ${{ github.event_name }}
GITHUB_REF_NAME: ${{ github.ref_name }}
GITHUB_SHA: ${{ github.sha }}
run: |
echo "COMPILER=SHR" >> "$GITHUB_ENV"
# If event is tag or release, use the tag; otherwise, use 0.0.0-SNAPSHOT
SAFE_VERSION="$(echo "${GITHUB_REF_NAME}" | sed 's|/|-|g')"
if [[ "${GITHUB_REF_TYPE}" == "tag" || "${GITHUB_EVENT_NAME}" == "release" ]]; then
VERSION="${SAFE_VERSION}"
else
VERSION="${SAFE_VERSION}-SNAPSHOT"
fi
# Strip leading 'v' if present
VERSION="${VERSION#v}"
SHA7="${GITHUB_SHA::7}"
if [ -n "$XLIBC" ]; then
LIBC="-$XLIBC" # libc or musl is set. prepend a - and use as final string
else
LIBC="-purego"
fi
echo "VERSION=$VERSION" >> $GITHUB_ENV
echo "SHA7=$SHA7" >> $GITHUB_ENV
echo "LIBC=$LIBC" >> $GITHUB_ENV
- name: Build ${{ matrix.name }}
run: |
set -euo pipefail
mkdir -p builds/${{ matrix.name }}/usr/bin
if [ "${{ matrix.os }}" == "windows" ]; then
binary_name=NZBreX.exe
else
binary_name=NZBreX
fi
echo "BINARY=$binary_name" >> $GITHUB_ENV
GOARCH=${{ matrix.arch }} GOOS=${{ matrix.os }} go build -ldflags="-s -w -X main.appVersion=${{ env.VERSION }}" -o builds/${{ matrix.name }}/usr/bin/$binary_name
if [ ! -f cleanHeaders.txt ]; then
echo "cleanHeaders.txt not found! Build cannot continue." >&2
exit 1
fi
if [ ! -f provider.sample.json ]; then
echo "provider.sample.json not found! Build cannot continue." >&2
exit 1
fi
if [ ! -f provider.ygg.json ]; then
echo "provider.ygg.json not found! Build cannot continue." >&2
exit 1
fi
sha256sum builds/${{ matrix.name }}/usr/bin/$binary_name > builds/${{ matrix.name }}/usr/bin/$binary_name.sha256sum
cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha256sum
sha512sum builds/${{ matrix.name }}/usr/bin/$binary_name > builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum
cat builds/${{ matrix.name }}/usr/bin/$binary_name.sha512sum
#
# packing the build
#
# .zip
zip -j "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.zip" builds/${{ matrix.name }}/usr/bin/$binary_name builds/${{ matrix.name }}/usr/bin/$binary_name.sha* cleanHeaders.txt provider.sample.json provider.ygg.json
#
# .tgz (tar + gzip)
tar -czf "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tgz" \
-C builds/${{ matrix.name }}/usr/bin $binary_name $binary_name.sha256sum $binary_name.sha512sum \
-C ${{ github.workspace }} cleanHeaders.txt provider.sample.json provider.ygg.json
#
# .xz (tar + xz)
tar -cJf "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tar.xz" \
-C builds/${{ matrix.name }}/usr/bin $binary_name $binary_name.sha256sum $binary_name.sha512sum \
-C ${{ github.workspace }} cleanHeaders.txt provider.sample.json provider.ygg.json
#
# done packing
#
# Only make .deb packages for Linux with glibc
matrix_libc="${{ matrix.libc }}"
matrix_distro="${{ matrix.distro }}"
if [ "${{ matrix.os }}" == "linux" ] && [ "${{ matrix.libc }}" = "glibc" -o -z "${{ matrix.libc }}" ]; then
mkdir -p builds/${{ matrix.name }}/DEBIAN
VERSION=$(echo "${{ env.VERSION }}" | sed 's/^v//')
if [[ ! $VERSION =~ ^[0-9] ]]; then
VERSION="0.0.$(date +%s)-$VERSION"
fi
ARCH=${{ matrix.arch }}
echo "Package: NZBreX" > builds/${{ matrix.name }}/DEBIAN/control
echo "Version: ${VERSION}" >> builds/${{ matrix.name }}/DEBIAN/control
echo "Maintainer: ${{ github.repository_owner }} <nzbrex@usenet-server.com>" >> builds/${{ matrix.name }}/DEBIAN/control
echo "Architecture: ${ARCH}" >> builds/${{ matrix.name }}/DEBIAN/control
echo "Description: NZBreX - a cmd line tool to re-upload articles missing from providers" >> builds/${{ matrix.name }}/DEBIAN/control
dpkg-deb --root-owner-group --build builds/${{ matrix.name }} NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.deb
fi
- name: "Upload binary to private GaRuS (Git-actions-Runner-upload-Server)"
run: |
upload_with_retry() {
local file="$1"
local filename="$2"
local attempt=1
local delay=30
local max_attempts=30
local FNstr=""
if [ -n "$filename" ]; then
# filename to use has been supplied in $2
FNstr=";filename=$filename"
fi
size=$(du -b $file|cut -f1)
human=$(du -h $file)
while [ $attempt -le $max_attempts ]; do
test $attempt -gt 1 && echo "Upload attempt $attempt for $file..."
if curl --silent -f -F "file=@$file$FNstr" \
-H "X-Git-Repo: ${{ github.repository }}" \
-H "X-Git-Ref: ${{ env.VERSION }}" \
-H "X-Git-SHA7: ${{ env.SHA7 }}" \
-H "X-Git-Comp: ${{ env.COMPILER }}" \
-H "X-Git-MATRIX: ${{ matrix.name }}" \
-H "X-Auth-Token: ${{ secrets.BUILD_TEST_UPLOAD_TOKEN }}" \
http://10.20.0.1:58080/upload.php; then
echo "Upload succeeded for $file size=$size [$human]"
return 0
else
echo "Upload failed for $file. Retrying in $delay seconds..."
sleep $delay
attempt=$(( attempt + 1 ))
fi
done
echo "Upload failed for $file after $max_attempts attempts."
return 1
}
set -e
for ext in zip deb tgz xz; do
file="NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.${ext}"
if [ -e "$file" ]; then
for algo in 256 512; do
sha="sha${algo}sum"
$sha "$file" > "$file.$sha"
echo -e "\n$file.$sha"
cat "$file.$sha" | cut -d" " -f1
upload_with_retry "$file.$sha"
done
upload_with_retry "$file"
fi
done
# catch the hash sums of the binary files for uploading
sha256file="builds/${{ matrix.name }}/usr/bin/${{ env.BINARY }}.sha256sum"
sha512file="builds/${{ matrix.name }}/usr/bin/${{ env.BINARY }}.sha512sum"
# finally rename the sha sums to good filenames
cp -v "$sha256file" "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum"
cp -v "$sha512file" "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum"
upload_with_retry "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum"
upload_with_retry "NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum"
#
DIST="dist.${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.$(hostname).tgz"
if [ -e "$DIST" ]; then
echo "$DIST already exists. Skipping tar."
else
ls -lha
tar -czvf "$DIST" builds/
echo "Created $DIST from builds/"
du -b "$DIST"; du -hs "$DIST";
sha256sum "$DIST" > "${DIST}.sha256sum"
sha512sum "$DIST" > "${DIST}.sha512sum"
upload_with_retry "${DIST}.sha512sum"
upload_with_retry "${DIST}.sha256sum"
upload_with_retry "${DIST}"
fi
# - name: "Upload Binary to github blob storage"
# uses: actions/upload-artifact@v4
# with:
# name: binary-${{ matrix.name }}
# path: builds/${{ matrix.name }}/usr/bin/*
# retention-days: 1
# if-no-files-found: warn
- name: "Upload Release Assets to github"
if: github.event_name == 'release'
uses: softprops/action-gh-release@v2.2.2
with:
files: |
NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.deb
NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.tgz
NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.xz
NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.zip
NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha256sum
NZBreX_${{ env.VERSION }}-${{ env.SHA7 }}-${{ matrix.name }}${{ env.LIBC }}.sha512sum
# - name: "Backup go cache"
# run: runner/go-cache-backup.sh
- name: "@End Clean directory"
run: |
echo "deleting workspace"
rm -rf dist build ${{ github.workspace }}/
mkdir -vp ${{ github.workspace }}