From 8f6da2e55742da30af10f64f9edc1be62e32a8c0 Mon Sep 17 00:00:00 2001 From: Devis Lucato Date: Mon, 20 Jan 2025 13:14:58 +0100 Subject: [PATCH] Multi-arch dockerization --- .github/workflows/docker-amd64.yml | 10 +-- .github/workflows/docker-arm64.yml | 18 +---- .github/workflows/docker-multiarch.yml | 98 ++++++++++++++++++++++++++ KernelMemory.sln | 2 + tools/dev/dockerize-amd64.sh | 60 +++++++--------- tools/dev/dockerize-arm64.sh | 63 +++++++---------- tools/dev/dockerize-multiarch.sh | 53 ++++++++++++++ 7 files changed, 204 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/docker-multiarch.yml create mode 100755 tools/dev/dockerize-multiarch.sh diff --git a/.github/workflows/docker-amd64.yml b/.github/workflows/docker-amd64.yml index 2401345e2..db10c18ae 100644 --- a/.github/workflows/docker-amd64.yml +++ b/.github/workflows/docker-amd64.yml @@ -20,11 +20,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: linux/amd64 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -38,13 +33,10 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - # list of Docker images to use as base name for tags images: | ${{ vars.DOCKERHUB_USERNAME }}/service - # Docker tags to generate tags: | - type=raw,priority=999,value=latest - type=raw,priority=800,value=${{ github.event.inputs.docker_tag }} + type=raw,priority=800,value=${{ github.event.inputs.docker_tag }}-amd64 - name: Build and push uses: docker/build-push-action@v6 diff --git a/.github/workflows/docker-arm64.yml b/.github/workflows/docker-arm64.yml index d6fae642d..2b63a9ea2 100644 --- a/.github/workflows/docker-arm64.yml +++ b/.github/workflows/docker-arm64.yml @@ -12,27 +12,14 @@ on: jobs: build-and-push-image: - runs-on: macos-latest + runs-on: ubuntu-24.04-arm env: DOCKER_TARGET_PLATFORM: linux/arm64 steps: - # https://github.com/actions/runner/issues/1456 - - name: Setup docker (missing on MacOS) - if: runner.os == 'macos' - run: | - export HOMEBREW_NO_AUTO_UPDATE=1 - brew install docker colima - colima start - - name: Checkout code uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: linux/arm64 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -46,12 +33,9 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - # list of Docker images to use as base name for tags images: | ${{ vars.DOCKERHUB_USERNAME }}/service - # Docker tags to generate tags: | - type=raw,priority=999,value=latest-arm64 type=raw,priority=800,value=${{ github.event.inputs.docker_tag }}-arm64 - name: Build and push diff --git a/.github/workflows/docker-multiarch.yml b/.github/workflows/docker-multiarch.yml new file mode 100644 index 000000000..95ddd4ace --- /dev/null +++ b/.github/workflows/docker-multiarch.yml @@ -0,0 +1,98 @@ +# Build Docker images and push them to docker hub + +name: Multi-arch Dockerization + +on: + workflow_dispatch: + inputs: + docker_tag: + description: "Enter KM version (for docker tag)" + required: true + default: "0.000.000000.0" + +jobs: + arm64: + runs-on: ubuntu-24.04-arm + env: + DOCKER_TARGET_PLATFORM: linux/arm64 + steps: + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + ${{ vars.DOCKERHUB_USERNAME }}/service + # Docker tags to generate + tags: | + type=raw,priority=800,value=${{ github.event.inputs.docker_tag }}-arm64 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: true + platforms: linux/arm64 + # See https://github.com/dotnet/dotnet-docker/blob/main/README.sdk.md#full-tag-listing + build-args: | + BUILD_IMAGE_TAG=9.0-noble-arm64v8 + RUN_IMAGE_TAG=9.0-alpine-arm64v8 + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} + + + amd64: + runs-on: ubuntu-latest + needs: arm64 + env: + DOCKER_TARGET_PLATFORM: linux/amd64 + steps: + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + # list of Docker images to use as base name for tags + images: | + ${{ vars.DOCKERHUB_USERNAME }}/service + # Docker tags to generate + tags: | + type=raw,priority=800,value=${{ github.event.inputs.docker_tag }}-amd64 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: true + platforms: linux/amd64 + # See https://github.com/dotnet/dotnet-docker/blob/main/README.sdk.md#full-tag-listing + build-args: | + BUILD_IMAGE_TAG=9.0-noble-amd64 + RUN_IMAGE_TAG=9.0-alpine-amd64 + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} diff --git a/KernelMemory.sln b/KernelMemory.sln index e710ef232..cb069aec6 100644 --- a/KernelMemory.sln +++ b/KernelMemory.sln @@ -102,6 +102,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{ .github\workflows\docker-arm64.yml = .github\workflows\docker-arm64.yml .github\workflows\fortify.yml = .github\workflows\fortify.yml .github\workflows\securitycodescan.yml = .github\workflows\securitycodescan.yml + .github\workflows\docker-multiarch.yml = .github\workflows\docker-multiarch.yml EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "service", "service", "{87DEAE8D-138C-4FDD-B4C9-11C3A7817E8F}" @@ -348,6 +349,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dev", "dev", "{3C1C483D-2C0 tools\dev\dockerize-arm64.sh = tools\dev\dockerize-arm64.sh tools\dev\get-azure-token.py = tools\dev\get-azure-token.py tools\dev\run-unit-tests.sh = tools\dev\run-unit-tests.sh + tools\dev\dockerize-multiarch.sh = tools\dev\dockerize-multiarch.sh EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "km-cli", "km-cli", "{239F5FE9-EAEE-4FA7-A3D5-09608FBA6EED}" diff --git a/tools/dev/dockerize-amd64.sh b/tools/dev/dockerize-amd64.sh index c46ee2ec0..27ed28fdb 100755 --- a/tools/dev/dockerize-amd64.sh +++ b/tools/dev/dockerize-amd64.sh @@ -7,37 +7,31 @@ cd "$HERE/../.." USR=kernelmemory IMG=${USR}/service -TAG1=:latest -# Prompt user for TAG2 -read -p "Enter TAG2 (e.g. '0.99.250214.1'): " TAG2 +# Prompt user for VERSION +read -p "Enter VERSION (e.g. '0.99.260214.1'): " VERSION -# Ensure TAG2 starts with ':' -if [[ "${TAG2:0:1}" != ":" ]]; then - TAG2=":${TAG2}" +# Ensure VERSION ends with arch name +if [[ "${VERSION:(-6)}" != "-amd64" ]]; then + VERSION="${VERSION}-amd64" fi -set +e -docker rmi ${IMG}${TAG1} >/dev/null 2>&1 -docker rmi ${IMG}${TAG2} >/dev/null 2>&1 -set -e - # Remove images if they exist -for IMAGE_TAG in "${TAG1}" "${TAG2}"; do - if docker rmi "${IMG}${IMAGE_TAG}" >/dev/null 2>&1; then - echo "Removed local image ${IMG}${IMAGE_TAG}" +for TAG in "${VERSION}" "latest"; do + if docker rmi "${IMG}:${TAG}" >/dev/null 2>&1; then + echo "Removed local image ${IMG}:${TAG}" else - echo "Image ${IMG}${IMAGE_TAG} was not found or failed to be removed." + echo "Image ${IMG}:${TAG} was not found or failed to be removed." fi done -# Check that all images have been removed -for IMAGE_TAG in "${TAG1}" "${TAG2}"; do - if [ -z "$(docker images -q "${IMG}${IMAGE_TAG}")" ]; then - echo "All ${IMG}${IMAGE_TAG} local images have been deleted." +# Verify that all images have been removed +for TAG in "${VERSION}" "latest"; do + if [ -z "$(docker images -q "${IMG}:${TAG}")" ]; then + echo "All ${IMG}:${TAG} local images have been deleted." else - echo "Some ${IMG}${IMAGE_TAG} local images are still present:" - docker images "${IMG}${IMAGE_TAG}" + echo "Some ${IMG}:${TAG} local images are still present:" + docker images "${IMG}:${TAG}" exit 1 fi done @@ -45,21 +39,17 @@ done # See https://github.com/dotnet/dotnet-docker/blob/main/README.sdk.md#full-tag-listing docker buildx build --no-cache --load \ --platform=linux/amd64 \ - --build-arg BUILD_IMAGE_TAG=8.0-jammy-amd64 \ - --build-arg RUN_IMAGE_TAG=8.0-alpine-amd64 \ - -t ${IMG}${TAG1} -t ${IMG}${TAG2} \ + --build-arg BUILD_IMAGE_TAG=9.0-noble-amd64 \ + --build-arg RUN_IMAGE_TAG=9.0-alpine-amd64 \ + -t "${IMG}:${VERSION}" \ . -# echo "Signing in as ${USR}..." -# docker login -u ${USR} - # Push images to Docker registry -for IMAGE_TAG in "${TAG1}" "${TAG2}"; do - echo "Pushing ${IMG}${IMAGE_TAG}..." - if ! docker push "${IMG}${IMAGE_TAG}"; then - echo "Failed to push ${IMG}${IMAGE_TAG}." - exit 1 - fi -done +echo "Pushing ${IMG}:${VERSION}..." +if ! docker push "${IMG}:${VERSION}"; then + echo "Failed to push ${IMG}:${VERSION}." + exit 1 +fi + -echo "Docker images push complete." +echo "Docker image push complete." diff --git a/tools/dev/dockerize-arm64.sh b/tools/dev/dockerize-arm64.sh index 532d18758..3f978930e 100755 --- a/tools/dev/dockerize-arm64.sh +++ b/tools/dev/dockerize-arm64.sh @@ -7,41 +7,31 @@ cd "$HERE/../.." USR=kernelmemory IMG=${USR}/service -TAG1=:latest-arm64 -# Prompt user for TAG2 -read -p "Enter TAG2 (e.g. '0.99.250214.1-arm64'): " TAG2 +# Prompt user for VERSION +read -p "Enter VERSION (e.g. '0.99.260214.1'): " VERSION -# Ensure TAG2 starts with ':' and ends with '-arm64' -if [[ "${TAG2:0:1}" != ":" ]]; then - TAG2=":${TAG2}" +# Ensure VERSION ends with arch name +if [[ "${VERSION:(-6)}" != "-arm64" ]]; then + VERSION="${VERSION}-arm64" fi -if [[ "${TAG2:(-6)}" != "-arm64" ]]; then - TAG2="${TAG2}-arm64" -fi - -set +e -docker rmi ${IMG}${TAG1} >/dev/null 2>&1 -docker rmi ${IMG}${TAG2} >/dev/null 2>&1 -set -e - # Remove images if they exist -for IMAGE_TAG in "${TAG1}" "${TAG2}"; do - if docker rmi "${IMG}${IMAGE_TAG}" >/dev/null 2>&1; then - echo "Removed local image ${IMG}${IMAGE_TAG}" +for TAG in "${VERSION}" "latest"; do + if docker rmi "${IMG}:${TAG}" >/dev/null 2>&1; then + echo "Removed local image ${IMG}:${TAG}" else - echo "Image ${IMG}${IMAGE_TAG} was not found or failed to be removed." + echo "Image ${IMG}:${TAG} was not found or failed to be removed." fi done -# Check that all images have been removed -for IMAGE_TAG in "${TAG1}" "${TAG2}"; do - if [ -z "$(docker images -q "${IMG}${IMAGE_TAG}")" ]; then - echo "All ${IMG}${IMAGE_TAG} local images have been deleted." +# Verify that all images have been removed +for TAG in "${VERSION}" "latest"; do + if [ -z "$(docker images -q "${IMG}:${TAG}")" ]; then + echo "All ${IMG}:${TAG} local images have been deleted." else - echo "Some ${IMG}${IMAGE_TAG} local images are still present:" - docker images "${IMG}${IMAGE_TAG}" + echo "Some ${IMG}:${TAG} local images are still present:" + docker images "${IMG}:${TAG}" exit 1 fi done @@ -49,21 +39,16 @@ done # See https://github.com/dotnet/dotnet-docker/blob/main/README.sdk.md#full-tag-listing docker buildx build --no-cache --load \ --platform=linux/arm64 \ - --build-arg BUILD_IMAGE_TAG=8.0-jammy-arm64v8 \ - --build-arg RUN_IMAGE_TAG=8.0-alpine-arm64v8 \ - -t ${IMG}${TAG1} -t ${IMG}${TAG2} \ + --build-arg BUILD_IMAGE_TAG=9.0-noble-arm64v8 \ + --build-arg RUN_IMAGE_TAG=9.0-alpine-arm64v8 \ + -t "${IMG}:${VERSION}" \ . -# echo "Signing in as ${USR}..." -# docker login -u ${USR} - # Push images to Docker registry -for IMAGE_TAG in "${TAG1}" "${TAG2}"; do - echo "Pushing ${IMG}${IMAGE_TAG}..." - if ! docker push "${IMG}${IMAGE_TAG}"; then - echo "Failed to push ${IMG}${IMAGE_TAG}." - exit 1 - fi -done +echo "Pushing ${IMG}:${VERSION}..." +if ! docker push "${IMG}:${VERSION}"; then + echo "Failed to push ${IMG}:${VERSION}." + exit 1 +fi -echo "Docker images push complete." +echo "Docker image push complete." diff --git a/tools/dev/dockerize-multiarch.sh b/tools/dev/dockerize-multiarch.sh new file mode 100755 index 000000000..ddfca1b0e --- /dev/null +++ b/tools/dev/dockerize-multiarch.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -e + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/" +cd "$HERE/../.." + +USR=kernelmemory +IMG=${USR}/service + +# Prompt user for VERSION +read -p "Enter VERSION (e.g. '0.99.260214.1'): " VERSION + +# Remove local images if they exist +for TAG in "latest" "${VERSION}-amd64" "${VERSION}-arm64"; do + if docker rmi "${IMG}:${TAG}" >/dev/null 2>&1; then + echo "Removed local image ${IMG}:${TAG}" + else + echo "Image ${IMG}:${TAG} was not found or failed to be removed." + fi +done + +# Check that all images have been removed +for TAG in "latest" "${VERSION}-amd64" "${VERSION}-arm64"; do + if [ -z "$(docker images -q "${IMG}:${TAG}")" ]; then + echo "All ${IMG}:${TAG} local images have been deleted." + else + echo "Some ${IMG}:${TAG} local images are still present:" + docker images "${IMG}:${TAG}" + exit 1 + fi +done + +# Pull images +docker pull --platform linux/amd64 "kernelmemory/service:${VERSION}-amd64" +docker pull --platform linux/arm64 "kernelmemory/service:${VERSION}-arm64" + +# Create manifest +docker manifest create kernelmemory/service:latest \ + "kernelmemory/service:${VERSION}-amd64" \ + "kernelmemory/service:${VERSION}-arm64" + +# Add images to manifest +docker manifest annotate kernelmemory/service:latest \ + "kernelmemory/service:${VERSION}-amd64" --os linux --arch amd64 +docker manifest annotate kernelmemory/service:latest \ + "kernelmemory/service:${VERSION}-arm64" --os linux --arch arm64 + +# Publish manifest +docker manifest push kernelmemory/service:latest + + +echo "Docker image push complete."