Change CI to only download source image once

.github/workflows/build_test_images.yaml

@@ -13,6 +13,7 @@ permissions:
 on:  # yamllint disable-line rule:truthy
   workflow_call:
 jobs:
+
   read_builds:
     runs-on: ubuntu-latest
     outputs:
@@ -28,9 +29,37 @@ jobs:
         id: builds-as-json
         run: ./bin/builds-as-json
 
+  ensure_source_image:
+    needs:
+      - read_builds
+    runs-on: ubuntu-latest
+    outputs:
+      source_image: ${{ steps.download_source.outputs.source_image_name }}
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Install script dependencies
+        run: pip install -r ./requirements.txt
+
+      - name: Write OpenStack credentials
+        run: echo "$OS_CLOUDS" > ./clouds.yaml
+        env:
+          OS_CLOUDS: ${{ secrets.OS_CLOUDS }}
+
+      - name: Download source image and upload to OpenStack
+        id: download_source
+        run: ./bin/ensure-source-image
+        env:
+          BUILD_MATRIX: ${{ needs.read_builds.outputs.builds }}
+          REPO_ROOT: ${{ github.workspace }}
+          GH_ACTIONS_RUN: "true"
+          OS_CLOUD: ${{ vars.TARGET_CLOUD }}
+
   build_images:
     needs:
       - read_builds
+      - ensure_source_image
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -68,6 +97,8 @@ jobs:
           ENVIRONMENT: ${{ vars.TARGET_CLOUD }}
           PACKER_TEMPLATE: ${{ matrix.template }}
           ENV_VAR_FILES: ${{ matrix.var-files }}
+          PKR_VAR_source_image_name: ${{ needs.ensure_source_image.outputs.source_image }}
+          SKIP_SOURCE_IMAGE: "true"
 
       - name: Install cosign
         uses: sigstore/cosign-installer@v3
@@ -128,6 +159,30 @@ jobs:
             cosign-bundle-url: ${{ steps.publish-image.outputs.cosign-bundle-url }}
             manifest-extra: ${{ steps.build-image.outputs.manifest-extra }}
 
+  delete_source_image:
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - build_images
+      - ensure_source_image
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Install script dependencies
+        run: pip install -r ./requirements.txt
+
+      - name: Write OpenStack credentials
+        run: echo "$OS_CLOUDS" > ./clouds.yaml
+        env:
+          OS_CLOUDS: ${{ secrets.OS_CLOUDS }}
+
+      - name: Delete source image
+        run: ./bin/cleanup-source-image
+        env:
+          PKR_VAR_source_image_name: ${{ needs.ensure_source_image.outputs.source_image }}
+          OS_CLOUD: ${{ vars.TARGET_CLOUD }}
+
   publish_manifest:
     # this job should always run, but needs to run after the build matrix
     needs:

bin/build-image

@@ -27,12 +27,8 @@ cleanup() {
   if [ -n "$PKR_VAR_floating_ip" ]; then
     openstack floating ip delete "$PKR_VAR_floating_ip"
   fi
-  # Delete the source image artefacts
-  if [ -n "$PKR_VAR_source_image_name" ]; then
-    openstack image delete "$PKR_VAR_source_image_name"
-    if [ -f "$PKR_VAR_source_image_name.download" ]; then
-      rm "$PKR_VAR_source_image_name.download"
-    fi
+  if [[ "${SKIP_SOURCE_IMAGE}" != "true" ]]; then
+    "$REPO_ROOT"/bin/cleanup-source-image
   fi
 }
 trap cleanup EXIT
@@ -45,16 +41,9 @@ do
   sleep 30
 done
 
-PKR_VAR_source_image_name="packer-$(uuidgen | tr '[:upper:]' '[:lower:]')"
-curl -Lo "$PKR_VAR_source_image_name.download" "$SOURCE_IMAGE_URL"
-openstack image create \
-  --progress \
-  --private \
-  --container-format "${SOURCE_IMAGE_CONTAINER_FORMAT:-bare}" \
-  --disk-format "${SOURCE_IMAGE_DISK_FORMAT:-qcow2}" \
-  --file "$PKR_VAR_source_image_name.download" \
-  "${SOURCE_IMAGE_PROPERTIES[@]}" \
-  "$PKR_VAR_source_image_name"
+if [[ "${SKIP_SOURCE_IMAGE}" != "true" ]]; then
+  source "$REPO_ROOT"/bin/ensure-source-image
+fi
 
 export PACKER_LOG=1 PKR_VAR_floating_ip PKR_VAR_source_image_name
 PACKER_ARGS=("${PACKER_VAR_FILES_ARGS[@]}" "${PACKER_EXTRA_ARGS[@]}")

bin/cleanup-source-image

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# Delete the source image artefacts
+if [ -n "$PKR_VAR_source_image_name" ]; then
+  openstack image delete "$PKR_VAR_source_image_name"
+  if [ -f "$PKR_VAR_source_image_name.download" ]; then
+    rm "$PKR_VAR_source_image_name.download"
+  fi
+else
+  echo "Failed to delete source image, PKR_VAR_source_image_name not set"
+fi

bin/ensure-source-image

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# Get source image from build matrix
+if [[ $GH_ACTIONS_RUN == "true" ]]; then
+  SOURCE_IMAGE_ENV=$(echo "$BUILD_MATRIX" | jq -r '
+    .[]."var-files" 
+    | split(",")[] 
+    | select(startswith("ubuntu-"))
+    ' | sort -u)
+  SOURCE_IMAGE_ENV_COUNT=$(echo "$SOURCE_IMAGE_ENV" | grep -c .)
+
+  if [ "$SOURCE_IMAGE_ENV_COUNT" -eq 1 ]; then
+    export ENV_VAR_FILES=$SOURCE_IMAGE_ENV
+    echo "$REPO_ROOT"
+    source "$REPO_ROOT/bin/env-vars"
+  else
+    echo "Expecting all images to have same source image, but found $SOURCE_IMAGE_ENV_COUNT"
+    exit 1
+  fi
+fi
+
+# Download source image
+PACKER_UUID=$(uuidgen | tr '[:upper:]' '[:lower:]')
+export PKR_VAR_source_image_name="packer-$PACKER_UUID"
+curl -Lo "$PKR_VAR_source_image_name.download" "$SOURCE_IMAGE_URL"
+openstack image create \
+  --progress \
+  --private \
+  --container-format "${SOURCE_IMAGE_CONTAINER_FORMAT:-bare}" \
+  --disk-format "${SOURCE_IMAGE_DISK_FORMAT:-qcow2}" \
+  --file "$PKR_VAR_source_image_name.download" \
+  "${SOURCE_IMAGE_PROPERTIES[@]}" \
+  "$PKR_VAR_source_image_name"
+
+if [[ $GH_ACTIONS_RUN == "true" ]]; then
+  echo "source_image_name=$PKR_VAR_source_image_name" >>"$GITHUB_OUTPUT"
+fi