Use SSH bastion for Leafcloud image builds

.github/builds.yaml

@@ -2,30 +2,30 @@
 # This file defines the builds that will be executed by the CI
 #####
 
-- name: ubuntu-desktop
-  template: linux-desktop
-  var-files: common,kvm,linux,ubuntu-jammy
+# - name: ubuntu-desktop
+#   template: linux-desktop
+#   var-files: common,kvm,linux,ubuntu-jammy
 
-- name: jupyter-repo2docker
-  template: jupyter-repo2docker
-  var-files: common,kvm,linux,ubuntu-jammy
+# - name: jupyter-repo2docker
+#   template: jupyter-repo2docker
+#   var-files: common,kvm,linux,ubuntu-jammy
 
-- name: linux-rstudio
-  template: linux-rstudio
-  var-files: common,kvm,linux,ubuntu-jammy
+# - name: linux-rstudio
+#   template: linux-rstudio
+#   var-files: common,kvm,linux,ubuntu-jammy
 
 - name: kubernetes-1-29-jammy
   template: kubernetes
   var-files: common,kvm,linux,ubuntu-jammy,kubernetes,kubernetes_1_29
 
-- name: kubernetes-1-30-jammy
-  template: kubernetes
-  var-files: common,kvm,linux,ubuntu-jammy,kubernetes,kubernetes_1_30
+# - name: kubernetes-1-30-jammy
+#   template: kubernetes
+#   var-files: common,kvm,linux,ubuntu-jammy,kubernetes,kubernetes_1_30
 
-- name: kubernetes-1-31-jammy
-  template: kubernetes
-  var-files: common,kvm,linux,ubuntu-jammy,kubernetes,kubernetes_1_31
+# - name: kubernetes-1-31-jammy
+#   template: kubernetes
+#   var-files: common,kvm,linux,ubuntu-jammy,kubernetes,kubernetes_1_31
 
-- name: kubernetes-1-32-jammy
-  template: kubernetes
-  var-files: common,kvm,linux,ubuntu-jammy,kubernetes,kubernetes_1_32
+# - name: kubernetes-1-32-jammy
+#   template: kubernetes
+#   var-files: common,kvm,linux,ubuntu-jammy,kubernetes,kubernetes_1_32

.github/workflows/build_test_images.yaml

@@ -30,14 +30,45 @@ jobs:
     permissions:
       contents: read
       packages: write
-      id-token: write         # required to get an OIDC token for signing
-      security-events: write  # required to upload SARIF files
+      id-token: write # required to get an OIDC token for signing
+      security-events: write # required to upload SARIF files
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
         with:
           submodules: recursive
 
+      - name: Read Bastion Variables
+        run: |
+          VAR_FILE="vars/${{ vars.TARGET_CLOUD }}/common.json"
+          if [ -f "$VAR_FILE" ]; then
+            echo "File exists!"
+          else
+            echo "File does not exist at $VAR_FILE"
+          fi
+          pwd
+          echo "BASTION_HOST=$(jq -r '.ssh_bastion_host' $VAR_FILE)" >> $GITHUB_ENV
+          echo "BASTION_USER=$(jq -r '.ssh_bastion_username' $VAR_FILE)" >> $GITHUB_ENV
+          echo "BASTION_KEY_PATH=$(jq -r '.ssh_bastion_private_key_file' $VAR_FILE)" >> $GITHUB_ENV
+
+      - name: Setup ssh
+        run: |
+          set -x
+          mkdir -p ~/.ssh
+          echo "${{ secrets[format('{0}_SSH_KEY_TEST', vars.TARGET_CLOUD)] }}" > ${{ env.BASTION_KEY_PATH }}
+          chmod 0600 ${{ env.BASTION_KEY_PATH }}
+
+          cat << EOF > ~/.ssh/config
+          Host jump
+            User "${{ env.BASTION_USER }}"
+            Hostname "${{ env.BASTION_HOST }}"
+            IdentityFile "${{ env.BASTION_KEY_PATH }}"
+          EOF
+          chmod 600 ~/.ssh/config
+
+      - name: Add bastion's ssh key to known_hosts
+        run: echo "${{ vars.BASTION_FINGERPRINTS_TEST }}" >> ~/.ssh/known_hosts
+
       - name: Write OpenStack credentials
         run: echo "$OS_CLOUDS" > ./clouds.yaml
         env:
@@ -94,8 +125,8 @@ jobs:
           scan-ref: "./${{ steps.publish-image.outputs.image-name }}"
           scanners: "vuln"
           format: table
-          exit-code: '1'
-          severity: 'CRITICAL'
+          exit-code: "1"
+          severity: "CRITICAL"
           ignore-unfixed: true
           trivyignores: .github/.trivyignore
         env:

ansible.cfg

@@ -0,0 +1,3 @@
+# [ssh_connection]
+# pipelining = True
+# retries = 10

bin/build-image

@@ -34,12 +34,12 @@ cleanup() {
 trap cleanup EXIT
 
 # Wait until we are able to allocate a FIP
-PKR_VAR_floating_ip=
-until \
-  PKR_VAR_floating_ip="$(openstack floating ip create -f value -c id "$FLOATING_IP_NETWORK")"
-do
-  sleep 30
-done
+# PKR_VAR_floating_ip=
+# until \
+#   PKR_VAR_floating_ip="$(openstack floating ip create -f value -c id "$FLOATING_IP_NETWORK")"
+# do
+#   sleep 30
+# done
 
 PKR_VAR_source_image_name="packer-$(uuidgen | tr '[:upper:]' '[:lower:]')"
 curl -Lo "$PKR_VAR_source_image_name.download" "$SOURCE_IMAGE_URL"
@@ -52,7 +52,8 @@ openstack image create \
   $SOURCE_IMAGE_PROPERTIES \
   "$PKR_VAR_source_image_name"
 
-export PACKER_LOG=1 PKR_VAR_floating_ip PKR_VAR_source_image_name
+# export PACKER_LOG=1 PKR_VAR_floating_ip PKR_VAR_source_image_name
+export PACKER_LOG=1 PKR_VAR_source_image_name
 packer build \
   --on-error=cleanup \
   $PACKER_VAR_FILES_ARGS \

packer/jupyter-repo2docker.pkr.hcl

@@ -14,6 +14,7 @@ variable "network" {
 
 variable "floating_ip" {
   type = string
+  default = null
 }
 
 variable "flavor" {
@@ -46,6 +47,18 @@ variable "ssh_username" {
   type = string
 }
 
+variable "ssh_bastion_host" {
+  type = string
+}
+
+variable "ssh_bastion_username" {
+  type = string
+}
+
+variable "ssh_bastion_private_key_file" {
+  type = string
+}
+
 variable "skip_create_image" {
   type = bool
   default = false
@@ -71,6 +84,9 @@ source "openstack" "jupyter-repo2docker" {
 
   communicator = "ssh"
   ssh_username = var.ssh_username
+  ssh_bastion_host = var.ssh_bastion_host
+  ssh_bastion_username = var.ssh_bastion_username
+  ssh_bastion_private_key_file = var.ssh_bastion_private_key_file
   ssh_clear_authorized_keys = true
 }
 
@@ -82,7 +98,7 @@ build {
     playbook_file = "${path.root}/../ansible/jupyter-repo2docker.yml"
     use_proxy = false
     extra_arguments = ["-v"]
-    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10"]
+    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10", "ANSIBLE_SSH_ARGS='-J jump'"]
   }
 
   post-processor "manifest" { }

packer/kubernetes.pkr.hcl

@@ -12,6 +12,7 @@ variable "network" {
 
 variable "floating_ip" {
   type = string
+  default = null
 }
 
 variable "flavor" {
@@ -44,6 +45,18 @@ variable "ssh_username" {
   type = string
 }
 
+variable "ssh_bastion_host" {
+  type = string
+}
+
+variable "ssh_bastion_username" {
+  type = string
+}
+
+variable "ssh_bastion_private_key_file" {
+  type = string
+}
+
 variable "skip_create_image" {
   type = bool
   default = false
@@ -404,6 +417,9 @@ source "openstack" "kubernetes" {
 
   communicator = "ssh"
   ssh_username = var.ssh_username
+  ssh_bastion_host = var.ssh_bastion_host
+  ssh_bastion_username = var.ssh_bastion_username
+  ssh_bastion_private_key_file = var.ssh_bastion_private_key_file
 }
 
 build {
@@ -548,7 +564,7 @@ build {
       "--extra-vars",
       "ubuntu_security_repo=${var.ubuntu_security_repo}",
     ]
-    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10"]
+    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10", "ANSIBLE_SSH_ARGS='-J jump -o ControlMaster=auto -o ControlPersist=240s -o PreferredAuthentications=publickey'", "ANSIBLE_SSH_PIPELINING=True"]
   }
 
   post-processor "manifest" {

packer/linux-desktop.pkr.hcl

@@ -14,6 +14,7 @@ variable "network" {
 
 variable "floating_ip" {
   type = string
+  default = null
 }
 
 variable "flavor" {
@@ -46,6 +47,18 @@ variable "ssh_username" {
   type = string
 }
 
+variable "ssh_bastion_host" {
+  type = string
+}
+
+variable "ssh_bastion_username" {
+  type = string
+}
+
+variable "ssh_bastion_private_key_file" {
+  type = string
+}
+
 variable "skip_create_image" {
   type = bool
   default = false
@@ -71,6 +84,9 @@ source "openstack" "linux-desktop" {
 
   communicator = "ssh"
   ssh_username = var.ssh_username
+  ssh_bastion_host = var.ssh_bastion_host
+  ssh_bastion_username = var.ssh_bastion_username
+  ssh_bastion_private_key_file = var.ssh_bastion_private_key_file
   ssh_clear_authorized_keys = true
 }
 
@@ -86,7 +102,7 @@ build {
       "--extra-vars",
       "desktop_enabled=yes",
     ]
-    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10"]
+    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10", "ANSIBLE_SSH_ARGS='-J jump'"]
   }
 
   post-processor "manifest" { }

packer/linux-rdp-gateway.pkr.hcl

@@ -11,6 +11,7 @@ variable "network" {
 
 variable "floating_ip" {
   type = string
+  default = null
 }
 
 variable "flavor" {
@@ -43,6 +44,18 @@ variable "ssh_username" {
   type = string
 }
 
+variable "ssh_bastion_host" {
+  type = string
+}
+
+variable "ssh_bastion_username" {
+  type = string
+}
+
+variable "ssh_bastion_private_key_file" {
+  type = string
+}
+
 variable "skip_create_image" {
   type = bool
   default = false
@@ -68,6 +81,9 @@ source "openstack" "linux-rdp-gateway" {
 
   communicator = "ssh"
   ssh_username = var.ssh_username
+  ssh_bastion_host = var.ssh_bastion_host
+  ssh_bastion_username = var.ssh_bastion_username
+  ssh_bastion_private_key_file = var.ssh_bastion_private_key_file
   ssh_clear_authorized_keys = true
 }
 
@@ -79,7 +95,7 @@ build {
     playbook_file = "${path.root}/../ansible/linux-rdp-gateway.yml"
     use_proxy = false
     extra_arguments = ["-v"]
-    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10"]
+    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10", "ANSIBLE_SSH_ARGS='-J jump'"]
   }
 
   post-processor "manifest" { }

packer/linux-rstudio.pkr.hcl

@@ -14,6 +14,7 @@ variable "network" {
 
 variable "floating_ip" {
   type = string
+  default = null
 }
 
 variable "flavor" {
@@ -46,6 +47,18 @@ variable "ssh_username" {
   type = string
 }
 
+variable "ssh_bastion_host" {
+  type = string
+}
+
+variable "ssh_bastion_username" {
+  type = string
+}
+
+variable "ssh_bastion_private_key_file" {
+  type = string
+}
+
 variable "skip_create_image" {
   type = bool
   default = false
@@ -71,6 +84,9 @@ source "openstack" "linux-rstudio" {
 
   communicator = "ssh"
   ssh_username = var.ssh_username
+  ssh_bastion_host = var.ssh_bastion_host
+  ssh_bastion_username = var.ssh_bastion_username
+  ssh_bastion_private_key_file = var.ssh_bastion_private_key_file
   ssh_clear_authorized_keys = true
 }
 
@@ -82,7 +98,7 @@ build {
     playbook_file = "${path.root}/../ansible/linux-rstudio.yml"
     use_proxy = false
     extra_arguments = ["-v"]
-    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10"]
+    ansible_env_vars = ["ANSIBLE_SSH_RETRIES=10", "ANSIBLE_SSH_ARGS='-J jump'"]
   }
 
   post-processor "manifest" { }