Stabilize the deployment process by adding check steps (#43)

* add checks and waits on resource creation

* fix lint issues

* addressing PR comments

* fix lint issues

Author: kingman

scripts/common.sh

@@ -49,7 +49,7 @@ run_terraform() {
   _TERRAFORM_RUN_DIR=$1
   shift
   terraform -chdir="${_TERRAFORM_RUN_DIR}" version
-  terraform -chdir="${_TERRAFORM_RUN_DIR}" init -input=false -migrate-state
+  terraform -chdir="${_TERRAFORM_RUN_DIR}" init -migrate-state
   terraform -chdir="${_TERRAFORM_RUN_DIR}" validate
   terraform -chdir="${_TERRAFORM_RUN_DIR}" apply -input=false -auto-approve "$@"
   unset _TERRAFORM_RUN_DIR
@@ -68,3 +68,31 @@ create_terraform_variables_file() {
   unset _TERRAFORM_RUN_DIR
   unset _TERRAFORM_VARIABLE_FILE_PATH
 }
+
+wait_for_state() {
+  _STATE_CHECK_CMD="$1"
+  _STATE_CHECK_CMD_ARGS="$2"
+  _VALIDATION_SEARCH_TERM="$3"
+  _ERROR_MESSAGE="$4"
+  _WAIT_PER_LOOP_SEC=${5:-10}
+  _MAX_TRIES=${6:-50}
+  _COUNTER=0
+
+  # shellcheck disable=SC2086
+  while ! "${_STATE_CHECK_CMD}" ${_STATE_CHECK_CMD_ARGS} | grep -i "${_VALIDATION_SEARCH_TERM}" && [ $_COUNTER -lt $_MAX_TRIES ]; do
+    sleep "${_WAIT_PER_LOOP_SEC}"
+    _COUNTER=$((_COUNTER + 1))
+  done
+  if [ $_COUNTER -eq "${_MAX_TRIES}" ]; then
+    echo "${_ERROR_MESSAGE}"
+    exit 1
+  fi
+
+  unset _STATE_CHECK_CMD
+  unset _STATE_CHECK_CMD_ARGS
+  unset _VALIDATION_SEARCH_TERM
+  unset _ERROR_MESSAGE
+  unset _WAIT_PER_LOOP_SEC
+  unset _MAX_TRIES
+  unset _COUNTER
+}

terraform/infra-setup/main.tf

@@ -118,6 +118,8 @@ module "gcp_network" {
       },
     ]
   }
+
+  depends_on = [local_file.tf_backend_config]
 }
 
 resource "google_project_iam_custom_role" "iap_admin_role" {
@@ -134,6 +136,8 @@ resource "google_project_iam_custom_role" "iap_admin_role" {
     "clientauthconfig.clients.delete",
     "clientauthconfig.clients.update"
   ]
+
+  depends_on = [local_file.tf_backend_config]
 }
 
 module "iap_bastion" {

terraform/modules/admin-lb/main.tf

@@ -30,13 +30,29 @@ resource "kubernetes_manifest" "admin-server-cert" {
   }
 }
 
+resource "null_resource" "wait_for_server_cert_creation" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/sh", "-c"]
+    command     = <<-EOT
+    . ../common.sh
+    wait_for_state kubectl "describe managedcertificate ${var.admin_server_cert_id} --namespace=${var.deployment_namespace}" 'certificate name' 'The managed server certificate was not created, please ensure that the managed certificate with ID ${var.admin_server_cert_id} is present on your GKE cluster before proceeding with the deployment.' 15 50
+    EOT
+  }
+
+  depends_on = [
+    kubernetes_manifest.admin-server-cert
+  ]
+}
+
 data "kubernetes_resource" "managed_certificate" {
   api_version = kubernetes_manifest.admin-server-cert.manifest.apiVersion
   kind        = kubernetes_manifest.admin-server-cert.manifest.kind
   metadata {
     name      = kubernetes_manifest.admin-server-cert.manifest.metadata.name
     namespace = kubernetes_manifest.admin-server-cert.manifest.metadata.namespace
   }
+
+  depends_on = [null_resource.wait_for_server_cert_creation]
 }
 
 resource "kubernetes_ingress_v1" "admin-ingress" {

terraform/modules/admin-lb/versions.tf

@@ -24,5 +24,10 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "2.19.0"
     }
+
+    null = {
+      source  = "hashicorp/null"
+      version = "3.2.1"
+    }
   }
 }

terraform/modules/emqx-ee/main.tf

@@ -36,6 +36,20 @@ module "emqx" {
   module_depends_on       = var.module_depends_on
 }
 
+resource "null_resource" "wait_for_neg_creation" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/sh", "-c"]
+    command     = <<-EOT
+    . ../common.sh
+    wait_for_state kubectl "describe svcneg ${var.mqtt_tcp_neg_id}" 'network endpoint groups' 'Network Endpoint Group for the MQTT service was not created, please ensure that the network endpoint group with ID ${var.mqtt_tcp_neg_id} is present before proceeding with the deployment. You can check this in the Cloud Console.' 30 50
+    EOT
+  }
+
+  depends_on = [
+    module.emqx.wait
+  ]
+}
+
 resource "local_file" "admin_service_manifest" {
   filename = "admin-service-manifest.yaml"
   content = templatefile("../modules/emqx-ee/admin-service-manifest.tftpl", {
@@ -63,6 +77,6 @@ data "kubernetes_resource" "tcp_neg" {
     namespace = var.deployment_namespace
   }
   depends_on = [
-    module.emqx.wait
+    null_resource.wait_for_neg_creation
   ]
 }

terraform/modules/emqx-ee/versions.tf

@@ -24,5 +24,10 @@ terraform {
       source  = "hashicorp/local"
       version = "2.4.0"
     }
+
+    null = {
+      source  = "hashicorp/null"
+      version = "3.2.1"
+    }
   }
 }

terraform/modules/emqx-oss/main.tf

@@ -36,6 +36,20 @@ module "emqx" {
   module_depends_on       = var.module_depends_on
 }
 
+resource "null_resource" "wait_for_neg_creation" {
+  provisioner "local-exec" {
+    interpreter = ["/bin/sh", "-c"]
+    command     = <<-EOT
+    . ../common.sh
+    wait_for_state kubectl "describe svcneg ${var.mqtt_tcp_neg_id}" 'network endpoint groups' 'Network Endpoint Group for the MQTT service was not created, please ensure that the network endpoint group with ID ${var.mqtt_tcp_neg_id} is present before proceeding with the deployment. You can check this in the Cloud Console.' 30 50
+    EOT
+  }
+
+  depends_on = [
+    module.emqx.wait
+  ]
+}
+
 data "kubernetes_resource" "tcp_neg" {
   api_version = "networking.gke.io/v1beta1"
   kind        = "ServiceNetworkEndpointGroup"
@@ -44,6 +58,6 @@ data "kubernetes_resource" "tcp_neg" {
     namespace = var.deployment_namespace
   }
   depends_on = [
-    module.emqx.wait
+    null_resource.wait_for_neg_creation
   ]
 }

terraform/modules/emqx-oss/versions.tf

@@ -24,5 +24,10 @@ terraform {
       source  = "hashicorp/local"
       version = "2.4.0"
     }
+
+    null = {
+      source  = "hashicorp/null"
+      version = "3.2.1"
+    }
   }
 }