From 0956aee29fdb65afa4157883269101c42e93e3c0 Mon Sep 17 00:00:00 2001
From: sam-at-luther <sam.wood@luthersystems.com>
Date: Thu, 28 Aug 2025 10:19:48 -0700
Subject: [PATCH 1/3] Add AL2023

---
 eks-vpc/worker_ami.tf | 49 +++++++++++++++++++++++++++++++------------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/eks-vpc/worker_ami.tf b/eks-vpc/worker_ami.tf
index 3aff28e..db889ff 100644
--- a/eks-vpc/worker_ami.tf
+++ b/eks-vpc/worker_ami.tf
@@ -1,47 +1,71 @@
-# # Only update the AMI when k8s version or instance type has changed. This avoids accidently
-# refreshing the nodes due to a new AMI added to AWS.
+# Prefer AL2023 if available; otherwise fall back to AL2.
+# Only refresh the AMI when k8s version, instance type, or userdata version changes.
+
 locals {
   # Detect architecture from instance type
   core        = substr(var.worker_instance_type, 0, 3)
   is_graviton = contains(["a1", "c6g", "m6g", "r6g", "t4g"], local.core)
   arch        = local.is_graviton ? "arm64" : "x86_64"
+
+  # Use the actual cluster version so the AMI tracks the real control plane version
   k8s_version = aws_eks_cluster.app.version
 }
 
+# Check if AL2023 images exist for this k8s version and arch
+data "aws_ami_ids" "al2023" {
+  owners = ["amazon"]
+  filter {
+    name   = "name"
+    values = ["amazon-eks-node-al2023-${local.arch}-standard-${local.k8s_version}-v*"]
+  }
+}
+
+# Most recent AL2023 (if present)
+data "aws_ami" "al2023" {
+  most_recent = true
+  owners      = ["amazon"]
+  filter {
+    name   = "name"
+    values = ["amazon-eks-node-al2023-${local.arch}-standard-${local.k8s_version}-v*"]
+  }
+}
 
-data "aws_ami" "eks_worker" {
+# Most recent AL2 (always resolve, used as fallback)
+data "aws_ami" "al2" {
   most_recent = true
   owners      = ["amazon"]
   filter {
     name = "name"
     values = [
-      "amazon-eks-node-al2023-${local.arch}-standard-${local.k8s_version}-v*",
       local.is_graviton ? "amazon-eks-${local.arch}-node-${local.k8s_version}-v*" : "amazon-eks-node-${local.k8s_version}-v*"
     ]
   }
 }
 
-
 locals {
-  # Prefer AL2023 since they are newer, fallback to AL2
-  selected_image_id = data.aws_ami.eks_worker.id
+  # Prefer AL2023 when any matching AMI exists; otherwise AL2
+  prefer_al2023     = length(try(data.aws_ami_ids.al2023.ids, [])) > 0
+  selected_image_id = local.prefer_al2023 ? data.aws_ami.al2023.id : data.aws_ami.al2.id
 }
 
 resource "terraform_data" "image_id" {
   input = local.selected_image_id
-
   lifecycle {
     ignore_changes = [input]
   }
-
-  # pull latest AMI if user data version changes
-  triggers_replace = [aws_eks_cluster.app.version, var.worker_instance_type, var.custom_instance_userdata_version]
+  # pull latest AMI if relevant inputs change
+  triggers_replace = [
+    aws_eks_cluster.app.version,
+    var.worker_instance_type,
+    var.custom_instance_userdata_version
+  ]
 }
 
 locals {
   image_id = terraform_data.image_id.output
 }
 
+# Safety: fail hard if no AMI was resolved
 resource "null_resource" "fail_if_no_ami" {
   count = local.selected_image_id == null ? 1 : 0
   provisioner "local-exec" {
@@ -49,6 +73,7 @@ resource "null_resource" "fail_if_no_ami" {
   }
 }
 
+# Lookup metadata for the selected AMI (used by is_al2023 and outputs)
 data "aws_ami" "selected" {
   most_recent = true
   owners      = ["amazon"]
@@ -65,11 +90,9 @@ locals {
 output "is_al2023" {
   value = local.is_al2023
 }
-
 output "worker_ami_id" {
   value = data.aws_ami.selected.id
 }
-
 output "worker_ami_name" {
   value = data.aws_ami.selected.name
 }

From f40ad9078c3af75e837f28f0ed7053b996fc4196 Mon Sep 17 00:00:00 2001
From: sam-at-luther <sam.wood@luthersystems.com>
Date: Thu, 28 Aug 2025 10:24:50 -0700
Subject: [PATCH 2/3] Attempt to fix al2023

---
 eks-vpc/worker_ami.tf | 79 +++++++++++++++++++++++++++++++++----------
 1 file changed, 61 insertions(+), 18 deletions(-)

diff --git a/eks-vpc/worker_ami.tf b/eks-vpc/worker_ami.tf
index db889ff..ae83a6f 100644
--- a/eks-vpc/worker_ami.tf
+++ b/eks-vpc/worker_ami.tf
@@ -7,45 +7,86 @@ locals {
   is_graviton = contains(["a1", "c6g", "m6g", "r6g", "t4g"], local.core)
   arch        = local.is_graviton ? "arm64" : "x86_64"
 
-  # Use the actual cluster version so the AMI tracks the real control plane version
+  # Track the real control plane version
   k8s_version = aws_eks_cluster.app.version
+
+  # EKS AMIs owner (EKS official account)
+  eks_ami_owner = "602401143452"
 }
 
-# Check if AL2023 images exist for this k8s version and arch
+# Find AL2023 candidates (doesn't fail if zero)
 data "aws_ami_ids" "al2023" {
-  owners = ["amazon"]
+  owners = [local.eks_ami_owner]
   filter {
     name   = "name"
     values = ["amazon-eks-node-al2023-${local.arch}-standard-${local.k8s_version}-v*"]
   }
+  filter {
+    name   = "state"
+    values = ["available"]
+  }
+}
+
+# Find AL2 candidates (doesn't fail if zero)
+data "aws_ami_ids" "al2" {
+  owners = [local.eks_ami_owner]
+  filter {
+    name   = "name"
+    values = [
+      # arm64 uses this pattern
+      local.is_graviton ? "amazon-eks-${local.arch}-node-${local.k8s_version}-v*" :
+      # x86_64 uses this pattern
+      "amazon-eks-node-${local.k8s_version}-v*"
+    ]
+  }
+  filter {
+    name   = "state"
+    values = ["available"]
+  }
+}
+
+locals {
+  has_al2023 = length(try(data.aws_ami_ids.al2023.ids, [])) > 0
+  has_al2    = length(try(data.aws_ami_ids.al2.ids, [])) > 0
 }
 
-# Most recent AL2023 (if present)
+# Resolve the most recent AL2023 only if any exist
 data "aws_ami" "al2023" {
+  count       = local.has_al2023 ? 1 : 0
   most_recent = true
-  owners      = ["amazon"]
+  owners      = [local.eks_ami_owner]
   filter {
     name   = "name"
     values = ["amazon-eks-node-al2023-${local.arch}-standard-${local.k8s_version}-v*"]
   }
+  filter {
+    name   = "state"
+    values = ["available"]
+  }
 }
 
-# Most recent AL2 (always resolve, used as fallback)
+# Resolve the most recent AL2 only if AL2023 does not exist and AL2 exists
 data "aws_ami" "al2" {
+  count       = (!local.has_al2023 && local.has_al2) ? 1 : 0
   most_recent = true
-  owners      = ["amazon"]
+  owners      = [local.eks_ami_owner]
   filter {
     name = "name"
     values = [
-      local.is_graviton ? "amazon-eks-${local.arch}-node-${local.k8s_version}-v*" : "amazon-eks-node-${local.k8s_version}-v*"
+      local.is_graviton ? "amazon-eks-${local.arch}-node-${local.k8s_version}-v*" :
+      "amazon-eks-node-${local.k8s_version}-v*"
     ]
   }
+  filter {
+    name   = "state"
+    values = ["available"]
+  }
 }
 
 locals {
-  # Prefer AL2023 when any matching AMI exists; otherwise AL2
-  prefer_al2023     = length(try(data.aws_ami_ids.al2023.ids, [])) > 0
-  selected_image_id = local.prefer_al2023 ? data.aws_ami.al2023.id : data.aws_ami.al2.id
+  selected_image_id = local.has_al2023
+    ? data.aws_ami.al2023[0].id
+    : (local.has_al2 ? data.aws_ami.al2[0].id : null)
 }
 
 resource "terraform_data" "image_id" {
@@ -69,14 +110,14 @@ locals {
 resource "null_resource" "fail_if_no_ami" {
   count = local.selected_image_id == null ? 1 : 0
   provisioner "local-exec" {
-    command = "echo 'ERROR: No EKS worker AMI found via filters!' && exit 1"
+    command = "echo 'ERROR: No EKS worker AMI found (AL2023 or AL2) for ${local.k8s_version} ${local.arch}!' && exit 1"
   }
 }
 
-# Lookup metadata for the selected AMI (used by is_al2023 and outputs)
+# Lookup metadata for the selected AMI (and detect AL2023)
 data "aws_ami" "selected" {
-  most_recent = true
-  owners      = ["amazon"]
+  count  = local.selected_image_id != null ? 1 : 0
+  owners = [local.eks_ami_owner]
   filter {
     name   = "image-id"
     values = [local.selected_image_id]
@@ -84,15 +125,17 @@ data "aws_ami" "selected" {
 }
 
 locals {
-  is_al2023 = can(regex("al2023", lower(data.aws_ami.selected.name)))
+  is_al2023 = length(data.aws_ami.selected) > 0 ? can(regex("al2023", lower(data.aws_ami.selected[0].name))) : false
 }
 
 output "is_al2023" {
   value = local.is_al2023
 }
+
 output "worker_ami_id" {
-  value = data.aws_ami.selected.id
+  value = length(data.aws_ami.selected) > 0 ? data.aws_ami.selected[0].id : null
 }
+
 output "worker_ami_name" {
-  value = data.aws_ami.selected.name
+  value = length(data.aws_ami.selected) > 0 ? data.aws_ami.selected[0].name : null
 }

From 2701f6a14fac66ee09e05944aee9ea24ab531b77 Mon Sep 17 00:00:00 2001
From: sam-at-luther <sam.wood@luthersystems.com>
Date: Thu, 28 Aug 2025 10:28:32 -0700
Subject: [PATCH 3/3] wip

---
 eks-vpc/worker_ami.tf | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/eks-vpc/worker_ami.tf b/eks-vpc/worker_ami.tf
index ae83a6f..87a19e9 100644
--- a/eks-vpc/worker_ami.tf
+++ b/eks-vpc/worker_ami.tf
@@ -8,10 +8,8 @@ locals {
   arch        = local.is_graviton ? "arm64" : "x86_64"
 
   # Track the real control plane version
-  k8s_version = aws_eks_cluster.app.version
-
-  # EKS AMIs owner (EKS official account)
-  eks_ami_owner = "602401143452"
+  k8s_version   = aws_eks_cluster.app.version
+  eks_ami_owner = "602401143452" # EKS official AMI account
 }
 
 # Find AL2023 candidates (doesn't fail if zero)
@@ -31,12 +29,9 @@ data "aws_ami_ids" "al2023" {
 data "aws_ami_ids" "al2" {
   owners = [local.eks_ami_owner]
   filter {
-    name   = "name"
+    name = "name"
     values = [
-      # arm64 uses this pattern
-      local.is_graviton ? "amazon-eks-${local.arch}-node-${local.k8s_version}-v*" :
-      # x86_64 uses this pattern
-      "amazon-eks-node-${local.k8s_version}-v*"
+      local.is_graviton ? "amazon-eks-${local.arch}-node-${local.k8s_version}-v*" : "amazon-eks-node-${local.k8s_version}-v*"
     ]
   }
   filter {
@@ -73,8 +68,7 @@ data "aws_ami" "al2" {
   filter {
     name = "name"
     values = [
-      local.is_graviton ? "amazon-eks-${local.arch}-node-${local.k8s_version}-v*" :
-      "amazon-eks-node-${local.k8s_version}-v*"
+      local.is_graviton ? "amazon-eks-${local.arch}-node-${local.k8s_version}-v*" : "amazon-eks-node-${local.k8s_version}-v*"
     ]
   }
   filter {
@@ -84,9 +78,8 @@ data "aws_ami" "al2" {
 }
 
 locals {
-  selected_image_id = local.has_al2023
-    ? data.aws_ami.al2023[0].id
-    : (local.has_al2 ? data.aws_ami.al2[0].id : null)
+  # Put the conditional on one line to avoid parsing issues
+  selected_image_id = local.has_al2023 ? data.aws_ami.al2023[0].id : (local.has_al2 ? data.aws_ami.al2[0].id : null)
 }
 
 resource "terraform_data" "image_id" {