awslabs · bradwatsonaws · Feb 13, 2025
diff --git a/Makefile b/Makefile
@@ -16,6 +16,9 @@ AMI_VARIANT ?= amazon-eks
 AMI_VERSION ?= v$(shell date '+%Y%m%d')
 os_distro ?= al2
 arch ?= x86_64
+aws_region ?= us-west-2
+binary_bucket_region ?= us-west-2
+binary_bucket_name ?= amazon-eks
 
 ifeq ($(os_distro), al2023)
 	AMI_VARIANT := $(AMI_VARIANT)-al2023
@@ -41,7 +44,7 @@ ami_name ?= $(AMI_VARIANT)-node-$(K8S_VERSION_MINOR)-$(AMI_VERSION)
 # ami owner overrides for cn/gov-cloud
 ifeq ($(aws_region), cn-northwest-1)
 	source_ami_owners ?= 141808717104
-else ifeq ($(aws_region), us-gov-west-1)
+else ifneq ($(filter $(aws_region),us-gov-west-1 us-gov-east-1),)
 	source_ami_owners ?= 045324592363
 endif
 
@@ -50,7 +53,7 @@ k8s=1.28
 
 .PHONY: build
 build: ## Build EKS Optimized AMI, default using AL2, use os_distro=al2023 for AL2023 AMI
-	$(MAKE) k8s $(shell hack/latest-binaries.sh $(k8s))
+	$(MAKE) k8s $(shell hack/latest-binaries.sh $(k8s) $(aws_region) $(binary_bucket_region) $(binary_bucket_name))
 
 .PHONY: fmt
 fmt: ## Format the source files

diff --git a/hack/latest-binaries.sh b/hack/latest-binaries.sh
@@ -4,16 +4,25 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-if [ "$#" -ne 1 ]; then
-  echo "usage: $0 KUBERNETES_MINOR_VERSION"
+if [ "$#" -ne 4 ]; then
+  echo "usage: $0 KUBERNETES_MINOR_VERSION AWS_REGION BINARY_BUCKET_REGION BINARY_BUCKET_NAME"
   exit 1
 fi
 
 MINOR_VERSION="${1}"
+AWS_REGION="${2}"
+BINARY_BUCKET_REGION="${3}"
+BINARY_BUCKET_NAME="${4}"
+
+# pass in the --no-sign-request flag if crossing partitions from a us-gov region to a non us-gov region
+NO_SIGN_REQUEST=""
+if [[ "${AWS_REGION}" == *"us-gov"* ]] && [[ "${BINARY_BUCKET_REGION}" != *"us-gov"* ]]; then
+  NO_SIGN_REQUEST="--no-sign-request"
+fi
 
 # retrieve the available "VERSION/BUILD_DATE" prefixes (e.g. "1.28.1/2023-09-14")
 # from the binary object keys, sorted in descending semver order, and pick the first one
-LATEST_BINARIES=$(aws s3api list-objects-v2 --bucket amazon-eks --prefix "${MINOR_VERSION}" --query 'Contents[*].[Key]' --output text | grep linux | cut -d'/' -f-2 | sort -Vru | head -n1)
+LATEST_BINARIES=$(aws s3api list-objects-v2 "${NO_SIGN_REQUEST}" --region "${BINARY_BUCKET_REGION}" --bucket "${BINARY_BUCKET_NAME}" --prefix "${MINOR_VERSION}" --query 'Contents[*].[Key]' --output text | grep linux | cut -d'/' -f-2 | sort -Vru | head -n1)
 
 if [ "${LATEST_BINARIES}" == "None" ]; then
   echo >&2 "No binaries available for minor version: ${MINOR_VERSION}"

diff --git a/templates/al2/provisioners/install-worker.sh b/templates/al2/provisioners/install-worker.sh
@@ -20,6 +20,7 @@ validate_env_set() {
   )
 }
 
+validate_env_set AWS_REGION
 validate_env_set BINARY_BUCKET_NAME
 validate_env_set BINARY_BUCKET_REGION
 validate_env_set CACHE_CONTAINER_IMAGES
@@ -124,6 +125,14 @@ else
   sudo yum install -y awscli
 fi
 
+################################################################################
+### AWS credentials ############################################################
+################################################################################
+
+# check for AWS credentials and store result in AWS_CREDS_OK
+AWS_CREDS_OK=$(aws sts get-caller-identity >/dev/null 2>&1 && echo true || echo false)
+echo "AWS credentials available: ${AWS_CREDS_OK}"
+
 ################################################################################
 ### systemd ####################################################################
 ################################################################################
@@ -275,17 +284,23 @@ fi
 S3_URL_BASE="https://$BINARY_BUCKET_NAME.s3.$BINARY_BUCKET_REGION.$S3_DOMAIN/$KUBERNETES_VERSION/$KUBERNETES_BUILD_DATE/bin/linux/$ARCH"
 S3_PATH="s3://$BINARY_BUCKET_NAME/$KUBERNETES_VERSION/$KUBERNETES_BUILD_DATE/bin/linux/$ARCH"
 
+# pass in the --no-sign-request flag if crossing partitions from a us-gov region to a non us-gov region
+NO_SIGN_REQUEST=""
+if [[ "$AWS_REGION" == *"us-gov"* ]] && [[ "$BINARY_BUCKET_REGION" != *"us-gov"* ]]; then
+  NO_SIGN_REQUEST="--no-sign-request"
+fi
+
 BINARIES=(
   kubelet
   aws-iam-authenticator
 )
 for binary in ${BINARIES[*]}; do
-  if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
-    echo "AWS cli present - using it to copy binaries from s3."
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$binary .
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$binary.sha256 .
+  if [ "$AWS_CREDS_OK" = "true" ]; then
+    echo "AWS credentials present - using them to copy binaries from s3."
+    aws s3 cp --region $BINARY_BUCKET_REGION $NO_SIGN_REQUEST $S3_PATH/$binary .
+    aws s3 cp --region $BINARY_BUCKET_REGION $NO_SIGN_REQUEST $S3_PATH/$binary.sha256 .
   else
-    echo "AWS cli missing - using wget to fetch binaries from s3. Note: This won't work for private bucket."
+    echo "AWS credentials missing - using wget to fetch binaries from s3. Note: This won't work for private bucket."
     sudo wget $S3_URL_BASE/$binary
     sudo wget $S3_URL_BASE/$binary.sha256
   fi
@@ -314,12 +329,12 @@ if [ "$PULL_CNI_FROM_GITHUB" = "true" ]; then
   sudo sha512sum -c "${CNI_PLUGIN_FILENAME}.tgz.sha512"
   rm "${CNI_PLUGIN_FILENAME}.tgz.sha512"
 else
-  if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
-    echo "AWS cli present - using it to copy binaries from s3."
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/${CNI_PLUGIN_FILENAME}.tgz .
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/${CNI_PLUGIN_FILENAME}.tgz.sha256 .
+  if [ "$AWS_CREDS_OK" = "true" ]; then
+    echo "AWS credentials present - using them to copy binaries from s3."
+    aws s3 cp --region $BINARY_BUCKET_REGION $NO_SIGN_REQUEST $S3_PATH/${CNI_PLUGIN_FILENAME}.tgz .
+    aws s3 cp --region $BINARY_BUCKET_REGION $NO_SIGN_REQUEST $S3_PATH/${CNI_PLUGIN_FILENAME}.tgz.sha256 .
   else
-    echo "AWS cli missing - using wget to fetch cni binaries from s3. Note: This won't work for private bucket."
+    echo "AWS credentials missing - using wget to fetch binaries from s3. Note: This won't work for private bucket."
     sudo wget "$S3_URL_BASE/${CNI_PLUGIN_FILENAME}.tgz"
     sudo wget "$S3_URL_BASE/${CNI_PLUGIN_FILENAME}.tgz.sha256"
   fi
@@ -374,12 +389,14 @@ sudo chmod +x /etc/eks/max-pods-calculator.sh
 ################################################################################
 ### ECR CREDENTIAL PROVIDER ####################################################
 ################################################################################
+
 ECR_CREDENTIAL_PROVIDER_BINARY="ecr-credential-provider"
-if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
-  echo "AWS cli present - using it to copy ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3."
-  aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$ECR_CREDENTIAL_PROVIDER_BINARY .
+
+if [ "$AWS_CREDS_OK" = "true" ]; then
+  echo "AWS credentials present - using them to copy ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3."
+  aws s3 cp --region $BINARY_BUCKET_REGION $NO_SIGN_REQUEST $S3_PATH/$ECR_CREDENTIAL_PROVIDER_BINARY .
 else
-  echo "AWS cli missing - using wget to fetch ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3. Note: This won't work for private bucket."
+  echo "AWS credentials missing - using wget to fetch ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3. Note: This won't work for private bucket."
   sudo wget "$S3_URL_BASE/$ECR_CREDENTIAL_PROVIDER_BINARY"
 fi
 sudo chmod +x $ECR_CREDENTIAL_PROVIDER_BINARY
@@ -505,7 +522,7 @@ if yum list installed | grep amazon-ssm-agent; then
 else
   if ! [[ -z "${SSM_AGENT_VERSION}" ]]; then
     echo "Installing amazon-ssm-agent@${SSM_AGENT_VERSION} from S3"
-    sudo yum install -y https://s3.${BINARY_BUCKET_REGION}.${S3_DOMAIN}/amazon-ssm-${BINARY_BUCKET_REGION}/${SSM_AGENT_VERSION}/linux_${ARCH}/amazon-ssm-agent.rpm
+    sudo yum install -y https://s3.${AWS_REGION}.${S3_DOMAIN}/amazon-ssm-${AWS_REGION}/${SSM_AGENT_VERSION}/linux_${ARCH}/amazon-ssm-agent.rpm
   else
     echo "Installing amazon-ssm-agent from AL core repository"
     sudo yum install -y amazon-ssm-agent

diff --git a/templates/al2/template.json b/templates/al2/template.json
@@ -198,6 +198,7 @@
       "script": "{{template_dir}}/provisioners/install-worker.sh",
       "environment_vars": [
         "AWS_ACCESS_KEY_ID={{user `aws_access_key_id`}}",
+        "AWS_REGION={{user `aws_region`}}",
         "AWS_SECRET_ACCESS_KEY={{user `aws_secret_access_key`}}",
         "AWS_SESSION_TOKEN={{user `aws_session_token`}}",
         "BINARY_BUCKET_NAME={{user `binary_bucket_name`}}",

diff --git a/templates/al2023/provisioners/install-worker.sh b/templates/al2023/provisioners/install-worker.sh
@@ -20,6 +20,7 @@ validate_env_set() {
   )
 }
 
+validate_env_set AWS_REGION
 validate_env_set BINARY_BUCKET_NAME
 validate_env_set BINARY_BUCKET_REGION
 validate_env_set CONTAINERD_VERSION
@@ -119,6 +120,14 @@ else
   sudo dnf install -y awscli
 fi
 
+################################################################################
+### AWS credentials ############################################################
+################################################################################
+
+# check for AWS credentials and store result in AWS_CREDS_OK
+AWS_CREDS_OK=$(aws sts get-caller-identity >/dev/null 2>&1 && echo true || echo false)
+echo "AWS credentials available: ${AWS_CREDS_OK}"
+
 ###############################################################################
 ### Containerd setup ##########################################################
 ###############################################################################
@@ -162,16 +171,22 @@ fi
 S3_URL_BASE="https://$BINARY_BUCKET_NAME.s3.$BINARY_BUCKET_REGION.$S3_DOMAIN/$KUBERNETES_VERSION/$KUBERNETES_BUILD_DATE/bin/linux/$ARCH"
 S3_PATH="s3://$BINARY_BUCKET_NAME/$KUBERNETES_VERSION/$KUBERNETES_BUILD_DATE/bin/linux/$ARCH"
 
+# pass in the --no-sign-request flag if crossing partitions from a us-gov region to a non us-gov region
+NO_SIGN_REQUEST=""
+if [[ "$AWS_REGION" == *"us-gov"* ]] && [[ "$BINARY_BUCKET_REGION" != *"us-gov"* ]]; then
+  NO_SIGN_REQUEST="--no-sign-request"
+fi
+
 BINARIES=(
   kubelet
 )
 for binary in ${BINARIES[*]}; do
-  if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
-    echo "AWS cli present - using it to copy binaries from s3."
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$binary .
-    aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$binary.sha256 .
+  if [ "$AWS_CREDS_OK" = "true" ]; then
+    echo "AWS credentials present - using them to copy binaries from s3."
+    aws s3 cp --region $BINARY_BUCKET_REGION $NO_SIGN_REQUEST $S3_PATH/$binary .
+    aws s3 cp --region $BINARY_BUCKET_REGION $NO_SIGN_REQUEST $S3_PATH/$binary.sha256 .
   else
-    echo "AWS cli missing - using wget to fetch binaries from s3. Note: This won't work for private bucket."
+    echo "AWS credentials missing - using wget to fetch binaries from s3. Note: This won't work for private bucket."
     sudo wget $S3_URL_BASE/$binary
     sudo wget $S3_URL_BASE/$binary.sha256
   fi
@@ -194,11 +209,11 @@ sudo systemctl enable ebs-initialize-bin@kubelet
 
 ECR_CREDENTIAL_PROVIDER_BINARY="ecr-credential-provider"
 
-if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then
-  echo "AWS cli present - using it to copy ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3."
-  aws s3 cp --region $BINARY_BUCKET_REGION $S3_PATH/$ECR_CREDENTIAL_PROVIDER_BINARY .
+if [ "$AWS_CREDS_OK" = "true" ]; then
+  echo "AWS credentials present - using them to copy ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3."
+  aws s3 cp --region $BINARY_BUCKET_REGION $NO_SIGN_REQUEST $S3_PATH/$ECR_CREDENTIAL_PROVIDER_BINARY .
 else
-  echo "AWS cli missing - using wget to fetch ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3. Note: This won't work for private bucket."
+  echo "AWS credentials missing - using wget to fetch ${ECR_CREDENTIAL_PROVIDER_BINARY} from s3. Note: This won't work for private bucket."
   sudo wget "$S3_URL_BASE/$ECR_CREDENTIAL_PROVIDER_BINARY"
 fi
 
@@ -215,7 +230,7 @@ if dnf list installed | grep amazon-ssm-agent; then
 else
   if ! [[ -z "${SSM_AGENT_VERSION}" ]]; then
     echo "Installing amazon-ssm-agent@${SSM_AGENT_VERSION} from S3"
-    sudo dnf install -y https://s3.${BINARY_BUCKET_REGION}.${S3_DOMAIN}/amazon-ssm-${BINARY_BUCKET_REGION}/${SSM_AGENT_VERSION}/linux_${ARCH}/amazon-ssm-agent.rpm
+    sudo dnf install -y https://s3.${AWS_REGION}.${S3_DOMAIN}/amazon-ssm-${AWS_REGION}/${SSM_AGENT_VERSION}/linux_${ARCH}/amazon-ssm-agent.rpm
   else
     echo "Installing amazon-ssm-agent from AL core repository"
     sudo dnf install -y amazon-ssm-agent

diff --git a/templates/al2023/template.json b/templates/al2023/template.json
@@ -202,6 +202,7 @@
       "script": "{{template_dir}}/provisioners/install-worker.sh",
       "environment_vars": [
         "AWS_ACCESS_KEY_ID={{user `aws_access_key_id`}}",
+        "AWS_REGION={{user `aws_region`}}",
         "AWS_SECRET_ACCESS_KEY={{user `aws_secret_access_key`}}",
         "AWS_SESSION_TOKEN={{user `aws_session_token`}}",
         "BINARY_BUCKET_NAME={{user `binary_bucket_name`}}",