perf: bootstrap credential validation with curl and check apiserver connectivity before starting kubelet (#6023)

Co-authored-by: Cameron Meissner <[email protected]>

Author: cameronmeissner

e2e/validation.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"strings"
 
 	"github.com/Azure/agentbaker/e2e/config"
 	"github.com/stretchr/testify/require"
@@ -38,6 +39,13 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) {
 	stdout := execResult.stdout.String()
 	require.NotContains(s.T, stdout, "--dynamic-config-dir", "kubelet flag '--dynamic-config-dir' should not be present in /etc/default/kubelet\nContents:\n%s")
 
+	kubeletLogs := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not retrieve kubelet logs with journalctl").stdout.String()
+	require.True(
+		s.T,
+		!strings.Contains(kubeletLogs, "unable to validate bootstrap credentials") && strings.Contains(kubeletLogs, "kubelet bootstrap token credential is valid"),
+		"expected to have successfully validated bootstrap token credential before kubelet startup, but did not",
+	)
+
 	// the instructions belows expects the SSH key to be uploaded to the user pool VM.
 	// which happens as a side-effect of execCommandOnVMForScenario, it's ugly but works.
 	// maybe we should use a single ssh key per cluster, but need to be careful with parallel test runs.
@@ -62,7 +70,7 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) {
 		//"cloud-config.txt", // file with UserData
 	})
 
-	execResult = execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo curl http://168.63.129.16:32526/vmSettings", 0, "curl to wireserver failed")
+	_ = execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo curl http://168.63.129.16:32526/vmSettings", 0, "curl to wireserver failed")
 
 	execResult = execOnVMForScenarioOnUnprivilegedPod(ctx, s, "curl https://168.63.129.16/machine/?comp=goalstate -H 'x-ms-version: 2015-04-05' -s --connect-timeout 4")
 	require.Equal(s.T, "28", execResult.exitCode, "curl to wireserver should fail")

e2e/validators.go

@@ -4,13 +4,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/tidwall/gjson"
 	"net"
 	"os"
 	"regexp"
 	"strings"
 	"time"
 
+	"github.com/tidwall/gjson"
+
 	"github.com/Azure/agentbaker/e2e/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -301,7 +302,7 @@ func ValidateContainerdWASMShims(ctx context.Context, s *Scenario) {
 
 func ValidateKubeletHasNotStopped(ctx context.Context, s *Scenario) {
 	command := "sudo journalctl -u kubelet"
-	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not retrieve kubelet logs")
+	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not retrieve kubelet logs with journalctl")
 	assert.NotContains(s.T, execResult.stdout.String(), "Stopped Kubelet")
 	assert.Contains(s.T, execResult.stdout.String(), "Started Kubelet")
 }
@@ -314,7 +315,7 @@ func ValidateServicesDoNotRestartKubelet(ctx context.Context, s *Scenario) {
 
 // ValidateKubeletHasFlags checks kubelet is started with the right flags and configs.
 func ValidateKubeletHasFlags(ctx context.Context, s *Scenario, filePath string) {
-	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not get kubelet logs")
+	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not retrieve kubelet logs with journalctl")
 	configFileFlags := fmt.Sprintf("FLAG: --config=\"%s\"", filePath)
 	require.Containsf(s.T, execResult.stdout.String(), configFileFlags, "expected to find flag %s, but not found", "config")
 }

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -492,6 +492,13 @@ configureKubeletServing() {
     fi
 }
 
+ensureKubeCACert() {
+    KUBE_CA_FILE="/etc/kubernetes/certs/ca.crt"
+    mkdir -p "$(dirname "${KUBE_CA_FILE}")"
+    echo "${KUBE_CA_CRT}" | base64 -d > "${KUBE_CA_FILE}"
+    chmod 0600 "${KUBE_CA_FILE}"
+}
+
 ensureKubelet() {
     KUBELET_DEFAULT_FILE=/etc/default/kubelet
     mkdir -p /etc/default
@@ -516,11 +523,6 @@ ensureKubelet() {
         echo "AZURE_ENVIRONMENT_FILEPATH=${AZURE_ENVIRONMENT_FILEPATH}" >> "${KUBELET_DEFAULT_FILE}"
     fi
     chmod 0600 "${KUBELET_DEFAULT_FILE}"
-    
-    KUBE_CA_FILE="/etc/kubernetes/certs/ca.crt"
-    mkdir -p "$(dirname "${KUBE_CA_FILE}")"
-    echo "${KUBE_CA_CRT}" | base64 -d > "${KUBE_CA_FILE}"
-    chmod 0600 "${KUBE_CA_FILE}"
 
     if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ] || [ -n "${TLS_BOOTSTRAP_TOKEN}" ]; then
         KUBELET_TLS_DROP_IN="/etc/systemd/system/kubelet.service.d/10-tlsbootstrap.conf"

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -52,6 +52,8 @@ source "${CSE_INSTALL_FILEPATH}"
 source "${CSE_DISTRO_INSTALL_FILEPATH}"
 source "${CSE_CONFIG_FILEPATH}"
 
+logs_to_events "AKS.CSE.ensureKubeCACert" ensureKubeCACert
+
 if [[ "${DISABLE_SSH}" == "true" ]]; then
     disableSSH || exit $ERR_DISABLE_SSH
 fi
@@ -337,7 +339,6 @@ if [ "${NEEDS_CONTAINERD}" == "true" ] &&  [ "${SHOULD_CONFIG_CONTAINERD_ULIMITS
   logs_to_events "AKS.CSE.setContainerdUlimits" configureContainerdUlimits
 fi
 
-logs_to_events "AKS.CSE.ensureKubelet" ensureKubelet
 if [ "${ENSURE_NO_DUPE_PROMISCUOUS_BRIDGE}" == "true" ]; then
     logs_to_events "AKS.CSE.ensureNoDupOnPromiscuBridge" ensureNoDupOnPromiscuBridge
 fi
@@ -401,6 +402,13 @@ else
     logs_to_events "AKS.CSE.apiserverNC" "retrycmd_if_failure ${API_SERVER_CONN_RETRIES} 1 10 nc -vz ${API_SERVER_NAME} 443" || time nc -vz ${API_SERVER_NAME} 443 || VALIDATION_ERR=$ERR_K8S_API_SERVER_CONN_FAIL
 fi
 
+echo "API server connection check code: $VALIDATION_ERR"
+if [ $VALIDATION_ERR -ne 0 ]; then
+    exit $VALIDATION_ERR
+fi
+
+logs_to_events "AKS.CSE.ensureKubelet" ensureKubelet
+
 if [[ ${ID} != "mariner" ]] && [[ ${ID} != "azurelinux" ]]; then
     echo "Recreating man-db auto-update flag file and kicking off man-db update process at $(date)"
     createManDbAutoUpdateFlagFile
@@ -458,10 +466,7 @@ else
     fi
 fi
 
-echo "Custom script finished. API server connection check code:" $VALIDATION_ERR
+echo "Custom script finished."
 echo $(date),$(hostname), endcustomscript>>/opt/m
 
-exit $VALIDATION_ERR
-
-
 #EOF

parts/linux/cloud-init/artifacts/kubelet.service

@@ -20,6 +20,8 @@ ExecStartPre=/bin/mount --make-shared /var/lib/kubelet
 ExecStartPre=-/sbin/ebtables -t nat --list
 ExecStartPre=-/sbin/iptables -t nat --numeric --list
 
+ExecStartPre=/bin/bash /opt/azure/containers/validate-kubelet-credentials.sh
+
 ExecStart=/usr/local/bin/kubelet \
         --enable-server \
         --node-labels="${KUBELET_NODE_LABELS}" \

parts/linux/cloud-init/artifacts/validate-kubelet-credentials.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+set -euo pipefail
+
+# this gives us logs_to_events
+source /opt/azure/containers/provision_source.sh
+
+KUBECONFIG_PATH="${KUBECONFIG_PATH:-/var/lib/kubelet/kubeconfig}"
+BOOTSTRAP_KUBECONFIG_PATH="${BOOTSTRAP_KUBECONFIG_PATH:-/var/lib/kubelet/bootstrap-kubeconfig}"
+
+MAX_RETRIES=${VALIDATE_KUBELET_CREDENTIALS_MAX_RETRIES:-30}
+RETRY_DELAY_SECONDS=${VALIDATE_KUBELET_CREDENTIALS_RETRY_DELAY_SECONDS:-2}
+RETRY_TIMEOUT_SECONDS=${VALIDATE_KUBELET_CREDENTIALS_RETRY_TIMEOUT_SECONDS:-5}
+
+function validateBootstrapKubeconfig {
+    local kubeconfig_path=$1
+
+    cacert=$(grep -Po "(?<=certificate-authority: ).*$" < "$kubeconfig_path")
+    apiserver_url=$(grep -Po "(?<=server: ).*$" < "$kubeconfig_path")
+    bootstrap_token=$(grep -Po "(?<=token: ).*$" < "$kubeconfig_path")
+
+    if [ -z "$cacert" ]; then
+        echo "could not read cluster CA file path from $kubeconfig_path, unable to validate bootstrap credentials"
+        exit 0
+    fi
+    if [ -z "$apiserver_url" ]; then
+        echo "could not read apiserver URL from $kubeconfig_path, unable to validate bootstrap credentials"
+        exit 0
+    fi
+    if [ -z "$bootstrap_token" ]; then
+        echo "could not read bootstrap token from $kubeconfig_path, unable to validate bootstrap credentials"
+        exit 0
+    fi
+
+    local retry_count=0
+    while true; do
+        code=$(curl -sL \
+            -m $RETRY_TIMEOUT_SECONDS \
+            -o /dev/null \
+            -w "%{http_code}" \
+            -H "Accept: application/json, */*" \
+            -H "Authorization: Bearer ${bootstrap_token//\"/}" \
+            --cacert "$cacert" \
+            "${apiserver_url}/version?timeout=${RETRY_TIMEOUT_SECONDS}s")
+
+        if [ $code -ge 200 ] && [ $code -lt 400 ]; then
+            echo "(retry=$retry_count) received valid HTTP status code from apiserver: $code"
+            break
+        fi
+
+        echo "(retry=$retry_count) received invalid HTTP status code from apiserver: $code"
+
+        retry_count=$(( $retry_count + 1 ))
+        if [ $retry_count -eq $MAX_RETRIES ]; then
+            echo "unable to validate bootstrap credentials after $retry_count attempts"
+            exit 0
+        fi
+
+        sleep $RETRY_DELAY_SECONDS
+    done
+}
+
+function validateKubeletCredentials {
+    if [ -f "$KUBECONFIG_PATH" ]; then
+        echo "client credential already exists within kubeconfig: $KUBECONFIG_PATH, no need to validate bootstrap credentials"
+        exit 0
+    fi
+
+    if [ ! -f "$BOOTSTRAP_KUBECONFIG_PATH" ]; then
+        echo "no bootstrap-kubeconfig found at $BOOTSTRAP_KUBECONFIG_PATH, no bootstrap credentials to validate"
+        exit 0
+    fi
+
+    if ! which curl >/dev/null 2>&1; then
+        echo "curl is not available, unable to validate bootstrap credentials"
+        exit 0
+    fi
+
+    echo "will validate bootstrap-kubeconfig: $BOOTSTRAP_KUBECONFIG_PATH"
+    validateBootstrapKubeconfig "$BOOTSTRAP_KUBECONFIG_PATH"
+    echo "kubelet bootstrap token credential is valid"
+}
+
+logs_to_events "AKS.Runtime.validateKubeletCredentials" validateKubeletCredentials

parts/linux/cloud-init/nodecustomdata.yml

@@ -409,6 +409,13 @@ write_files:
   content: !!binary |
     {{GetVariableProperty "cloudInitData" "ensureIMDSRestrictionScript"}}
 
+- path: /opt/azure/containers/validate-kubelet-credentials.sh
+  permissions: "0755"
+  encoding: gzip
+  owner: root
+  content: !!binary |
+    {{GetVariableProperty "cloudInitData" "validateKubeletCredentialsScript"}}
+
 - path: /etc/kubernetes/certs/ca.crt
   permissions: "0600"
   encoding: base64

pkg/agent/baker_test.go

@@ -787,7 +787,7 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
 				config.KubeletConfig = map[string]string{}
 			}, nil),
 
-		Entry("AKSUbuntu1804 with kubelet client certificatet", "AKSUbuntu1804+WithKubeletClientCert", "1.18.3",
+		Entry("AKSUbuntu1804 with kubelet client certificate", "AKSUbuntu1804+WithKubeletClientCert", "1.18.3",
 			func(config *datamodel.NodeBootstrappingConfiguration) {
 				config.ContainerService.Properties.CertificateProfile = &datamodel.CertificateProfile{
 					ClientCertificate: "fooBarBaz",
@@ -798,12 +798,14 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
 				etcDefaultKubelet := o.files["/etc/default/kubelet"].value
 				etcDefaultKubeletService := o.files["/etc/systemd/system/kubelet.service"].value
 				kubeletSh := o.files["/opt/azure/containers/kubelet.sh"].value
+				validateCredentials := o.files["/opt/azure/containers/validate-kubelet-credentials.sh"].value
 				caCRT := o.files["/etc/kubernetes/certs/ca.crt"].value
 				kubeconfig := o.files["/var/lib/kubelet/kubeconfig"].value
 
 				Expect(etcDefaultKubelet).NotTo(BeEmpty())
 				Expect(etcDefaultKubeletService).NotTo(BeEmpty())
 				Expect(kubeletSh).NotTo(BeEmpty())
+				Expect(validateCredentials).ToNot(BeEmpty())
 				Expect(caCRT).NotTo(BeEmpty())
 				Expect(kubeconfig).ToNot(BeEmpty())
 
@@ -822,13 +824,15 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
 				etcDefaultKubelet := o.files["/etc/default/kubelet"].value
 				etcDefaultKubeletService := o.files["/etc/systemd/system/kubelet.service"].value
 				kubeletSh := o.files["/opt/azure/containers/kubelet.sh"].value
+				validateCredentials := o.files["/opt/azure/containers/validate-kubelet-credentials.sh"].value
 				bootstrapKubeconfig := o.files["/var/lib/kubelet/bootstrap-kubeconfig"].value
 				caCRT := o.files["/etc/kubernetes/certs/ca.crt"].value
 
 				Expect(etcDefaultKubelet).NotTo(BeEmpty())
 				Expect(bootstrapKubeconfig).NotTo(BeEmpty())
 				Expect(kubeletSh).NotTo(BeEmpty())
 				Expect(etcDefaultKubeletService).NotTo(BeEmpty())
+				Expect(validateCredentials).ToNot(BeEmpty())
 				Expect(caCRT).NotTo(BeEmpty())
 
 				Expect(bootstrapKubeconfig).To(ContainSubstring("token"))

pkg/agent/const.go

@@ -73,6 +73,7 @@ const (
 	migPartitionScript                  = "linux/cloud-init/artifacts/mig-partition.sh"
 	migPartitionSystemdService          = "linux/cloud-init/artifacts/mig-partition.service"
 	ensureIMDSRestrictionScript         = "linux/cloud-init/artifacts/ensure_imds_restriction.sh"
+	validateKubeletCredentialsScript    = "linux/cloud-init/artifacts/validate-kubelet-credentials.sh"
 
 	// scripts and service for enabling ipv6 dual stack.
 	dhcpv6SystemdService            = "linux/cloud-init/artifacts/dhcpv6.service"