feat: add explicit DNS check (#5771)

Azure · Feb 8, 2025 · c396f5a · c396f5a
1 parent e2da3a1
commit c396f5a
Show file tree

Hide file tree

Showing 266 changed files with 2,846 additions and 176 deletions.
diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh
@@ -125,6 +125,8 @@ ERR_LOOKUP_DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION_TAG=213
 # Error either getting the install mode or cleaning up container images
 ERR_CLEANUP_CONTAINER_IMAGES=214
 
+ERR_DNS_HEALTH_FAIL=215 # Error checking DNS health
+
 # For both Ubuntu and Mariner, /etc/*-release should exist.
 # For unit tests, the OS and OS_VERSION will be set in the unit test script.
 # So whether it's if or else actually doesn't matter to our unit test.
@@ -737,6 +739,28 @@ removeKubeletFlag() {
     fi
 }
 
+verify_DNS_health(){
+    local domain_name=$1
+    if [ -z "$domain_name" ]; then
+        echo "DNS domain is empty"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_no_domain=$(dig +norec +short +tries=5 +timeout=5 .)
+    if [ $? -ne 0 ]; then
+        echo "Failed to resolve root domain '.'"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_domain=$(dig +tries=5 +timeout=5 +short $domain_name)
+    ret_code=$?
+    if [ ret_code -ne 0 ] || [ -z "$dig_check_domain" ]; then
+        echo "Failed to resolve domain $domain_name return code: $ret_code"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+    echo "DNS health check passed"
+}
+
 oras_login_with_kubelet_identity() {
     local acr_url=$1
     local client_id=$2

diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh
@@ -72,6 +72,12 @@ if [[ "${SHOULD_CONFIGURE_CUSTOM_CA_TRUST}" == "true" ]]; then
     logs_to_events "AKS.CSE.configureCustomCaCertificate" configureCustomCaCertificate || exit $ERR_UPDATE_CA_CERTS
 fi
 
+domain_name="mcr.microsoft.com"
+if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
+    domain_name="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER%/}"
+fi
+verify_DNS_health $domain_name || exit $ERR_DNS_HEALTH_FAIL
+
 if [[ -n "${OUTBOUND_COMMAND}" ]]; then
     if [[ -n "${PROXY_VARS}" ]]; then
         eval $PROXY_VARS

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/line33.sh b/pkg/agent/testdata/AKSUbuntu1604+Containerd/line33.sh
@@ -66,6 +66,12 @@ if [[ "${SHOULD_CONFIGURE_CUSTOM_CA_TRUST}" == "true" ]]; then
     logs_to_events "AKS.CSE.configureCustomCaCertificate" configureCustomCaCertificate || exit $ERR_UPDATE_CA_CERTS
 fi
 
+domain_name="mcr.microsoft.com"
+if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
+    domain_name="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER%/}"
+fi
+verify_DNS_health $domain_name || exit $ERR_DNS_HEALTH_FAIL
+
 if [[ -n "${OUTBOUND_COMMAND}" ]]; then
     if [[ -n "${PROXY_VARS}" ]]; then
         eval $PROXY_VARS

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/line9.sh b/pkg/agent/testdata/AKSUbuntu1604+Containerd/line9.sh
@@ -117,6 +117,8 @@ ERR_LOOKUP_DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION_TAG=213
 
 ERR_CLEANUP_CONTAINER_IMAGES=214
 
+ERR_DNS_HEALTH_FAIL=215 
+
 if find /etc -type f,l -name "*-release" -print -quit 2>/dev/null | grep -q '.'; then
     OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(coreos)|ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }')
     OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"')
@@ -695,6 +697,28 @@ removeKubeletFlag() {
     fi
 }
 
+verify_DNS_health(){
+    local domain_name=$1
+    if [ -z "$domain_name" ]; then
+        echo "DNS domain is empty"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_no_domain=$(dig +norec +short +tries=5 +timeout=5 .)
+    if [ $? -ne 0 ]; then
+        echo "Failed to resolve root domain '.'"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_domain=$(dig +tries=5 +timeout=5 +short $domain_name)
+    ret_code=$?
+    if [ ret_code -ne 0 ] || [ -z "$dig_check_domain" ]; then
+        echo "Failed to resolve domain $domain_name return code: $ret_code"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+    echo "DNS health check passed"
+}
+
 oras_login_with_kubelet_identity() {
     local acr_url=$1
     local client_id=$2

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line33.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line33.sh
@@ -66,6 +66,12 @@ if [[ "${SHOULD_CONFIGURE_CUSTOM_CA_TRUST}" == "true" ]]; then
     logs_to_events "AKS.CSE.configureCustomCaCertificate" configureCustomCaCertificate || exit $ERR_UPDATE_CA_CERTS
 fi
 
+domain_name="mcr.microsoft.com"
+if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
+    domain_name="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER%/}"
+fi
+verify_DNS_health $domain_name || exit $ERR_DNS_HEALTH_FAIL
+
 if [[ -n "${OUTBOUND_COMMAND}" ]]; then
     if [[ -n "${PROXY_VARS}" ]]; then
         eval $PROXY_VARS

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line9.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line9.sh
@@ -117,6 +117,8 @@ ERR_LOOKUP_DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION_TAG=213
 
 ERR_CLEANUP_CONTAINER_IMAGES=214
 
+ERR_DNS_HEALTH_FAIL=215 
+
 if find /etc -type f,l -name "*-release" -print -quit 2>/dev/null | grep -q '.'; then
     OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(coreos)|ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }')
     OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"')
@@ -695,6 +697,28 @@ removeKubeletFlag() {
     fi
 }
 
+verify_DNS_health(){
+    local domain_name=$1
+    if [ -z "$domain_name" ]; then
+        echo "DNS domain is empty"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_no_domain=$(dig +norec +short +tries=5 +timeout=5 .)
+    if [ $? -ne 0 ]; then
+        echo "Failed to resolve root domain '.'"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_domain=$(dig +tries=5 +timeout=5 +short $domain_name)
+    ret_code=$?
+    if [ ret_code -ne 0 ] || [ -z "$dig_check_domain" ]; then
+        echo "Failed to resolve domain $domain_name return code: $ret_code"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+    echo "DNS health check passed"
+}
+
 oras_login_with_kubelet_identity() {
     local acr_url=$1
     local client_id=$2

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line33.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line33.sh
@@ -66,6 +66,12 @@ if [[ "${SHOULD_CONFIGURE_CUSTOM_CA_TRUST}" == "true" ]]; then
     logs_to_events "AKS.CSE.configureCustomCaCertificate" configureCustomCaCertificate || exit $ERR_UPDATE_CA_CERTS
 fi
 
+domain_name="mcr.microsoft.com"
+if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
+    domain_name="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER%/}"
+fi
+verify_DNS_health $domain_name || exit $ERR_DNS_HEALTH_FAIL
+
 if [[ -n "${OUTBOUND_COMMAND}" ]]; then
     if [[ -n "${PROXY_VARS}" ]]; then
         eval $PROXY_VARS

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line9.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line9.sh
@@ -117,6 +117,8 @@ ERR_LOOKUP_DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION_TAG=213
 
 ERR_CLEANUP_CONTAINER_IMAGES=214
 
+ERR_DNS_HEALTH_FAIL=215 
+
 if find /etc -type f,l -name "*-release" -print -quit 2>/dev/null | grep -q '.'; then
     OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(coreos)|ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }')
     OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"')
@@ -695,6 +697,28 @@ removeKubeletFlag() {
     fi
 }
 
+verify_DNS_health(){
+    local domain_name=$1
+    if [ -z "$domain_name" ]; then
+        echo "DNS domain is empty"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_no_domain=$(dig +norec +short +tries=5 +timeout=5 .)
+    if [ $? -ne 0 ]; then
+        echo "Failed to resolve root domain '.'"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_domain=$(dig +tries=5 +timeout=5 +short $domain_name)
+    ret_code=$?
+    if [ ret_code -ne 0 ] || [ -z "$dig_check_domain" ]; then
+        echo "Failed to resolve domain $domain_name return code: $ret_code"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+    echo "DNS health check passed"
+}
+
 oras_login_with_kubelet_identity() {
     local acr_url=$1
     local client_id=$2

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line33.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line33.sh
@@ -66,6 +66,12 @@ if [[ "${SHOULD_CONFIGURE_CUSTOM_CA_TRUST}" == "true" ]]; then
     logs_to_events "AKS.CSE.configureCustomCaCertificate" configureCustomCaCertificate || exit $ERR_UPDATE_CA_CERTS
 fi
 
+domain_name="mcr.microsoft.com"
+if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
+    domain_name="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER%/}"
+fi
+verify_DNS_health $domain_name || exit $ERR_DNS_HEALTH_FAIL
+
 if [[ -n "${OUTBOUND_COMMAND}" ]]; then
     if [[ -n "${PROXY_VARS}" ]]; then
         eval $PROXY_VARS

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line9.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line9.sh
@@ -117,6 +117,8 @@ ERR_LOOKUP_DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION_TAG=213
 
 ERR_CLEANUP_CONTAINER_IMAGES=214
 
+ERR_DNS_HEALTH_FAIL=215 
+
 if find /etc -type f,l -name "*-release" -print -quit 2>/dev/null | grep -q '.'; then
     OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(coreos)|ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }')
     OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"')
@@ -695,6 +697,28 @@ removeKubeletFlag() {
     fi
 }
 
+verify_DNS_health(){
+    local domain_name=$1
+    if [ -z "$domain_name" ]; then
+        echo "DNS domain is empty"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_no_domain=$(dig +norec +short +tries=5 +timeout=5 .)
+    if [ $? -ne 0 ]; then
+        echo "Failed to resolve root domain '.'"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_domain=$(dig +tries=5 +timeout=5 +short $domain_name)
+    ret_code=$?
+    if [ ret_code -ne 0 ] || [ -z "$dig_check_domain" ]; then
+        echo "Failed to resolve domain $domain_name return code: $ret_code"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+    echo "DNS health check passed"
+}
+
 oras_login_with_kubelet_identity() {
     local acr_url=$1
     local client_id=$2

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line33.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line33.sh
@@ -66,6 +66,12 @@ if [[ "${SHOULD_CONFIGURE_CUSTOM_CA_TRUST}" == "true" ]]; then
     logs_to_events "AKS.CSE.configureCustomCaCertificate" configureCustomCaCertificate || exit $ERR_UPDATE_CA_CERTS
 fi
 
+domain_name="mcr.microsoft.com"
+if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
+    domain_name="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER%/}"
+fi
+verify_DNS_health $domain_name || exit $ERR_DNS_HEALTH_FAIL
+
 if [[ -n "${OUTBOUND_COMMAND}" ]]; then
     if [[ -n "${PROXY_VARS}" ]]; then
         eval $PROXY_VARS

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line9.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line9.sh
@@ -117,6 +117,8 @@ ERR_LOOKUP_DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION_TAG=213
 
 ERR_CLEANUP_CONTAINER_IMAGES=214
 
+ERR_DNS_HEALTH_FAIL=215 
+
 if find /etc -type f,l -name "*-release" -print -quit 2>/dev/null | grep -q '.'; then
     OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(coreos)|ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }')
     OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"')
@@ -695,6 +697,28 @@ removeKubeletFlag() {
     fi
 }
 
+verify_DNS_health(){
+    local domain_name=$1
+    if [ -z "$domain_name" ]; then
+        echo "DNS domain is empty"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_no_domain=$(dig +norec +short +tries=5 +timeout=5 .)
+    if [ $? -ne 0 ]; then
+        echo "Failed to resolve root domain '.'"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_domain=$(dig +tries=5 +timeout=5 +short $domain_name)
+    ret_code=$?
+    if [ ret_code -ne 0 ] || [ -z "$dig_check_domain" ]; then
+        echo "Failed to resolve domain $domain_name return code: $ret_code"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+    echo "DNS health check passed"
+}
+
 oras_login_with_kubelet_identity() {
     local acr_url=$1
     local client_id=$2

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/line33.sh b/pkg/agent/testdata/AKSUbuntu1604+Docker/line33.sh
@@ -66,6 +66,12 @@ if [[ "${SHOULD_CONFIGURE_CUSTOM_CA_TRUST}" == "true" ]]; then
     logs_to_events "AKS.CSE.configureCustomCaCertificate" configureCustomCaCertificate || exit $ERR_UPDATE_CA_CERTS
 fi
 
+domain_name="mcr.microsoft.com"
+if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
+    domain_name="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER%/}"
+fi
+verify_DNS_health $domain_name || exit $ERR_DNS_HEALTH_FAIL
+
 if [[ -n "${OUTBOUND_COMMAND}" ]]; then
     if [[ -n "${PROXY_VARS}" ]]; then
         eval $PROXY_VARS

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/line9.sh b/pkg/agent/testdata/AKSUbuntu1604+Docker/line9.sh
@@ -117,6 +117,8 @@ ERR_LOOKUP_DISABLE_KUBELET_SERVING_CERTIFICATE_ROTATION_TAG=213
 
 ERR_CLEANUP_CONTAINER_IMAGES=214
 
+ERR_DNS_HEALTH_FAIL=215 
+
 if find /etc -type f,l -name "*-release" -print -quit 2>/dev/null | grep -q '.'; then
     OS=$(sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(coreos)|ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }')
     OS_VERSION=$(sort -r /etc/*-release | gawk 'match($0, /^(VERSION_ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }' | tr -d '"')
@@ -695,6 +697,28 @@ removeKubeletFlag() {
     fi
 }
 
+verify_DNS_health(){
+    local domain_name=$1
+    if [ -z "$domain_name" ]; then
+        echo "DNS domain is empty"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_no_domain=$(dig +norec +short +tries=5 +timeout=5 .)
+    if [ $? -ne 0 ]; then
+        echo "Failed to resolve root domain '.'"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+
+    dig_check_domain=$(dig +tries=5 +timeout=5 +short $domain_name)
+    ret_code=$?
+    if [ ret_code -ne 0 ] || [ -z "$dig_check_domain" ]; then
+        echo "Failed to resolve domain $domain_name return code: $ret_code"
+        return $ERR_DNS_HEALTH_FAIL
+    fi
+    echo "DNS health check passed"
+}
+
 oras_login_with_kubelet_identity() {
     local acr_url=$1
     local client_id=$2