Skip to content

Commit 3eeb00f

Browse files
fseldowfseldow
authored
bug: fix get kubelet identity not retrying when imds has temp 400 response (#6087)
1 parent 549a7fe commit 3eeb00f

182 files changed

Lines changed: 1276 additions & 1002 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎parts/linux/cloud-init/artifacts/cse_helpers.sh‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ ERR_CNI_VERSION_INVALID=206 # reference CNI (not azure cni) needs a valid versio
115115
ERR_ORAS_PULL_K8S_FAIL=207 # Error pulling kube-node artifact via oras from registry
116116
ERR_ORAS_PULL_CREDENTIAL_PROVIDER=208 # Error pulling credential provider artifact with oras from registry
117117
ERR_ORAS_PULL_CONTAINERD_WASM=209 # Error pulling containerd wasm artifact with oras from registry
118-
ERR_ORAS_PULL_FAIL_RESERVE_2=210 # Error pulling artifact with oras from registry with incorrect acr cache setting
118+
ERR_ORAS_IMDS_TIMEOUT=210 # Error timeout waiting for IMDS response
119119
ERR_ORAS_PULL_NETWORK_TIMEOUT=211 # Error pulling oras tokens for login
120120
ERR_ORAS_PULL_UNAUTHORIZED=212 # Error pulling artifact with oras from registry with authorization issue
121121

@@ -253,14 +253,21 @@ retrycmd_get_tarball_from_registry_with_oras() {
253253
retrycmd_get_access_token_for_oras() {
254254
retries=$1; wait_sleep=$2; url=$3
255255
for i in $(seq 1 $retries); do
256-
ACCESS_TOKEN_OUTPUT=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url")
257-
if [ -n "$ACCESS_TOKEN_OUTPUT" ]; then
256+
response=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url" -w "\n%{http_code}")
257+
ACCESS_TOKEN_OUTPUT=$(echo "$response" | sed '$d')
258+
http_code=$(echo "$response" | tail -n1)
259+
if [ -n "$ACCESS_TOKEN_OUTPUT" ] && [ "$http_code" -eq 200 ]; then
258260
echo "$ACCESS_TOKEN_OUTPUT"
259261
return 0
260262
fi
261263
sleep $wait_sleep
262264
done
263-
return $ERR_ORAS_PULL_NETWORK_TIMEOUT
265+
if [ -n "$http_code" ]; then
266+
echo "failed to retrieve kubelet identity token from IMDS, http code: $http_code, msg: $ACCESS_TOKEN_OUTPUT"
267+
return $ERR_ORAS_PULL_UNAUTHORIZED
268+
fi
269+
echo "timeout waiting for IMDS response to retrieve kubelet identity token"
270+
return $ERR_ORAS_IMDS_TIMEOUT
264271
}
265272
retrycmd_get_refresh_token_for_oras() {
266273
retries=$1; wait_sleep=$2; acr_url=$3; tenant_id=$4; ACCESS_TOKEN=$5
@@ -810,16 +817,12 @@ oras_login_with_kubelet_identity() {
810817

811818
set +x
812819
access_url="http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/&client_id=$client_id"
813-
raw_access_token=$(retrycmd_get_access_token_for_oras 10 5 $access_url)
820+
raw_access_token=$(retrycmd_get_access_token_for_oras 5 15 $access_url)
814821
ret_code=$?
815822
if [ $ret_code -ne 0 ]; then
816-
echo "failed to retrieve access token: $ret_code"
823+
echo $raw_access_token
817824
return $ret_code
818825
fi
819-
if [[ "$raw_access_token" == *"error"* ]]; then
820-
echo "failed to retrieve access token"
821-
return $ERR_ORAS_PULL_UNAUTHORIZED
822-
fi
823826
ACCESS_TOKEN=$(echo "$raw_access_token" | jq -r .access_token)
824827
if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" == "null" ]; then
825828
echo "failed to parse access token"

‎pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎pkg/agent/testdata/AKSUbuntu1604+Containerd/line9.sh‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ ERR_CNI_VERSION_INVALID=206
109109
ERR_ORAS_PULL_K8S_FAIL=207
110110
ERR_ORAS_PULL_CREDENTIAL_PROVIDER=208
111111
ERR_ORAS_PULL_CONTAINERD_WASM=209
112-
ERR_ORAS_PULL_FAIL_RESERVE_2=210
112+
ERR_ORAS_IMDS_TIMEOUT=210
113113
ERR_ORAS_PULL_NETWORK_TIMEOUT=211
114114
ERR_ORAS_PULL_UNAUTHORIZED=212
115115

@@ -236,14 +236,21 @@ retrycmd_get_tarball_from_registry_with_oras() {
236236
retrycmd_get_access_token_for_oras() {
237237
retries=$1; wait_sleep=$2; url=$3
238238
for i in $(seq 1 $retries); do
239-
ACCESS_TOKEN_OUTPUT=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url")
240-
if [ -n "$ACCESS_TOKEN_OUTPUT" ]; then
239+
response=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url" -w "\n%{http_code}")
240+
ACCESS_TOKEN_OUTPUT=$(echo "$response" | sed '$d')
241+
http_code=$(echo "$response" | tail -n1)
242+
if [ -n "$ACCESS_TOKEN_OUTPUT" ] && [ "$http_code" -eq 200 ]; then
241243
echo "$ACCESS_TOKEN_OUTPUT"
242244
return 0
243245
fi
244246
sleep $wait_sleep
245247
done
246-
return $ERR_ORAS_PULL_NETWORK_TIMEOUT
248+
if [ -n "$http_code" ]; then
249+
echo "failed to retrieve kubelet identity token from IMDS, http code: $http_code, msg: $ACCESS_TOKEN_OUTPUT"
250+
return $ERR_ORAS_PULL_UNAUTHORIZED
251+
fi
252+
echo "timeout waiting for IMDS response to retrieve kubelet identity token"
253+
return $ERR_ORAS_IMDS_TIMEOUT
247254
}
248255
retrycmd_get_refresh_token_for_oras() {
249256
retries=$1; wait_sleep=$2; acr_url=$3; tenant_id=$4; ACCESS_TOKEN=$5
@@ -762,16 +769,12 @@ oras_login_with_kubelet_identity() {
762769

763770
set +x
764771
access_url="http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/&client_id=$client_id"
765-
raw_access_token=$(retrycmd_get_access_token_for_oras 10 5 $access_url)
772+
raw_access_token=$(retrycmd_get_access_token_for_oras 5 15 $access_url)
766773
ret_code=$?
767774
if [ $ret_code -ne 0 ]; then
768-
echo "failed to retrieve access token: $ret_code"
775+
echo $raw_access_token
769776
return $ret_code
770777
fi
771-
if [[ "$raw_access_token" == *"error"* ]]; then
772-
echo "failed to retrieve access token"
773-
return $ERR_ORAS_PULL_UNAUTHORIZED
774-
fi
775778
ACCESS_TOKEN=$(echo "$raw_access_token" | jq -r .access_token)
776779
if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" == "null" ]; then
777780
echo "failed to parse access token"

‎pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line9.sh‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ ERR_CNI_VERSION_INVALID=206
109109
ERR_ORAS_PULL_K8S_FAIL=207
110110
ERR_ORAS_PULL_CREDENTIAL_PROVIDER=208
111111
ERR_ORAS_PULL_CONTAINERD_WASM=209
112-
ERR_ORAS_PULL_FAIL_RESERVE_2=210
112+
ERR_ORAS_IMDS_TIMEOUT=210
113113
ERR_ORAS_PULL_NETWORK_TIMEOUT=211
114114
ERR_ORAS_PULL_UNAUTHORIZED=212
115115

@@ -236,14 +236,21 @@ retrycmd_get_tarball_from_registry_with_oras() {
236236
retrycmd_get_access_token_for_oras() {
237237
retries=$1; wait_sleep=$2; url=$3
238238
for i in $(seq 1 $retries); do
239-
ACCESS_TOKEN_OUTPUT=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url")
240-
if [ -n "$ACCESS_TOKEN_OUTPUT" ]; then
239+
response=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url" -w "\n%{http_code}")
240+
ACCESS_TOKEN_OUTPUT=$(echo "$response" | sed '$d')
241+
http_code=$(echo "$response" | tail -n1)
242+
if [ -n "$ACCESS_TOKEN_OUTPUT" ] && [ "$http_code" -eq 200 ]; then
241243
echo "$ACCESS_TOKEN_OUTPUT"
242244
return 0
243245
fi
244246
sleep $wait_sleep
245247
done
246-
return $ERR_ORAS_PULL_NETWORK_TIMEOUT
248+
if [ -n "$http_code" ]; then
249+
echo "failed to retrieve kubelet identity token from IMDS, http code: $http_code, msg: $ACCESS_TOKEN_OUTPUT"
250+
return $ERR_ORAS_PULL_UNAUTHORIZED
251+
fi
252+
echo "timeout waiting for IMDS response to retrieve kubelet identity token"
253+
return $ERR_ORAS_IMDS_TIMEOUT
247254
}
248255
retrycmd_get_refresh_token_for_oras() {
249256
retries=$1; wait_sleep=$2; acr_url=$3; tenant_id=$4; ACCESS_TOKEN=$5
@@ -762,16 +769,12 @@ oras_login_with_kubelet_identity() {
762769

763770
set +x
764771
access_url="http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/&client_id=$client_id"
765-
raw_access_token=$(retrycmd_get_access_token_for_oras 10 5 $access_url)
772+
raw_access_token=$(retrycmd_get_access_token_for_oras 5 15 $access_url)
766773
ret_code=$?
767774
if [ $ret_code -ne 0 ]; then
768-
echo "failed to retrieve access token: $ret_code"
775+
echo $raw_access_token
769776
return $ret_code
770777
fi
771-
if [[ "$raw_access_token" == *"error"* ]]; then
772-
echo "failed to retrieve access token"
773-
return $ERR_ORAS_PULL_UNAUTHORIZED
774-
fi
775778
ACCESS_TOKEN=$(echo "$raw_access_token" | jq -r .access_token)
776779
if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" == "null" ]; then
777780
echo "failed to parse access token"

‎pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line9.sh‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ ERR_CNI_VERSION_INVALID=206
109109
ERR_ORAS_PULL_K8S_FAIL=207
110110
ERR_ORAS_PULL_CREDENTIAL_PROVIDER=208
111111
ERR_ORAS_PULL_CONTAINERD_WASM=209
112-
ERR_ORAS_PULL_FAIL_RESERVE_2=210
112+
ERR_ORAS_IMDS_TIMEOUT=210
113113
ERR_ORAS_PULL_NETWORK_TIMEOUT=211
114114
ERR_ORAS_PULL_UNAUTHORIZED=212
115115

@@ -236,14 +236,21 @@ retrycmd_get_tarball_from_registry_with_oras() {
236236
retrycmd_get_access_token_for_oras() {
237237
retries=$1; wait_sleep=$2; url=$3
238238
for i in $(seq 1 $retries); do
239-
ACCESS_TOKEN_OUTPUT=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url")
240-
if [ -n "$ACCESS_TOKEN_OUTPUT" ]; then
239+
response=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url" -w "\n%{http_code}")
240+
ACCESS_TOKEN_OUTPUT=$(echo "$response" | sed '$d')
241+
http_code=$(echo "$response" | tail -n1)
242+
if [ -n "$ACCESS_TOKEN_OUTPUT" ] && [ "$http_code" -eq 200 ]; then
241243
echo "$ACCESS_TOKEN_OUTPUT"
242244
return 0
243245
fi
244246
sleep $wait_sleep
245247
done
246-
return $ERR_ORAS_PULL_NETWORK_TIMEOUT
248+
if [ -n "$http_code" ]; then
249+
echo "failed to retrieve kubelet identity token from IMDS, http code: $http_code, msg: $ACCESS_TOKEN_OUTPUT"
250+
return $ERR_ORAS_PULL_UNAUTHORIZED
251+
fi
252+
echo "timeout waiting for IMDS response to retrieve kubelet identity token"
253+
return $ERR_ORAS_IMDS_TIMEOUT
247254
}
248255
retrycmd_get_refresh_token_for_oras() {
249256
retries=$1; wait_sleep=$2; acr_url=$3; tenant_id=$4; ACCESS_TOKEN=$5
@@ -762,16 +769,12 @@ oras_login_with_kubelet_identity() {
762769

763770
set +x
764771
access_url="http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/&client_id=$client_id"
765-
raw_access_token=$(retrycmd_get_access_token_for_oras 10 5 $access_url)
772+
raw_access_token=$(retrycmd_get_access_token_for_oras 5 15 $access_url)
766773
ret_code=$?
767774
if [ $ret_code -ne 0 ]; then
768-
echo "failed to retrieve access token: $ret_code"
775+
echo $raw_access_token
769776
return $ret_code
770777
fi
771-
if [[ "$raw_access_token" == *"error"* ]]; then
772-
echo "failed to retrieve access token"
773-
return $ERR_ORAS_PULL_UNAUTHORIZED
774-
fi
775778
ACCESS_TOKEN=$(echo "$raw_access_token" | jq -r .access_token)
776779
if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" == "null" ]; then
777780
echo "failed to parse access token"

‎pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line9.sh‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ ERR_CNI_VERSION_INVALID=206
109109
ERR_ORAS_PULL_K8S_FAIL=207
110110
ERR_ORAS_PULL_CREDENTIAL_PROVIDER=208
111111
ERR_ORAS_PULL_CONTAINERD_WASM=209
112-
ERR_ORAS_PULL_FAIL_RESERVE_2=210
112+
ERR_ORAS_IMDS_TIMEOUT=210
113113
ERR_ORAS_PULL_NETWORK_TIMEOUT=211
114114
ERR_ORAS_PULL_UNAUTHORIZED=212
115115

@@ -236,14 +236,21 @@ retrycmd_get_tarball_from_registry_with_oras() {
236236
retrycmd_get_access_token_for_oras() {
237237
retries=$1; wait_sleep=$2; url=$3
238238
for i in $(seq 1 $retries); do
239-
ACCESS_TOKEN_OUTPUT=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url")
240-
if [ -n "$ACCESS_TOKEN_OUTPUT" ]; then
239+
response=$(timeout 60 curl -v -s -H "Metadata:true" --noproxy "*" "$url" -w "\n%{http_code}")
240+
ACCESS_TOKEN_OUTPUT=$(echo "$response" | sed '$d')
241+
http_code=$(echo "$response" | tail -n1)
242+
if [ -n "$ACCESS_TOKEN_OUTPUT" ] && [ "$http_code" -eq 200 ]; then
241243
echo "$ACCESS_TOKEN_OUTPUT"
242244
return 0
243245
fi
244246
sleep $wait_sleep
245247
done
246-
return $ERR_ORAS_PULL_NETWORK_TIMEOUT
248+
if [ -n "$http_code" ]; then
249+
echo "failed to retrieve kubelet identity token from IMDS, http code: $http_code, msg: $ACCESS_TOKEN_OUTPUT"
250+
return $ERR_ORAS_PULL_UNAUTHORIZED
251+
fi
252+
echo "timeout waiting for IMDS response to retrieve kubelet identity token"
253+
return $ERR_ORAS_IMDS_TIMEOUT
247254
}
248255
retrycmd_get_refresh_token_for_oras() {
249256
retries=$1; wait_sleep=$2; acr_url=$3; tenant_id=$4; ACCESS_TOKEN=$5
@@ -762,16 +769,12 @@ oras_login_with_kubelet_identity() {
762769

763770
set +x
764771
access_url="http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/&client_id=$client_id"
765-
raw_access_token=$(retrycmd_get_access_token_for_oras 10 5 $access_url)
772+
raw_access_token=$(retrycmd_get_access_token_for_oras 5 15 $access_url)
766773
ret_code=$?
767774
if [ $ret_code -ne 0 ]; then
768-
echo "failed to retrieve access token: $ret_code"
775+
echo $raw_access_token
769776
return $ret_code
770777
fi
771-
if [[ "$raw_access_token" == *"error"* ]]; then
772-
echo "failed to retrieve access token"
773-
return $ERR_ORAS_PULL_UNAUTHORIZED
774-
fi
775778
ACCESS_TOKEN=$(echo "$raw_access_token" | jq -r .access_token)
776779
if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" == "null" ]; then
777780
echo "failed to parse access token"

0 commit comments

Comments
 (0)