Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: fix scenario that kube binary version is different from k8s version for network isolated cluster #5335

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

fix: fix scenario that kube binary version is different from k8s version for network isolated cluster #5335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4c51a8d
[NI] fix scenario that kube binary version is different from k8s version
fseldow Jan 20, 2025
4ee3e0e
ut
fseldow Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions e2e/scenario_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,6 @@ func Test_AzureLinuxV2_ARM64AirGap(t *testing.T) {
Cluster: ClusterKubenetAirgap,
VHD: config.VHDAzureLinuxV2Gen2Arm64,
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.CustomKubeBinaryURL = "https://acs-mirror.azureedge.net/kubernetes/v1.24.9/binaries/kubernetes-node-linux-arm64.tar.gz"
nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5"
nbc.IsARM64 = true

Expand Down Expand Up @@ -364,7 +363,6 @@ func Test_MarinerV2_ARM64AirGap(t *testing.T) {
Cluster: ClusterKubenetAirgap,
VHD: config.VHDCBLMarinerV2Gen2Arm64,
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.CustomKubeBinaryURL = "https://acs-mirror.azureedge.net/kubernetes/v1.24.9/binaries/kubernetes-node-linux-arm64.tar.gz"
nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5"
nbc.IsARM64 = true

Expand Down
23 changes: 23 additions & 0 deletions parts/linux/cloud-init/artifacts/cse_helpers.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,29 @@ removeKubeletNodeLabel() {
fi
}

# generate kubenode binary registry url from acs-mirror url
updateKubeBinaryRegistryURL() {
if [[ -n "${KUBE_BINARY_URL}" ]] && isRegistryUrl "${KUBE_BINARY_URL}"; then
# will not catch this path for now
echo "KUBE_BINARY_URL is a registry url, will use it to pull the kube binary"
KUBE_BINARY_REGISTRY_URL="${KUBE_BINARY_URL}"
else
fseldow marked this conversation as resolved.
Show resolved Hide resolved
url_regex='https://[^/]+/kubernetes/v[0-9]+\.[0-9]+\..+/binaries/.+'
binary_version="v${KUBERNETES_VERSION}" # by default use Kubernetes versions
if [[ -n ${KUBE_BINARY_URL} ]]; then
if [[ ${KUBE_BINARY_URL} =~ $url_regex ]]; then
version_with_prefix="${KUBE_BINARY_URL#*kubernetes/}"
# Extract the version part
binary_version="${version_with_prefix%%/*}"
echo "Extracted version: $binary_version from KUBE_BINARY_URL: ${KUBE_BINARY_URL}"
else
echo "KUBE_BINARY_URL is formatted unexpectedly, will use the kubernetes version as binary version: $binary_version"
fi
fi
KUBE_BINARY_REGISTRY_URL="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:${binary_version}-linux-${CPU_ARCH}"
fi
}

# removes the specified FLAG_STRING (which should be in the form of 'key=value') from KUBELET_FLAGS
removeKubeletFlag() {
local FLAG_STRING=$1
Expand Down
5 changes: 3 additions & 2 deletions parts/linux/cloud-init/artifacts/cse_install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -614,9 +614,10 @@ installKubeletKubectlAndKubeProxy() {
if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
# network isolated cluster
echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
registry_url="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:v${KUBERNETES_VERSION}-linux-${CPU_ARCH}"
updateKubeBinaryRegistryURL

K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads" # /opt folder will return permission error
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} $registry_url false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
# no egress traffic, default install will fail
# will exit if the download fails

Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData
View file
Open in desktop

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+Containerd/line40.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,9 +561,10 @@ installKubeletKubectlAndKubeProxy() {
if [[ "$install_default_if_missing" == true ]]; then
if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
registry_url="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:v${KUBERNETES_VERSION}-linux-${CPU_ARCH}"
updateKubeBinaryRegistryURL

K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads"
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} $registry_url false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}

#TODO: remove the condition check on KUBE_BINARY_URL once RP change is released
elif (($(echo ${KUBERNETES_VERSION} | cut -d"." -f2) >= 17)) && [ -n "${KUBE_BINARY_URL}" ]; then
Expand Down
20 changes: 20 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+Containerd/line9.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,26 @@ removeKubeletNodeLabel() {
fi
}

updateKubeBinaryRegistryURL() {
if [[ -n "${KUBE_BINARY_URL}" ]] && isRegistryUrl "${KUBE_BINARY_URL}"; then
echo "KUBE_BINARY_URL is a registry url, will use it to pull the kube binary"
KUBE_BINARY_REGISTRY_URL="${KUBE_BINARY_URL}"
else
url_regex='https://[^/]+/kubernetes/v[0-9]+\.[0-9]+\..+/binaries/.+'
binary_version="v${KUBERNETES_VERSION}"
if [[ -n ${KUBE_BINARY_URL} ]]; then
if [[ ${KUBE_BINARY_URL} =~ $url_regex ]]; then
version_with_prefix="${KUBE_BINARY_URL#*kubernetes/}"
binary_version="${version_with_prefix%%/*}"
echo "Extracted version: $binary_version from KUBE_BINARY_URL: ${KUBE_BINARY_URL}"
else
echo "KUBE_BINARY_URL is formatted unexpectedly, will use the kubernetes version as binary version: $binary_version"
fi
fi
KUBE_BINARY_REGISTRY_URL="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:${binary_version}-linux-${CPU_ARCH}"
fi
}

removeKubeletFlag() {
local FLAG_STRING=$1
if grep -e ",${FLAG_STRING}" <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData
View file
Open in desktop

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line40.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,9 +561,10 @@ installKubeletKubectlAndKubeProxy() {
if [[ "$install_default_if_missing" == true ]]; then
if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
registry_url="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:v${KUBERNETES_VERSION}-linux-${CPU_ARCH}"
updateKubeBinaryRegistryURL

K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads"
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} $registry_url false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}

#TODO: remove the condition check on KUBE_BINARY_URL once RP change is released
elif (($(echo ${KUBERNETES_VERSION} | cut -d"." -f2) >= 17)) && [ -n "${KUBE_BINARY_URL}" ]; then
Expand Down
20 changes: 20 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line9.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,26 @@ removeKubeletNodeLabel() {
fi
}

updateKubeBinaryRegistryURL() {
if [[ -n "${KUBE_BINARY_URL}" ]] && isRegistryUrl "${KUBE_BINARY_URL}"; then
echo "KUBE_BINARY_URL is a registry url, will use it to pull the kube binary"
KUBE_BINARY_REGISTRY_URL="${KUBE_BINARY_URL}"
else
url_regex='https://[^/]+/kubernetes/v[0-9]+\.[0-9]+\..+/binaries/.+'
binary_version="v${KUBERNETES_VERSION}"
if [[ -n ${KUBE_BINARY_URL} ]]; then
if [[ ${KUBE_BINARY_URL} =~ $url_regex ]]; then
version_with_prefix="${KUBE_BINARY_URL#*kubernetes/}"
binary_version="${version_with_prefix%%/*}"
echo "Extracted version: $binary_version from KUBE_BINARY_URL: ${KUBE_BINARY_URL}"
else
echo "KUBE_BINARY_URL is formatted unexpectedly, will use the kubernetes version as binary version: $binary_version"
fi
fi
KUBE_BINARY_REGISTRY_URL="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:${binary_version}-linux-${CPU_ARCH}"
fi
}

removeKubeletFlag() {
local FLAG_STRING=$1
if grep -e ",${FLAG_STRING}" <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData
View file
Open in desktop

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line40.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,9 +561,10 @@ installKubeletKubectlAndKubeProxy() {
if [[ "$install_default_if_missing" == true ]]; then
if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
registry_url="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:v${KUBERNETES_VERSION}-linux-${CPU_ARCH}"
updateKubeBinaryRegistryURL

K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads"
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} $registry_url false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}

#TODO: remove the condition check on KUBE_BINARY_URL once RP change is released
elif (($(echo ${KUBERNETES_VERSION} | cut -d"." -f2) >= 17)) && [ -n "${KUBE_BINARY_URL}" ]; then
Expand Down
20 changes: 20 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line9.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,26 @@ removeKubeletNodeLabel() {
fi
}

updateKubeBinaryRegistryURL() {
if [[ -n "${KUBE_BINARY_URL}" ]] && isRegistryUrl "${KUBE_BINARY_URL}"; then
echo "KUBE_BINARY_URL is a registry url, will use it to pull the kube binary"
KUBE_BINARY_REGISTRY_URL="${KUBE_BINARY_URL}"
else
url_regex='https://[^/]+/kubernetes/v[0-9]+\.[0-9]+\..+/binaries/.+'
binary_version="v${KUBERNETES_VERSION}"
if [[ -n ${KUBE_BINARY_URL} ]]; then
if [[ ${KUBE_BINARY_URL} =~ $url_regex ]]; then
version_with_prefix="${KUBE_BINARY_URL#*kubernetes/}"
binary_version="${version_with_prefix%%/*}"
echo "Extracted version: $binary_version from KUBE_BINARY_URL: ${KUBE_BINARY_URL}"
else
echo "KUBE_BINARY_URL is formatted unexpectedly, will use the kubernetes version as binary version: $binary_version"
fi
fi
KUBE_BINARY_REGISTRY_URL="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:${binary_version}-linux-${CPU_ARCH}"
fi
}

removeKubeletFlag() {
local FLAG_STRING=$1
if grep -e ",${FLAG_STRING}" <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData
View file
Open in desktop

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line40.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,9 +561,10 @@ installKubeletKubectlAndKubeProxy() {
if [[ "$install_default_if_missing" == true ]]; then
if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
registry_url="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:v${KUBERNETES_VERSION}-linux-${CPU_ARCH}"
updateKubeBinaryRegistryURL

K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads"
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} $registry_url false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}

#TODO: remove the condition check on KUBE_BINARY_URL once RP change is released
elif (($(echo ${KUBERNETES_VERSION} | cut -d"." -f2) >= 17)) && [ -n "${KUBE_BINARY_URL}" ]; then
Expand Down
20 changes: 20 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line9.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,26 @@ removeKubeletNodeLabel() {
fi
}

updateKubeBinaryRegistryURL() {
if [[ -n "${KUBE_BINARY_URL}" ]] && isRegistryUrl "${KUBE_BINARY_URL}"; then
echo "KUBE_BINARY_URL is a registry url, will use it to pull the kube binary"
KUBE_BINARY_REGISTRY_URL="${KUBE_BINARY_URL}"
else
url_regex='https://[^/]+/kubernetes/v[0-9]+\.[0-9]+\..+/binaries/.+'
binary_version="v${KUBERNETES_VERSION}"
if [[ -n ${KUBE_BINARY_URL} ]]; then
if [[ ${KUBE_BINARY_URL} =~ $url_regex ]]; then
version_with_prefix="${KUBE_BINARY_URL#*kubernetes/}"
binary_version="${version_with_prefix%%/*}"
echo "Extracted version: $binary_version from KUBE_BINARY_URL: ${KUBE_BINARY_URL}"
else
echo "KUBE_BINARY_URL is formatted unexpectedly, will use the kubernetes version as binary version: $binary_version"
fi
fi
KUBE_BINARY_REGISTRY_URL="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:${binary_version}-linux-${CPU_ARCH}"
fi
}

removeKubeletFlag() {
local FLAG_STRING=$1
if grep -e ",${FLAG_STRING}" <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
Expand Down
Loading
Loading