Skip to content

Commit d3c8260

Browse files
abhinavnagarajabhinavnagaraj
committed
PE-7463 handle etcd health check failure during kube-join
1 parent 3541733 commit d3c8260

1 file changed

Lines changed: 162 additions & 18 deletions

File tree

‎scripts/kube-join.sh‎

Lines changed: 162 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ export PATH="$PATH:$root_path/usr/local/bin"
2020

2121
KUBE_VIP_LOC="/etc/kubernetes/manifests/kube-vip.yaml"
2222

23+
# Prevents infinite retry loops (original used until loop)
24+
MAX_RETRIES=3
25+
# Time to wait for etcd/static pods to become ready
26+
ETCD_HEALTH_TIMEOUT=60
27+
RETRY_DELAY=30
28+
2329
restart_containerd() {
2430
if systemctl cat spectro-containerd >/dev/null 2<&1; then
2531
systemctl restart spectro-containerd
@@ -30,6 +36,100 @@ restart_containerd() {
3036
fi
3137
}
3238

39+
# Function to wait for static pods to be created and running
40+
wait_for_static_pods() {
41+
local timeout=$1
42+
local start_time=$(date +%s)
43+
44+
echo "Waiting for static pods to be created and running..."
45+
46+
# Check if crictl is available
47+
if ! command -v crictl >/dev/null 2>&1; then
48+
echo "crictl not available, checking only static pod manifests"
49+
# Fallback: just check if manifests exist and wait
50+
while [ $(($(date +%s) - start_time)) -lt $timeout ]; do
51+
if [ -f /etc/kubernetes/manifests/etcd.yaml ] && [ -f /etc/kubernetes/manifests/kube-apiserver.yaml ] && [ -f /etc/kubernetes/manifests/kube-controller-manager.yaml ] && [ -f /etc/kubernetes/manifests/kube-scheduler.yaml ]; then
52+
echo "Static pod manifests are ready, waiting additional time for pods to stabilize..."
53+
sleep 15
54+
return 0
55+
fi
56+
sleep 5
57+
done
58+
echo "Timeout waiting for static pod manifests after ${timeout}s"
59+
return 1
60+
fi
61+
62+
while [ $(($(date +%s) - start_time)) -lt $timeout ]; do
63+
# Check if all required static pod manifests exist
64+
if [ -f /etc/kubernetes/manifests/etcd.yaml ] && [ -f /etc/kubernetes/manifests/kube-apiserver.yaml ] && [ -f /etc/kubernetes/manifests/kube-controller-manager.yaml ] && [ -f /etc/kubernetes/manifests/kube-scheduler.yaml ]; then
65+
# Check if etcd container is running
66+
local etcd_container_id=$(crictl ps --name etcd --quiet 2>/dev/null)
67+
if [ -n "$etcd_container_id" ] && crictl ps --id "$etcd_container_id" --format table | grep -q "Running"; then
68+
echo "Static pods are running, waiting additional time for etcd to stabilize..."
69+
sleep 10
70+
return 0
71+
fi
72+
fi
73+
sleep 5
74+
done
75+
76+
echo "Timeout waiting for static pods to be ready after ${timeout}s"
77+
return 1
78+
}
79+
80+
# Function to check if etcd is healthy (only if etcdctl and crictl are available)
81+
check_etcd_health_if_available() {
82+
local timeout=$1
83+
local start_time=$(date +%s)
84+
85+
# Check if etcdctl is available
86+
if ! command -v etcdctl >/dev/null 2>&1; then
87+
echo "etcdctl not available, skipping etcd health check"
88+
return 0
89+
fi
90+
91+
# Check if crictl is available
92+
if ! command -v crictl >/dev/null 2>&1; then
93+
echo "crictl not available, skipping etcd health check"
94+
return 0
95+
fi
96+
97+
echo "Checking etcd health with etcdctl..."
98+
99+
while [ $(($(date +%s) - start_time)) -lt $timeout ]; do
100+
# Check if etcd container is running
101+
local etcd_container_id=$(crictl ps --name etcd --quiet 2>/dev/null)
102+
if [ -n "$etcd_container_id" ] && crictl ps --id "$etcd_container_id" --format table | grep -q "Running"; then
103+
# Try to connect to local etcd directly
104+
if timeout 10s etcdctl endpoint health --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key >/dev/null 2>&1; then
105+
echo "etcd is healthy"
106+
return 0
107+
fi
108+
fi
109+
sleep 5
110+
done
111+
112+
echo "etcd health check timed out after ${timeout}s"
113+
return 1
114+
}
115+
116+
# Function to clean up cluster state before retry
117+
cleanup_cluster_state() {
118+
local node_name=$(hostname)
119+
echo "Cleaning up cluster state for node: $node_name"
120+
121+
# Try to remove node from cluster if possible
122+
if [ -f /etc/kubernetes/admin.conf ]; then
123+
kubectl delete node $node_name --kubeconfig=/etc/kubernetes/admin.conf --ignore-not-found=true 2>/dev/null || true
124+
fi
125+
126+
# For control plane nodes, try to remove etcd member
127+
if [ "$NODE_ROLE" = "controlplane" ]; then
128+
# This would need to be done from another control plane node
129+
echo "Note: etcd member cleanup should be done from another control plane node"
130+
fi
131+
}
132+
33133
do_kubeadm_reset() {
34134
if [ -S /run/spectro/containerd/containerd.sock ]; then
35135
kubeadm reset -f --cri-socket unix:///run/spectro/containerd/containerd.sock --cleanup-tmp-dir
@@ -59,24 +159,68 @@ restore_kube_vip_manifest_after_reset() {
59159
fi
60160
}
61161

62-
if [ "$PROXY_CONFIGURED" = true ]; then
63-
until HTTP_PROXY=$proxy_http http_proxy=$proxy_http HTTPS_PROXY=$proxy_https https_proxy=$proxy_https NO_PROXY=$proxy_no no_proxy=$proxy_no kubeadm join --config "$root_path"/opt/kubeadm/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
64-
do
162+
# Function to attempt kubeadm join with proper error handling
163+
attempt_kubeadm_join() {
164+
local attempt=$1
165+
local max_retries=$2
166+
167+
echo "Attempting kubeadm join (attempt $attempt/$max_retries)"
168+
169+
if [ "$PROXY_CONFIGURED" = true ]; then
170+
HTTP_PROXY=$proxy_http http_proxy=$proxy_http HTTPS_PROXY=$proxy_https https_proxy=$proxy_https NO_PROXY=$proxy_no no_proxy=$proxy_no kubeadm join --config "$root_path"/opt/kubeadm/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5
171+
else
172+
kubeadm join --config "$root_path"/opt/kubeadm/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5
173+
fi
174+
}
175+
176+
# Main join logic with intelligent retry
177+
retry_count=0
178+
while [ $retry_count -lt $MAX_RETRIES ]; do
179+
retry_count=$((retry_count + 1))
180+
181+
if attempt_kubeadm_join $retry_count $MAX_RETRIES; then
182+
echo "kubeadm join completed successfully"
183+
break
184+
else
185+
echo "kubeadm join failed on attempt $retry_count"
186+
187+
# Check if this is an etcd health check failure
188+
if grep -q "etcd cluster is not healthy" /var/log/kube-join.log 2>/dev/null; then
189+
echo "Detected etcd health check failure"
190+
191+
# For first attempt, try to wait for static pods to be ready
192+
if [ $retry_count -eq 1 ]; then
193+
echo "Waiting for static pods to be ready before retry..."
194+
if wait_for_static_pods $ETCD_HEALTH_TIMEOUT; then
195+
echo "Static pods are ready, checking etcd health if available..."
196+
if check_etcd_health_if_available 30; then
197+
echo "etcd is healthy, retrying join without reset"
198+
continue
199+
else
200+
echo "etcd health check failed or etcdctl not available, retrying join without reset"
201+
continue
202+
fi
203+
else
204+
echo "Static pods not ready, proceeding with reset"
205+
fi
206+
fi
207+
fi
208+
209+
# If we've reached max retries, exit with error
210+
if [ $retry_count -eq $MAX_RETRIES ]; then
211+
echo "Maximum retry attempts ($MAX_RETRIES) reached. Exiting."
212+
exit 1
213+
fi
214+
215+
# Clean up cluster state before reset
216+
cleanup_cluster_state
217+
218+
# Reset and retry
65219
backup_kube_vip_manifest_if_present
66-
echo "failed to apply kubeadm join, will retry in 10s";
220+
echo "Resetting kubeadm and retrying in ${RETRY_DELAY}s"
67221
do_kubeadm_reset
68-
echo "retrying in 10s"
69-
sleep 10;
222+
echo "Retrying in ${RETRY_DELAY}s"
223+
sleep $RETRY_DELAY
70224
restore_kube_vip_manifest_after_reset
71-
done;
72-
else
73-
until kubeadm join --config "$root_path"/opt/kubeadm/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
74-
do
75-
backup_kube_vip_manifest_if_present
76-
echo "failed to apply kubeadm join, will retry in 10s";
77-
do_kubeadm_reset
78-
echo "retrying in 10s"
79-
sleep 10;
80-
restore_kube_vip_manifest_after_reset
81-
done;
82-
fi
225+
fi
226+
done

0 commit comments

Comments
 (0)