Skip to content

Commit c73f866

Browse files
author
tazhate
committed
fix(ci): classify per-chain outcome instead of binary timeout/pass
Previous batch script only counted PASS when .status.height >= 1 and lumped everything else into "timeout: no-status", which hid OOM kills, ImagePullBackOff, CrashLoopBackOff, PVC binding hangs, and admission rejections under the same useless reason. Reworked classify(): PASS_SYNC .status.height >= 1 PASS_SOFT pod Ready and controller wrote .status.phase PASS_STARTED pod Ready >= POD_READY_GRACE (60s) with no restarts FAIL_OOM any container OOMKilled (lastState.terminated.reason) FAIL_CRASH CrashLoopBackOff / RunContainerError / config errors FAIL_IMAGE ImagePullBackOff / ErrImagePull / InvalidImageName FAIL_PENDING pod stuck Pending (PVC unbound / unschedulable) FAIL_APPLY kubectl apply rejected (CRD enum / admission webhook) FAIL_NO_STATUS pod Ready but controller never wrote status apply errors now get attributed to the right chain by parsing kubectl output, so a single bad sample doesn't shadow the rest. Per-FAIL artifacts also dump previous-instance logs so OOM/crash root causes are visible without re-running. Summary in $GITHUB_STEP_SUMMARY now shows the breakdown table plus a collapsible per-chain section. Context: third sweep was about to repeat the "everything failed with no useful info" experience because the old success criterion required height>0 even though many adapters need many minutes (or a healthy peer set) before the controller writes height. Switched to a layered criterion that catches "node started but sync hasn't reached height write yet" as a soft pass, and split actual failure modes apart so the summary tells you what to investigate. ~25min.
1 parent cdced7b commit c73f866

1 file changed

Lines changed: 189 additions & 43 deletions

File tree

‎scripts/manual-batch-test.sh‎

Lines changed: 189 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
#!/usr/bin/env bash
2-
# Manual batch e2e: applies all ChainInstance samples in batches, verifies
3-
# pod start + .status.height > 0, then deletes the batch before the next.
2+
# Manual batch e2e: applies all ChainInstance samples in batches and
3+
# classifies the outcome of each (sync started / pod started / OOM / crash /
4+
# image-pull / pending / no-status).
45
#
56
# Env:
67
# BATCH_SIZE (default 5)
78
# READY_TIMEOUT (default 600 sec per batch)
8-
# STORAGE_CLASS (default "standard" — kind default)
9+
# STORAGE_CLASS (default "standard")
910
# STORAGE_SIZE (default 5Gi)
1011
# NAMESPACE (default chainplane-batch)
11-
# CHAINS_FILTER (regex, empty = all)
12+
# CHAINS_FILTER (regex on filename, empty = all)
1213
# ARTIFACTS_DIR (default ./artifacts)
1314
# SAMPLES_DIR (default config/samples)
15+
# POD_READY_GRACE (default 60 — sec a pod must stay Ready without crashes
16+
# to count as PASS_STARTED)
1417

1518
set -uo pipefail
1619

@@ -22,6 +25,7 @@ NAMESPACE="${NAMESPACE:-chainplane-batch}"
2225
CHAINS_FILTER="${CHAINS_FILTER:-}"
2326
ARTIFACTS_DIR="${ARTIFACTS_DIR:-artifacts}"
2427
SAMPLES_DIR="${SAMPLES_DIR:-config/samples}"
28+
POD_READY_GRACE="${POD_READY_GRACE:-60}"
2529

2630
mkdir -p "${ARTIFACTS_DIR}"
2731

@@ -31,7 +35,6 @@ done
3135

3236
kubectl create namespace "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f -
3337

34-
# Build sample list (skip the placeholder aggregate file).
3538
mapfile -t ALL_SAMPLES < <(
3639
find "${SAMPLES_DIR}" -maxdepth 1 -name 'chains_v1alpha2_chaininstance_*.yaml' \
3740
! -name 'chains_v1alpha2_chaininstance.yaml' | sort
@@ -51,10 +54,21 @@ if [[ ${TOTAL} -eq 0 ]]; then
5154
exit 1
5255
fi
5356

54-
echo "Discovered ${TOTAL} sample(s); batch size=${BATCH_SIZE}; per-batch timeout=${READY_TIMEOUT}s"
57+
echo "Discovered ${TOTAL} sample(s); batch=${BATCH_SIZE}; timeout=${READY_TIMEOUT}s; ready-grace=${POD_READY_GRACE}s"
5558

5659
declare -a RESULTS_NAME RESULTS_STATUS RESULTS_REASON
5760

61+
# Status taxonomy:
62+
# PASS_SYNC .status.height >= 1
63+
# PASS_SOFT pod Ready + .status.phase set (controller saw the node)
64+
# PASS_STARTED pod Ready >= POD_READY_GRACE without container restarts
65+
# FAIL_OOM any container OOMKilled
66+
# FAIL_CRASH CrashLoopBackOff / RunContainerError / non-OOM termination
67+
# FAIL_IMAGE ImagePullBackOff / ErrImagePull
68+
# FAIL_PENDING PVC unbound / unschedulable / pod stuck Pending
69+
# FAIL_APPLY kubectl apply rejected the manifest (CRD/admission)
70+
# FAIL_NO_STATUS pod Ready but controller never wrote .status
71+
5872
patch_sample() {
5973
local in="$1" out="$2"
6074
yq eval "
@@ -67,20 +81,96 @@ patch_sample() {
6781
" "${in}" > "${out}"
6882
}
6983

84+
# Echoes one of: PASS_SYNC|PASS_SOFT|PASS_STARTED|FAIL_OOM|FAIL_CRASH|
85+
# FAIL_IMAGE|FAIL_PENDING|FAIL_NO_STATUS|PROGRESS plus a reason string,
86+
# separated by tab. PROGRESS = not done yet, keep waiting.
87+
classify() {
88+
local nm="$1" first_ready_ts="$2"
89+
local height phase pod_json
90+
height="$(kubectl -n "${NAMESPACE}" get chaininstance "${nm}" -o jsonpath='{.status.height}' 2>/dev/null || true)"
91+
phase="$(kubectl -n "${NAMESPACE}" get chaininstance "${nm}" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
92+
93+
if [[ -n "${height}" && "${height}" =~ ^[1-9][0-9]*$ ]]; then
94+
printf 'PASS_SYNC\theight=%s phase=%s\n' "${height}" "${phase:-?}"
95+
return
96+
fi
97+
98+
pod_json="$(kubectl -n "${NAMESPACE}" get pods -l "chains.chainplane.io/instance=${nm}" -o json 2>/dev/null || echo '{"items":[]}')"
99+
local n_pods
100+
n_pods=$(jq '.items | length' <<<"${pod_json}")
101+
102+
if [[ "${n_pods}" -eq 0 ]]; then
103+
printf 'PROGRESS\tno-pod-yet phase=%s\n' "${phase:-?}"
104+
return
105+
fi
106+
107+
# Look at all containers across all pods for fault states.
108+
local oom crash image waiting_reason terminated_reason restart_count ready phase_pod
109+
oom=$(jq -r '[.items[].status.containerStatuses[]?.lastState.terminated // empty | select(.reason=="OOMKilled")] | length' <<<"${pod_json}")
110+
waiting_reason=$(jq -r '[.items[].status.containerStatuses[]?.state.waiting.reason // empty] | join(",")' <<<"${pod_json}")
111+
terminated_reason=$(jq -r '[.items[].status.containerStatuses[]?.lastState.terminated.reason // empty] | join(",")' <<<"${pod_json}")
112+
restart_count=$(jq -r '[.items[].status.containerStatuses[]?.restartCount // 0] | add // 0' <<<"${pod_json}")
113+
ready=$(jq -r '[.items[].status.conditions[]? | select(.type=="Ready") | .status] | join(",")' <<<"${pod_json}")
114+
phase_pod=$(jq -r '[.items[].status.phase] | join(",")' <<<"${pod_json}")
115+
116+
if [[ "${oom}" -gt 0 ]]; then
117+
printf 'FAIL_OOM\trestarts=%s pod-phase=%s\n' "${restart_count}" "${phase_pod}"
118+
return
119+
fi
120+
if [[ "${waiting_reason}" =~ ImagePullBackOff|ErrImagePull|InvalidImageName ]]; then
121+
printf 'FAIL_IMAGE\twaiting=%s\n' "${waiting_reason}"
122+
return
123+
fi
124+
if [[ "${waiting_reason}" =~ CrashLoopBackOff|RunContainerError|CreateContainerConfigError ]]; then
125+
printf 'FAIL_CRASH\twaiting=%s last=%s restarts=%s\n' "${waiting_reason}" "${terminated_reason}" "${restart_count}"
126+
return
127+
fi
128+
if [[ "${phase_pod}" == "Pending" || "${phase_pod}" =~ Pending ]]; then
129+
printf 'PROGRESS\tpod-pending\n'
130+
return
131+
fi
132+
133+
# Pod Ready?
134+
if [[ "${ready}" =~ True ]]; then
135+
if [[ "${first_ready_ts}" == "0" ]]; then
136+
# Caller will record the first-ready timestamp for grace tracking.
137+
printf 'JUST_READY\tpod-ready phase=%s\n' "${phase:-?}"
138+
return
139+
fi
140+
local now elapsed
141+
now=$(date +%s)
142+
elapsed=$(( now - first_ready_ts ))
143+
if [[ -n "${phase}" && "${phase}" != "Pending" ]]; then
144+
printf 'PASS_SOFT\tphase=%s ready-for=%ss\n' "${phase}" "${elapsed}"
145+
return
146+
fi
147+
if [[ ${elapsed} -ge ${POD_READY_GRACE} && ${restart_count} -eq 0 ]]; then
148+
printf 'PASS_STARTED\tready-for=%ss restarts=0\n' "${elapsed}"
149+
return
150+
fi
151+
printf 'PROGRESS\tready-but-no-status ready-for=%ss\n' "${elapsed}"
152+
return
153+
fi
154+
155+
printf 'PROGRESS\tnot-ready phase=%s waiting=%s\n' "${phase_pod}" "${waiting_reason}"
156+
}
157+
70158
dump_artifacts() {
71159
local name="$1"
72160
local logf="${ARTIFACTS_DIR}/${name}.log"
73161
{
74162
echo "=== describe chaininstance/${name} ==="
75163
kubectl -n "${NAMESPACE}" describe chaininstance "${name}" 2>&1 || true
76164
echo
77-
echo "=== describe sts ==="
78-
kubectl -n "${NAMESPACE}" describe sts -l "chains.chainplane.io/instance=${name}" 2>&1 || true
165+
echo "=== describe pods ==="
166+
kubectl -n "${NAMESPACE}" describe pods -l "chains.chainplane.io/instance=${name}" 2>&1 || true
79167
echo
80-
echo "=== pod logs ==="
168+
echo "=== pod logs (current + previous) ==="
81169
for p in $(kubectl -n "${NAMESPACE}" get pods -l "chains.chainplane.io/instance=${name}" -o name 2>/dev/null); do
82-
echo "--- ${p} ---"
170+
echo "--- ${p} current ---"
83171
kubectl -n "${NAMESPACE}" logs "${p}" --all-containers --tail=200 2>&1 || true
172+
echo "--- ${p} previous ---"
173+
kubectl -n "${NAMESPACE}" logs "${p}" --all-containers --previous --tail=100 2>&1 || true
84174
done
85175
echo
86176
echo "=== events ==="
@@ -93,6 +183,7 @@ run_batch() {
93183
local tmpdir
94184
tmpdir="$(mktemp -d)"
95185
local -a names=()
186+
local -A apply_failed=()
96187

97188
for f in "${files[@]}"; do
98189
local nm
@@ -102,47 +193,82 @@ run_batch() {
102193
done
103194

104195
echo "--- applying batch: ${names[*]}"
105-
kubectl apply -f "${tmpdir}/" || true
196+
local apply_log="${tmpdir}/apply.log"
197+
if ! kubectl apply -f "${tmpdir}/" >"${apply_log}" 2>&1; then
198+
cat "${apply_log}"
199+
# Mark per-file apply failures based on stderr text.
200+
while IFS= read -r line; do
201+
for nm in "${names[@]}"; do
202+
if [[ "${line}" =~ \"${nm}\" || "${line}" =~ /${nm}\.yaml ]]; then
203+
apply_failed[$nm]="${line}"
204+
fi
205+
done
206+
done < "${apply_log}"
207+
else
208+
cat "${apply_log}"
209+
fi
106210

107211
local deadline=$(( $(date +%s) + READY_TIMEOUT ))
108-
local -A done_map=()
212+
local -A done_status=()
213+
local -A done_reason=()
214+
local -A first_ready=()
215+
216+
# Pre-fail the ones that didn't even apply.
217+
for nm in "${names[@]}"; do
218+
if [[ -n "${apply_failed[$nm]:-}" ]]; then
219+
done_status[$nm]="FAIL_APPLY"
220+
done_reason[$nm]="${apply_failed[$nm]}"
221+
echo " FAIL_APPLY ${nm}"
222+
fi
223+
done
109224

110225
while [[ $(date +%s) -lt ${deadline} ]]; do
111226
local all_done=1
112227
for nm in "${names[@]}"; do
113-
[[ -n "${done_map[$nm]:-}" ]] && continue
114-
local height phase
115-
height="$(kubectl -n "${NAMESPACE}" get chaininstance "${nm}" -o jsonpath='{.status.height}' 2>/dev/null || true)"
116-
phase="$(kubectl -n "${NAMESPACE}" get chaininstance "${nm}" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
117-
if [[ -n "${height}" && "${height}" =~ ^[1-9][0-9]*$ ]]; then
118-
done_map[$nm]="PASS:height=${height},phase=${phase}"
119-
echo " PASS ${nm} height=${height} phase=${phase}"
120-
else
121-
all_done=0
122-
fi
228+
[[ -n "${done_status[$nm]:-}" ]] && continue
229+
local out status reason
230+
out="$(classify "${nm}" "${first_ready[$nm]:-0}")"
231+
status="${out%%$'\t'*}"
232+
reason="${out#*$'\t'}"
233+
case "${status}" in
234+
JUST_READY)
235+
first_ready[$nm]=$(date +%s)
236+
all_done=0
237+
;;
238+
PROGRESS)
239+
all_done=0
240+
;;
241+
*)
242+
done_status[$nm]="${status}"
243+
done_reason[$nm]="${reason}"
244+
echo " ${status} ${nm} (${reason})"
245+
;;
246+
esac
123247
done
124248
[[ ${all_done} -eq 1 ]] && break
125249
sleep 10
126250
done
127251

128252
for nm in "${names[@]}"; do
129-
if [[ -n "${done_map[$nm]:-}" ]]; then
130-
RESULTS_NAME+=("${nm}")
131-
RESULTS_STATUS+=("PASS")
132-
RESULTS_REASON+=("${done_map[$nm]#PASS:}")
133-
else
134-
local reason
135-
reason="$(kubectl -n "${NAMESPACE}" get chaininstance "${nm}" -o jsonpath='phase={.status.phase} height={.status.height}' 2>/dev/null || echo 'no-status')"
136-
RESULTS_NAME+=("${nm}")
137-
RESULTS_STATUS+=("FAIL")
138-
RESULTS_REASON+=("timeout: ${reason}")
139-
dump_artifacts "${nm}"
140-
echo " FAIL ${nm} (${reason})"
253+
if [[ -z "${done_status[$nm]:-}" ]]; then
254+
# Final classify pass — turn JUST_READY/PROGRESS into best terminal state.
255+
local out status reason
256+
out="$(classify "${nm}" "${first_ready[$nm]:-0}")"
257+
status="${out%%$'\t'*}"
258+
reason="${out#*$'\t'}"
259+
case "${status}" in
260+
PASS_*|FAIL_*) done_status[$nm]="${status}"; done_reason[$nm]="${reason}" ;;
261+
*) done_status[$nm]="FAIL_NO_STATUS"; done_reason[$nm]="timeout: ${reason}" ;;
262+
esac
263+
echo " ${done_status[$nm]} ${nm} (${done_reason[$nm]})"
141264
fi
265+
RESULTS_NAME+=("${nm}")
266+
RESULTS_STATUS+=("${done_status[$nm]}")
267+
RESULTS_REASON+=("${done_reason[$nm]}")
268+
[[ "${done_status[$nm]}" =~ ^FAIL_ ]] && dump_artifacts "${nm}"
142269
done
143270

144-
kubectl delete -f "${tmpdir}/" --wait=false 2>&1 | tail -5 || true
145-
# Best-effort PVC cleanup (statefulset PVCs aren't auto-deleted)
271+
kubectl delete -f "${tmpdir}/" --wait=false >/dev/null 2>&1 || true
146272
kubectl -n "${NAMESPACE}" delete pvc -l app.kubernetes.io/managed-by=chainplane --wait=false 2>/dev/null || true
147273
rm -rf "${tmpdir}"
148274
}
@@ -159,32 +285,52 @@ done
159285

160286
echo
161287
echo "=== SUMMARY ==="
162-
fail=0
163288
{
164-
printf "| %-40s | %-6s | %s\n" "chain" "status" "reason"
165-
printf "|%s|%s|%s\n" "$(printf '%.s-' {1..42})" "$(printf '%.s-' {1..8})" "$(printf '%.s-' {1..40})"
289+
printf "| %-40s | %-15s | %s\n" "chain" "status" "reason"
290+
printf "|%s|%s|%s\n" "$(printf '%.s-' {1..42})" "$(printf '%.s-' {1..17})" "$(printf '%.s-' {1..40})"
166291
for n in "${!RESULTS_NAME[@]}"; do
167-
printf "| %-40s | %-6s | %s\n" "${RESULTS_NAME[$n]}" "${RESULTS_STATUS[$n]}" "${RESULTS_REASON[$n]}"
292+
printf "| %-40s | %-15s | %s\n" "${RESULTS_NAME[$n]}" "${RESULTS_STATUS[$n]}" "${RESULTS_REASON[$n]}"
168293
done
169294
} > "${ARTIFACTS_DIR}/summary.txt"
170295
cat "${ARTIFACTS_DIR}/summary.txt"
171296

297+
declare -A counts
172298
for n in "${!RESULTS_NAME[@]}"; do
173-
[[ "${RESULTS_STATUS[$n]}" == "FAIL" ]] && fail=$((fail + 1))
299+
s="${RESULTS_STATUS[$n]}"
300+
counts[$s]=$(( ${counts[$s]:-0} + 1 ))
301+
done
302+
303+
pass=0; fail=0
304+
for s in "${!counts[@]}"; do
305+
[[ "$s" =~ ^PASS_ ]] && pass=$(( pass + counts[$s] ))
306+
[[ "$s" =~ ^FAIL_ ]] && fail=$(( fail + counts[$s] ))
174307
done
175308

176309
echo
177-
echo "passed=$(( ${#RESULTS_NAME[@]} - fail )) failed=${fail} total=${#RESULTS_NAME[@]}"
310+
echo "totals: pass=${pass} fail=${fail} total=${#RESULTS_NAME[@]}"
311+
echo "breakdown:"
312+
for s in PASS_SYNC PASS_SOFT PASS_STARTED FAIL_OOM FAIL_CRASH FAIL_IMAGE FAIL_PENDING FAIL_APPLY FAIL_NO_STATUS; do
313+
echo " ${s}=${counts[$s]:-0}"
314+
done
178315

179316
if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
180317
{
181318
echo "## Manual batch e2e summary"
182319
echo
183-
echo "passed=**$(( ${#RESULTS_NAME[@]} - fail ))** failed=**${fail}** total=**${#RESULTS_NAME[@]}**"
320+
echo "**pass=${pass}** **fail=${fail}** total=${#RESULTS_NAME[@]}"
321+
echo
322+
echo "| status | count |"
323+
echo "|---|---|"
324+
for s in PASS_SYNC PASS_SOFT PASS_STARTED FAIL_OOM FAIL_CRASH FAIL_IMAGE FAIL_PENDING FAIL_APPLY FAIL_NO_STATUS; do
325+
echo "| ${s} | ${counts[$s]:-0} |"
326+
done
327+
echo
328+
echo '<details><summary>Per-chain results</summary>'
184329
echo
185330
echo '```'
186331
cat "${ARTIFACTS_DIR}/summary.txt"
187332
echo '```'
333+
echo '</details>'
188334
} >> "${GITHUB_STEP_SUMMARY}"
189335
fi
190336

0 commit comments

Comments
 (0)