Skip to content

Commit 902b006

Browse files
weicaoweicao
authored
fix(tidb): preserve PITR log task on monitor failure (#3252)
1 parent ad4cb78 commit 902b006

2 files changed

Lines changed: 616 additions & 31 deletions

File tree

‎addons/tidb/dataprotection/backup-pitr.sh‎

Lines changed: 189 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,202 @@
11
#!/bin/bash
22

3-
function save_backup_status() {
3+
PITR_TERMINATION_REQUESTED=false
4+
5+
function run_br() {
6+
/br "$@"
7+
}
8+
9+
function get_log_backup_status() {
410
# shellcheck disable=SC2086
5-
res=$(/br log status --task-name=pitr --pd "$PD_ADDRESS" $EXTRA_ARGS)
6-
start_time_str=$(echo "$res" | awk -F': ' '/^\s*start:/ {print $2}')
7-
checkpoint_time_str=$(echo "$res" | awk -F': ' '/^checkpoint\[global\]:/ {print $2}' | cut -d';' -f1)
8-
start_time=$(date -d "$start_time_str" -u '+%Y-%m-%dT%H:%M:%SZ')
9-
checkpoint_time=$(date -d "$checkpoint_time_str" -u '+%Y-%m-%dT%H:%M:%SZ')
10-
11-
# use datasafed to get backup size
12-
total_size=$(datasafed stat / | grep TotalSize | awk '{print $2}')
11+
run_br log status --task-name=pitr --pd "$PD_ADDRESS" $EXTRA_ARGS
12+
}
13+
14+
function normalize_utc_time() {
15+
date -d "$1" -u '+%Y-%m-%dT%H:%M:%SZ'
16+
}
17+
18+
function get_backup_total_size() {
19+
datasafed stat / | awk '/TotalSize/ {print $2; exit}'
20+
}
21+
22+
function is_non_negative_int64() {
23+
local value="$1"
24+
local normalized
25+
local max_int64="9223372036854775807"
26+
27+
case "$value" in
28+
''|*[!0-9]*) return 1 ;;
29+
esac
30+
31+
normalized="$value"
32+
while [ "${normalized#0}" != "$normalized" ]; do
33+
normalized="${normalized#0}"
34+
done
35+
[ -n "$normalized" ] || normalized=0
36+
37+
if [ "${#normalized}" -gt 19 ]; then
38+
return 1
39+
fi
40+
# Compare equal-width decimal strings without overflowing Bash 3.2 integers.
41+
# shellcheck disable=SC2071
42+
if [ "${#normalized}" -eq 19 ] && [[ "$normalized" > "$max_int64" ]]; then
43+
return 1
44+
fi
45+
}
46+
47+
function save_backup_status() {
48+
local res start_time_str checkpoint_time_str start_time checkpoint_time total_size
49+
50+
if ! res=$(get_log_backup_status); then
51+
echo "ERROR: failed to query TiDB log backup status" >&2
52+
return 1
53+
fi
54+
55+
start_time_str=$(printf '%s\n' "$res" | awk -F': ' '/^[[:space:]]*start:/ {print $2; exit}')
56+
checkpoint_time_str=$(printf '%s\n' "$res" | awk -F': ' '/^[[:space:]]*checkpoint\[global\]:/ {print $2; exit}' | cut -d';' -f1)
57+
if [ -z "$start_time_str" ] || [ -z "$checkpoint_time_str" ]; then
58+
echo "ERROR: log backup status is missing start or global checkpoint" >&2
59+
return 1
60+
fi
61+
62+
if ! start_time=$(normalize_utc_time "$start_time_str"); then
63+
echo "ERROR: invalid log backup start time: $start_time_str" >&2
64+
return 1
65+
fi
66+
if ! checkpoint_time=$(normalize_utc_time "$checkpoint_time_str"); then
67+
echo "ERROR: invalid log backup checkpoint time: $checkpoint_time_str" >&2
68+
return 1
69+
fi
70+
if [ "$start_time" \> "$checkpoint_time" ]; then
71+
echo "ERROR: log backup start time is later than checkpoint" >&2
72+
return 1
73+
fi
74+
if ! total_size=$(get_backup_total_size) || [ -z "$total_size" ]; then
75+
echo "ERROR: failed to read log backup size" >&2
76+
return 1
77+
fi
78+
if ! is_non_negative_int64 "$total_size"; then
79+
echo "ERROR: log backup size must be a non-negative decimal integer within int64 range" >&2
80+
return 1
81+
fi
82+
1383
echo "start_time: $start_time, checkpoint_time: $checkpoint_time, total_size: $total_size"
1484
DP_save_backup_status_info "$total_size" "$start_time" "$checkpoint_time" "" ""
1585
}
1686

17-
# if the script exits with a non-zero exit code, touch a file to indicate that the backup failed,
18-
# the sync progress container will check this file and exit if it exists
87+
function save_backup_status_with_retry() {
88+
local attempts="${PITR_STATUS_RETRY_ATTEMPTS-3}"
89+
local interval="${PITR_STATUS_RETRY_INTERVAL_SECONDS-2}"
90+
local attempt=1
91+
92+
case "$attempts" in
93+
''|*[!0-9]*|0)
94+
echo "ERROR: PITR_STATUS_RETRY_ATTEMPTS must be a positive integer" >&2
95+
return 2
96+
;;
97+
esac
98+
case "$interval" in
99+
''|*[!0-9]*)
100+
echo "ERROR: PITR_STATUS_RETRY_INTERVAL_SECONDS must be a non-negative integer" >&2
101+
return 2
102+
;;
103+
esac
104+
105+
while [ "$attempt" -le "$attempts" ]; do
106+
if save_backup_status; then
107+
return 0
108+
fi
109+
echo "WARN: log backup status attempt $attempt/$attempts failed" >&2
110+
if [ "$attempt" -lt "$attempts" ]; then
111+
sleep "$interval"
112+
fi
113+
attempt=$((attempt + 1))
114+
done
115+
116+
echo "ERROR: log backup status failed after $attempts attempts" >&2
117+
return 1
118+
}
119+
120+
function start_log_backup() {
121+
# shellcheck disable=SC2086
122+
run_br log start --task-name=pitr --pd "$PD_ADDRESS" --storage "s3://$BUCKET$DP_BACKUP_BASE_PATH?access-key=$ACCESS_KEY_ID&secret-access-key=$SECRET_ACCESS_KEY" --s3.endpoint "$ENDPOINT" $BR_EXTRA_ARGS
123+
}
124+
125+
function stop_log_backup() {
126+
# shellcheck disable=SC2086
127+
run_br log stop --task-name=pitr --pd "$PD_ADDRESS" --storage "s3://$BUCKET$DP_BACKUP_BASE_PATH?access-key=$ACCESS_KEY_ID&secret-access-key=$SECRET_ACCESS_KEY" --s3.endpoint "$ENDPOINT" $EXTRA_ARGS
128+
}
129+
130+
function ensure_log_backup_started() {
131+
local status_rc
132+
133+
if save_backup_status_with_retry; then
134+
echo "INFO: attached to existing TiDB log backup task"
135+
return 0
136+
else
137+
status_rc=$?
138+
fi
139+
if [ "$status_rc" -eq 2 ]; then
140+
return "$status_rc"
141+
fi
142+
143+
echo "INFO: no usable TiDB log backup task status; starting task"
144+
if ! start_log_backup; then
145+
echo "ERROR: failed to start TiDB log backup task" >&2
146+
return 1
147+
fi
148+
return 0
149+
}
150+
151+
function finish_log_backup() {
152+
local exit_code="$1"
153+
local termination_requested="${2:-false}"
154+
155+
if [ "$termination_requested" = "true" ]; then
156+
save_backup_status_with_retry || echo "WARN: final log backup status could not be saved" >&2
157+
if ! stop_log_backup; then
158+
echo "ERROR: failed to stop TiDB log backup task during explicit termination" >&2
159+
[ "$exit_code" -eq 0 ] && exit_code=1
160+
fi
161+
elif [ "$exit_code" -ne 0 ]; then
162+
echo "ERROR: PITR monitor failed with exit code $exit_code; preserving the log backup task" >&2
163+
else
164+
echo "INFO: PITR monitor exited without an explicit termination request; preserving the log backup task" >&2
165+
fi
166+
167+
return "$exit_code"
168+
}
169+
170+
function handle_termination() {
171+
PITR_TERMINATION_REQUESTED=true
172+
exit 0
173+
}
174+
19175
function handle_exit() {
20-
exit_code=$?
21-
save_backup_status
22-
# FIXME: an unexpected pod restart will remove previous log backup progress
23-
# shellcheck disable=SC2086
24-
/br log stop --task-name=pitr --pd "$PD_ADDRESS" --storage "s3://$BUCKET$DP_BACKUP_BASE_PATH?access-key=$ACCESS_KEY_ID&secret-access-key=$SECRET_ACCESS_KEY" --s3.endpoint "$ENDPOINT" $EXTRA_ARGS
25-
if [ $exit_code -ne 0 ]; then
26-
echo "failed with exit code $exit_code"
27-
exit $exit_code
28-
fi
176+
local exit_code=$?
177+
local final_exit_code
178+
179+
trap - EXIT TERM INT
180+
set +e
181+
finish_log_backup "$exit_code" "$PITR_TERMINATION_REQUESTED"
182+
final_exit_code=$?
183+
exit "$final_exit_code"
29184
}
30185

31-
trap handle_exit EXIT
186+
function main() {
187+
setStorageVar
188+
ensure_log_backup_started
32189

33-
setStorageVar
190+
set +x
191+
while true; do
192+
save_backup_status_with_retry || return $?
193+
sleep "${PITR_STATUS_INTERVAL_SECONDS:-20}"
194+
done
195+
}
34196

35-
echo "start log backup"
36-
# shellcheck disable=SC2086
37-
/br log start --task-name=pitr --pd "$PD_ADDRESS" --storage "s3://$BUCKET$DP_BACKUP_BASE_PATH?access-key=$ACCESS_KEY_ID&secret-access-key=$SECRET_ACCESS_KEY" --s3.endpoint "$ENDPOINT" $BR_EXTRA_ARGS
197+
# This is magic for shellspec ut framework, do not modify!
198+
${__SOURCED__:+false} : || return 0
38199

39-
set +x
40-
while true; do
41-
save_backup_status
42-
# todo: prune outdated log
43-
sleep 20
44-
done
200+
trap handle_exit EXIT
201+
trap handle_termination TERM INT
202+
main "$@"

0 commit comments

Comments
 (0)