K8SPXC-1617: Run a full backup after each PiTR in test

cmd/pitr/pxc/pxc.go

@@ -219,6 +219,7 @@ func GetPXCFirstHost(ctx context.Context, pxcServiceName string) (string, error)
 	sort.Strings(nodes)
 	lastHost := ""
 	for _, node := range nodes {
+		log.Printf("PXC Node: %s", node)
 		if strings.Contains(node, "wsrep_ready:ON:wsrep_connected:ON:wsrep_local_state_comment:Synced:wsrep_cluster_status:Primary") {
 			nodeArr := strings.Split(node, ":")
 			lastHost = nodeArr[0]
@@ -229,6 +230,8 @@ func GetPXCFirstHost(ctx context.Context, pxcServiceName string) (string, error)
 		return "", errors.New("can't find host")
 	}
 
+	log.Printf("connecting to %s", lastHost)
+
 	return lastHost, nil
 }
 

e2e-tests/functions

@@ -67,6 +67,13 @@ log() {
        echo "[$(date +%Y-%m-%dT%H:%M:%S%z)]" $*
 }
 
+sleep_with_log() {
+	local d=$1
+
+	log "sleeping for ${d} seconds"
+	sleep ${d}
+}
+
 
 HELM_VERSION=$(helm version -c | $sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/')
 if [ "${HELM_VERSION:0:2}" == "v2" ]; then

e2e-tests/pitr/conf/restore-on-pitr-minio-time.yaml

@@ -4,7 +4,7 @@ metadata:
   name: restore-on-pitr-minio-time
 spec:
   pxcCluster: pitr
-  backupName: on-pitr-minio
+  backupName: on-pitr-minio-2
   pitr:
     type: date
     date: "<datetime>"

e2e-tests/pitr/run

@@ -14,6 +14,22 @@ if [[ $IMAGE_PXC =~ 5\.7 ]]; then
 	exit 0
 fi
 
+run_backup() {
+	local cluster=$1
+	local backup=$2
+	local name=$3
+	if [ -z ${name} ]; then
+		name=$backup
+	fi
+
+	log "run backup pxc-backup/${name}"
+	cat $test_dir/conf/${backup}.yml \
+		| yq eval '.metadata.name="'${name}'"' \
+		| kubectl_bin apply -f -
+
+	wait_backup $name
+}
+
 write_test_data() {
 	local cluster=$1
 	local config=$2
@@ -34,9 +50,9 @@ write_test_data() {
 	run_mysql \
 		'INSERT test.test (id) VALUES (100500); INSERT test.test (id) VALUES (100501); INSERT test.test (id) VALUES (100502);' \
 		"-h $proxy -uroot -proot_password"
-	sleep 30
+	sleep_with_log 30
 	for i in $(seq 0 $((size - 1))); do
-		compare_mysql_cmd "select-3" "SELECT * from test.test;" "-h $cluster-pxc-$i.$cluster-pxc -uroot -proot_password"
+		compare_mysql_cmd "select-2" "SELECT * from test.test;" "-h $cluster-pxc-$i.$cluster-pxc -uroot -proot_password"
 	done
 
 	if [ "$(is_keyring_plugin_in_use "$cluster")" ]; then
@@ -46,12 +62,18 @@ write_test_data() {
 
 write_data_for_pitr() {
 	local cluster=$1
+	local start=$2
+	local rows=$3
+
 	local proxy=$(get_proxy "$cluster")
 
-	desc "write data for pitr"
-	run_mysql \
-		'INSERT test.test (id) VALUES (100503); INSERT test.test (id) VALUES (100504); INSERT test.test (id) VALUES (100505);' \
-		"-h $proxy -uroot -proot_password"
+	local sql=""
+	for ((i=0; i<rows; i++)); do
+		sql+="INSERT test.test (id) VALUES ($((start+i)));"
+	done
+
+	log "inserting ${rows} rows for pitr check"
+	run_mysql "${sql}" "-h ${proxy} -uroot -proot_password"
 }
 
 check_latest_restorable_time() {
@@ -60,19 +82,73 @@ check_latest_restorable_time() {
 	desc 'check latest restorable time'
 	latest_restorable_time=$(kubectl_bin get pxc-backup ${backup} -o jsonpath='{.status.latestRestorableTime}')
 	if [[ ${latest_restorable_time} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T ]]; then
-		echo "Latest restorable time is ${latest_restorable_time}"
+		log "Latest restorable time is ${latest_restorable_time}"
 	else
-		echo "Latest restorable time is not set"
+		log "Latest restorable time is not set"
 		exit 1
 	fi
 }
 
+check_binlog_collection() {
+	local timeout=60
+	local binlogs_exist=0
+
+	for i in $(seq 1 5); do
+		log "Checking if binlogs exist in bucket (attempt $i)..."
+		binlogs_exist=$(
+			kubectl_bin run -n "${NAMESPACE}" -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
+				/usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
+				/usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 ls operator-testing/binlogs/ | grep -c "binlog" | cat
+			exit "${PIPESTATUS[0]}"
+		)
+		if [ "$binlogs_exist" -gt 0 ]; then
+			log "${binlogs_exist} binlogs found in bucket"
+			break
+		else
+			d=$((timeout * i))
+			log "No binlogs found in bucket."
+			sleep_with_log ${d}
+		fi
+	done
+
+	if [ "$binlogs_exist" -eq 0 ]; then
+		log "Binlogs are not found in S3"
+		exit 1
+	fi
+
+	desc "binlogs are collected: OK"
+}
+
+check_binlog_gap() {
+	local backup=$1
+
+	if [[ $(kubectl get pxc-backup ${backup} -o jsonpath='{.status.conditions}' | grep -c 'Binlog with GTID set') -eq 1 ]]; then
+		log "Binlog gap detected"
+		exit 1
+	fi
+
+	desc "no binlog gap after pxc-backup/${backup}: OK"
+}
+
+get_gtid_executed() {
+	local host=$1
+
+	local gtid=$(run_mysql "SELECT @@gtid_executed;" "-h ${host} -uroot -proot_password" | $sed 's/\([a-f0-9-]\{36\}\):[0-9]*-\([0-9]*\).*/\1:\2/')
+
+	if [[ ! ${gtid} =~ ${GTID_PATTERN} ]]; then
+		printf "Some garbage --> %s <-- instead of legit GTID. Exiting" ${gtid}
+		exit 1
+	fi
+
+	echo ${gtid}
+}
+
 main() {
 	create_infra $namespace
 	deploy_cert_manager
 	kubectl_bin apply -f "$test_dir/conf/issuer.yml"
 	kubectl_bin apply -f "$test_dir/conf/cert.yml"
-	sleep 25
+	sleep_with_log 25
 	# We are using minio with tls enabled to check if `verifyTLS: false` works fine
 	start_minio "tls-minio"
 
@@ -87,86 +163,58 @@ main() {
 	write_test_data "$cluster"
 
 	# test changing xtrabackup password
-	desc "changing xtrabackup password multiple times"
+	desc "patching xtrabackup password"
 	patch_secret "my-cluster-secrets" "xtrabackup" "$(echo -n "pass1" | base64)"
 	wait_cluster_consistency ${cluster} 3 2
+
+	desc "patching xtrabackup password"
 	patch_secret "my-cluster-secrets" "xtrabackup" "$(echo -n "pass2" | base64)"
 	wait_cluster_consistency ${cluster} 3 2
+
+	desc "patching xtrabackup password"
 	patch_secret "my-cluster-secrets" "xtrabackup" "$(echo -n "pass3" | base64)"
 	wait_cluster_consistency ${cluster} 3 2
 
 	compare_kubectl secret/${cluster}-mysql-init
 
-	desc 'show binlog events'
 	proxy=$(get_proxy "$cluster")
-	run_mysql "SHOW BINLOG EVENTS IN 'binlog.000005';" "-h ${proxy} -uroot -proot_password"
-	run_mysql "SHOW BINLOG EVENTS IN 'binlog.000006';" "-h ${proxy} -uroot -proot_password"
-
-	time_now=$(run_mysql "SELECT now();" "-h ${proxy} -uroot -proot_password")
-	gtid=$(run_mysql "SELECT @@gtid_executed;" "-h ${proxy} -uroot -proot_password" | $sed 's/\([a-f0-9-]\{36\}\):[0-9]*-\([0-9]*\).*/\1:\2/')
+	gtid=$(get_gtid_executed ${proxy})
 
-	if [[ ! ${gtid} =~ ${GTID_PATTERN} ]]; then
-		printf "Some garbage --> %s <-- instead of legit GTID. Exiting" ${gtid}
-		exit 1
-	fi
-
-	write_data_for_pitr "$cluster"
-	sleep 120 # need to wait while collector catch new data
+	write_data_for_pitr "$cluster" 100503 3
+	sleep_with_log 130 # wait for two binlog collection cycles
 	check_latest_restorable_time "on-pitr-minio"
 
-	timeout=60
-	binlogs_exist=0
-	for i in $(seq 1 5); do
-		echo "Checking if binlogs exist in bucket (attempt $i)..."
-		binlogs_exist=$(
-			kubectl_bin run -n "${NAMESPACE}" -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
-				/usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
-				/usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 ls operator-testing/binlogs/ | grep -c "binlog" | cat
-			exit "${PIPESTATUS[0]}"
-		)
-		if [ "$binlogs_exist" -gt 0 ]; then
-			echo "${binlogs_exist} binlogs found in bucket"
-			break
-		else
-			d=$((timeout * i))
-			echo "No binlogs found in bucket. Sleeping for ${d} seconds..."
-			sleep ${d}
-		fi
-	done
+	check_binlog_collection
 
-	if [ "$binlogs_exist" -eq 0 ]; then
-		echo "Binlogs are not found in S3"
-		exit 1
-	fi
+	check_binlog_gap
 
+	desc "[CASE 1] PiTR with GTID: ${gtid}"
 	run_recovery_check_pitr "$cluster" "restore-on-pitr-minio-gtid" "on-pitr-minio" "select-2" "" "" "$gtid"
-	desc "done gtid type"
+	desc "[CASE 1] PiTR with GTID: OK"
 
-	desc 'check for passwords leak'
-	check_passwords_leak
+	run_backup "$cluster" "on-pitr-minio" "on-pitr-minio-2"
+	sleep_with_log 130 # wait for two binlog collection cycles
+	check_binlog_gap "on-pitr-minio-2"
 
-	sleep 60
-	if [[ $(kubectl get pxc-backup on-pitr-minio -o jsonpath='{.status.conditions}' | grep -c 'Binlog with GTID set') -eq 1 ]]; then
-		echo "Binlog gap detected"
-		exit 1
-	fi
+	time_now=$(run_mysql "SELECT now();" "-h ${proxy} -uroot -proot_password")
 
+	write_data_for_pitr "$cluster" 100503 3
+
+	desc "[CASE 2] PiTR with datetime: ${time_now}"
 	run_recovery_check_pitr "$cluster" "restore-on-pitr-minio-time" "on-pitr-minio" "select-3" "$time_now" "" ""
-	desc "done date type"
-	sleep 60
-	if [[ $(kubectl get pxc-backup on-pitr-minio -o jsonpath='{.status.conditions}' | grep -c 'Binlog with GTID set') -eq 1 ]]; then
-		echo "Binlog gap detected"
-		exit 1
-	fi
+	desc "[CASE 2] PiTR with datetime: OK"
+
+	run_backup "$cluster" "on-pitr-minio" "on-pitr-minio-3"
+	sleep_with_log 130 # wait for two binlog collection cycles
+	check_binlog_gap "on-pitr-minio-3"
+
+	write_data_for_pitr "$cluster" 100503 3
+	sleep_with_log 130 # wait for two binlog collection cycles
 
-	dest=$(sed 's,/,\\/,g' <<<$(kubectl get pxc-backup on-pitr-minio -o jsonpath='{.status.destination}'))
+	dest=$(sed 's,/,\\/,g' <<<$(kubectl get pxc-backup on-pitr-minio-3 -o jsonpath='{.status.destination}'))
+	desc "[CASE 3] PiTR with latest using backupSource: ${dest}"
 	run_recovery_check_pitr "$cluster" "restore-on-pitr-minio" "on-pitr-minio" "select-4" "" "$dest" ""
-	desc "done latest type"
-	sleep 60
-	if [[ $(kubectl get pxc-backup on-pitr-minio -o jsonpath='{.status.conditions}' | grep -c 'Binlog with GTID set') -eq 1 ]]; then
-		echo "Binlog gap detected"
-		exit 1
-	fi
+	desc "[CASE 3] PiTR with latest using backupSource: OK"
 
 	destroy $namespace
 	desc "test passed"