K8SPG-260: Fix restart after upgrade (#918)

* K8SPG-260: Fix restart after upgrade

* fix envtest

* Add cluster restart check to upgrade-minor tests

* K8SPG-643: Always try stanza-upgrade after a failed stanza-create

* fix upgrade-minor test

* improve major-upgrade

---------

Co-authored-by: Eleonora Zinchenko <[email protected]>
Co-authored-by: Viacheslav Sarzhan <[email protected]>

Author: 3 people

e2e-tests/functions

@@ -2,7 +2,7 @@
 
 # set root repo relatively to a test dir
 ROOT_REPO=${ROOT_REPO:-$(realpath ../../..)}
-CERT_MANAGER_VER="1.14.2"
+CERT_MANAGER_VER="1.15.3"
 test_name=$(basename "$(pwd)")
 source "${ROOT_REPO}/e2e-tests/vars.sh"
 
@@ -58,6 +58,25 @@ deploy_operator() {
 		| kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f -
 }
 
+update_operator() {
+	local cw_prefix=""
+
+	if [[ $OPERATOR_NS ]]; then
+		cw_prefix="cw-"
+	fi
+
+	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/crd.yaml"
+	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/${cw_prefix}rbac.yaml"
+
+	local disable_telemetry=true
+	if [ "${test_name}" == "telemetry-transfer" ]; then
+		disable_telemetry=false
+	fi
+
+	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" patch deployment percona-postgresql-operator -p \
+		'{"spec":{"template":{"spec":{"containers":[{"name":"operator","image":"'${IMAGE}'"}]}}}}'
+}
+
 deploy_operator_gh() {
 	local git_tag="$1"
 	local cw_prefix=""
@@ -740,3 +759,17 @@ get_container_image() {
 
 	echo "${IMAGE_BASE}:${operatorVersion}-ppg${pgVersion}-${component}"
 }
+
+get_postgresql_logs() {
+	local pgVersion=$1
+
+	for pod in $(kubectl get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do
+		local phase=$(kubectl -n ${NAMESPACE} get ${pod} -o jsonpath={".status.phase"})
+		if [[ "${phase}" != "Running" ]]; then
+			echo "Waiting for ${pod} to start running"
+			continue
+		fi
+		echo "find /pgdata/pg${pgVersion}/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \
+			| kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null
+	done
+}

e2e-tests/tests/major-upgrade/01-create-cluster.yaml

@@ -7,7 +7,7 @@ commands:
       set -o xtrace
 
       source ../../functions
-      
+
       get_cr \
         | yq eval '
           .spec.postgresVersion = 12 |

e2e-tests/tests/major-upgrade/10-assert.yaml

@@ -1,6 +1,10 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 360
+commands:
+- script: |-
+    kubectl -n ${NAMESPACE} get pg,pod,job
+    sleep 5
 ---
 apiVersion: pgv2.percona.com/v2
 kind: PerconaPGCluster
@@ -20,3 +24,19 @@ status:
     ready: 3
     size: 3
   state: ready
+---
+kind: Job
+apiVersion: batch/v1
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: major-upgrade
+    postgres-operator.crunchydata.com/pgbackrest: ''
+    postgres-operator.crunchydata.com/pgbackrest-backup: replica-create
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+  ownerReferences:
+    - apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGBackup
+      controller: true
+      blockOwnerDeletion: true
+status:
+  succeeded: 1

e2e-tests/tests/major-upgrade/11-assert.yaml

@@ -1,31 +1,21 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 660
+commands:
+- script: |-
+    kubectl get postgrescluster major-upgrade \
+      -n ${NAMESPACE} \
+      -o yaml \
+    | yq eval '.status.pgbackrest.repos' -
 ---
-kind: Job
-apiVersion: batch/v1
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
 metadata:
-  annotations:
-    postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-12-to-13
-  labels:
-    postgres-operator.crunchydata.com/pgbackrest-backup: manual
-    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
-  ownerReferences:
-    - apiVersion: pgv2.percona.com/v2
-      kind: PerconaPGBackup
-      controller: true
-      blockOwnerDeletion: true
+  name: major-upgrade
 status:
-  succeeded: 1
----
-apiVersion: pgv2.percona.com/v2
-kind: PerconaPGBackup
-metadata:
-  name: backup-after-12-to-13
-spec:
-  pgCluster: major-upgrade
-  repoName: repo1
-  options:
-    - --type=full
-status:
-  state: Succeeded
+  pgbackrest:
+    repos:
+    - bound: true
+      name: repo2
+      replicaCreateBackupComplete: true
+      stanzaCreated: true

e2e-tests/tests/major-upgrade/11-change-repo.yaml

@@ -0,0 +1,12 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      kubectl patch \
+        -n $NAMESPACE \
+        perconapgcluster major-upgrade \
+        --type='json' \
+        -p='[{"op": "replace", "path": "/spec/backups/pgbackrest/repos/0/name", "value": "repo2"}]'

e2e-tests/tests/major-upgrade/12-assert.yaml

@@ -1,30 +1,31 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 720
+timeout: 660
 ---
-apiVersion: pgv2.percona.com/v2
-kind: PerconaPGRestore
+kind: Job
+apiVersion: batch/v1
 metadata:
-  name: restore-after-12-to-13
-spec:
-  pgCluster: major-upgrade
-  repoName: repo1
+  annotations:
+    postgres-operator.crunchydata.com/pgbackrest-backup: backup-after-12-to-13
+  labels:
+    postgres-operator.crunchydata.com/pgbackrest-backup: manual
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo2
+  ownerReferences:
+    - apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGBackup
+      controller: true
+      blockOwnerDeletion: true
 status:
-  state: Succeeded
+  succeeded: 1
 ---
 apiVersion: pgv2.percona.com/v2
-kind: PerconaPGCluster
+kind: PerconaPGBackup
 metadata:
-  name: major-upgrade
+  name: backup-after-12-to-13
+spec:
+  pgCluster: major-upgrade
+  repoName: repo2
+  options:
+    - --type=full
 status:
-  pgbouncer:
-    ready: 3
-    size: 3
-  postgres:
-    instances:
-    - name: instance1
-      ready: 3
-      size: 3
-    ready: 3
-    size: 3
-  state: ready
+  state: Succeeded

e2e-tests/tests/major-upgrade/11-run-backup.yaml renamed to e2e-tests/tests/major-upgrade/12-run-backup.yaml

@@ -4,6 +4,6 @@ metadata:
   name: backup-after-12-to-13
 spec:
   pgCluster: major-upgrade
-  repoName: repo1
+  repoName: repo2
   options:
     - --type=full

e2e-tests/tests/major-upgrade/13-assert.yaml

@@ -1,10 +1,52 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 30
+timeout: 720
+commands:
+- script: |-
+      set -o errexit
+
+      kubectl -n ${NAMESPACE} get pod
+
+      for pod in $(kubectl -n ${NAMESPACE} get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do
+              phase=$(kubectl -n ${NAMESPACE} get pod/${pod} -o jsonpath={".status.phase"})
+              if [[ "${phase}" != "Running" ]]; then
+                      echo "Waiting for ${pod} to start running"
+                      continue
+              fi
+              echo "PostgreSQL logs from ${pod}:"
+              echo "find /pgdata/pg13/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \
+                      | kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null
+      done
+
+      sleep 30
+collectors:
+- type: pod
+  selector: "postgres-operator.crunchydata.com/data=postgres"
+  tail: 30
 ---
-kind: ConfigMap
-apiVersion: v1
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
 metadata:
-  name: 05-read-from-primary
-data:
-  data: ' 100500'
+  name: restore-after-12-to-13
+spec:
+  pgCluster: major-upgrade
+  repoName: repo2
+status:
+  state: Succeeded
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: major-upgrade
+status:
+  pgbouncer:
+    ready: 3
+    size: 3
+  postgres:
+    instances:
+    - name: instance1
+      ready: 3
+      size: 3
+    ready: 3
+    size: 3
+  state: ready

e2e-tests/tests/major-upgrade/12-run-restore.yaml renamed to e2e-tests/tests/major-upgrade/13-run-restore.yaml

@@ -4,4 +4,4 @@ metadata:
   name: restore-after-12-to-13
 spec:
   pgCluster: major-upgrade
-  repoName: repo1
+  repoName: repo2

e2e-tests/tests/major-upgrade/14-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: 05-read-from-primary
+data:
+  data: ' 100500'