Improve the backup e2e tests to allow multiple tests

e2e/fixtures/factory.go

@@ -784,6 +784,13 @@ func (factory *Factory) CreateIfAbsent(object client.Object) error {
 			objectCopy,
 		)
 
+	if objectCopy.GetDeletionTimestamp() != nil {
+		return fmt.Errorf(
+			"resource is currently under deletion with deletion timestamp: %s",
+			objectCopy.GetDeletionTimestamp().String(),
+		)
+	}
+
 	if err != nil {
 		if k8serrors.IsNotFound(err) {
 			return ctrlClient.Create(context.Background(), object)

e2e/fixtures/fdb_backup.go

@@ -234,7 +234,19 @@ func (fdbBackup *FdbBackup) WaitForRestorableVersion(version uint64) {
 		status := &fdbv1beta2.FoundationDBLiveBackupStatusState{}
 		g.Expect(json.Unmarshal([]byte(out), status)).NotTo(gomega.HaveOccurred())
 
-		log.Println("Backup status:", status)
+		var latestRestorableVersion uint64
+		if status.LatestRestorablePoint != nil {
+			latestRestorableVersion = ptr.Deref(status.LatestRestorablePoint.Version, 0)
+		}
+
+		log.Println(
+			"Backup status running:",
+			status.Running,
+			"restorable:",
+			ptr.Deref(status.Restorable, false),
+			"latestRestorablePoint:",
+			latestRestorableVersion,
+		)
 		g.Expect(ptr.Deref(status.Restorable, false)).To(gomega.BeTrue())
 		g.Expect(status.LatestRestorablePoint).NotTo(gomega.BeNil())
 
@@ -271,5 +283,13 @@ func (fdbBackup *FdbBackup) Destroy() {
 		}
 
 		g.Expect(err).NotTo(gomega.HaveOccurred())
+	}).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).To(gomega.Succeed())
+
+	// Ensure that the resource is removed.
+	gomega.Eventually(func(g gomega.Gomega) {
+		backup := &fdbv1beta2.FoundationDBBackup{}
+		err := fdbBackup.fdbCluster.getClient().
+			Get(context.Background(), client.ObjectKeyFromObject(fdbBackup.backup), backup)
+		g.Expect(k8serrors.IsNotFound(err)).To(gomega.BeTrue())
 	}).WithTimeout(10 * time.Minute).WithPolling(1 * time.Second).To(gomega.Succeed())
 }

e2e/fixtures/fdb_cluster.go

@@ -1740,10 +1740,10 @@ func (fdbCluster *FdbCluster) ClearRange(prefixBytes []byte, timeout int) {
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	end := FdbPrintable(endBytes)
 	_, stderr, err := fdbCluster.RunFdbCliCommandInOperatorWithoutRetry(fmt.Sprintf(
-		"writemode on; clearrange %s %s",
+		"writemode on; option on ACCESS_SYSTEM_KEYS; clearrange %s %s",
 		begin,
 		end,
-	), false, timeout)
+	), true, timeout)
 
 	gomega.Expect(err).NotTo(gomega.HaveOccurred(), stderr)
 }
@@ -1811,7 +1811,7 @@ func (fdbCluster *FdbCluster) GenerateRandomValues(
 	for i := 0; i < n; i++ {
 		res = append(res, KeyValue{
 			Key:   append([]byte{prefix}, index...),
-			Value: []byte(fdbCluster.factory.RandStringRunes(4)),
+			Value: []byte(fdbCluster.factory.RandStringRunes(24)),
 		})
 		index, err = FdbStrinc(index)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())

e2e/fixtures/fdb_restore.go

@@ -26,56 +26,71 @@ import (
 	"strconv"
 	"time"
 
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/v2/api/v1beta2"
 	"github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+// FdbRestore represents a fdbv1beta2.FoundationDBRestore resource for doing restores to a FdbCluster.
+type FdbRestore struct {
+	restore    *fdbv1beta2.FoundationDBRestore
+	fdbCluster *FdbCluster
+}
+
 // CreateRestoreForCluster will create a FoundationDBRestore resource based on the provided backup resource.
 // For more information how the backup system with the operator is working please look at
 // the operator documentation: https://github.com/FoundationDB/fdb-kubernetes-operator/v2/blob/master/docs/manual/backup.md
-func (factory *Factory) CreateRestoreForCluster(backup *FdbBackup) {
+func (factory *Factory) CreateRestoreForCluster(backup *FdbBackup) *FdbRestore {
 	gomega.Expect(backup).NotTo(gomega.BeNil())
-	restore := &fdbv1beta2.FoundationDBRestore{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      backup.fdbCluster.Name(),
-			Namespace: backup.fdbCluster.Namespace(),
-		},
-		Spec: fdbv1beta2.FoundationDBRestoreSpec{
-			DestinationClusterName: backup.fdbCluster.Name(),
-			BlobStoreConfiguration: backup.backup.Spec.BlobStoreConfiguration,
-			CustomParameters:       backup.backup.Spec.CustomParameters,
+	restore := &FdbRestore{
+		restore: &fdbv1beta2.FoundationDBRestore{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      backup.fdbCluster.Name(),
+				Namespace: backup.fdbCluster.Namespace(),
+			},
+			Spec: fdbv1beta2.FoundationDBRestoreSpec{
+				DestinationClusterName: backup.fdbCluster.Name(),
+				BlobStoreConfiguration: backup.backup.Spec.BlobStoreConfiguration,
+				CustomParameters:       backup.backup.Spec.CustomParameters,
+			},
 		},
+		fdbCluster: backup.fdbCluster,
 	}
-	gomega.Expect(factory.CreateIfAbsent(restore)).NotTo(gomega.HaveOccurred())
+
+	gomega.Expect(factory.CreateIfAbsent(restore.restore)).NotTo(gomega.HaveOccurred())
 
 	factory.AddShutdownHook(func() error {
-		return factory.GetControllerRuntimeClient().Delete(context.Background(), restore)
+		restore.Destroy()
+		return nil
 	})
 
-	waitForRestoreToComplete(backup)
+	restore.waitForRestoreToComplete(backup)
+
+	return restore
 }
 
 // waitForRestoreToComplete waits until the restore completed.
-func waitForRestoreToComplete(backup *FdbBackup) {
-	ctrlClient := backup.fdbCluster.getClient()
+func (restore *FdbRestore) waitForRestoreToComplete(backup *FdbBackup) {
+	ctrlClient := restore.fdbCluster.getClient()
 
 	lastReconcile := time.Now()
 	gomega.Eventually(func(g gomega.Gomega) fdbv1beta2.FoundationDBRestoreState {
-		restore := &fdbv1beta2.FoundationDBRestore{}
-		g.Expect(ctrlClient.Get(context.Background(), client.ObjectKeyFromObject(backup.backup), restore)).
+		currentRestore := &fdbv1beta2.FoundationDBRestore{}
+		g.Expect(ctrlClient.Get(context.Background(), client.ObjectKeyFromObject(restore.restore), currentRestore)).
 			To(gomega.Succeed())
-		log.Println("restore state:", restore.Status.State)
+		log.Println("restore state:", currentRestore.Status.State)
 
 		if time.Since(lastReconcile) > time.Minute {
 			lastReconcile = time.Now()
-			patch := client.MergeFrom(restore.DeepCopy())
-			if restore.Annotations == nil {
-				restore.Annotations = make(map[string]string)
+			patch := client.MergeFrom(currentRestore.DeepCopy())
+			if currentRestore.Annotations == nil {
+				currentRestore.Annotations = make(map[string]string)
 			}
-			restore.Annotations["foundationdb.org/reconcile"] = strconv.FormatInt(
+			currentRestore.Annotations["foundationdb.org/reconcile"] = strconv.FormatInt(
 				time.Now().UnixNano(),
 				10,
 			)
@@ -84,10 +99,10 @@ func waitForRestoreToComplete(backup *FdbBackup) {
 			// This should speed up the reconcile phase.
 			gomega.Expect(ctrlClient.Patch(
 				context.Background(),
-				restore,
+				currentRestore,
 				patch)).To(gomega.Succeed())
 
-			out, _, err := backup.fdbCluster.ExecuteCmdOnPod(
+			out, _, err := restore.fdbCluster.ExecuteCmdOnPod(
 				*backup.GetBackupPod(),
 				fdbv1beta2.MainContainerName,
 				"fdbrestore status --dest_cluster_file $FDB_CLUSTER_FILE",
@@ -98,6 +113,27 @@ func waitForRestoreToComplete(backup *FdbBackup) {
 			g.Expect(err).To(gomega.Succeed())
 		}
 
-		return restore.Status.State
+		return currentRestore.Status.State
 	}).WithTimeout(20 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Equal(fdbv1beta2.CompletedFoundationDBRestoreState))
 }
+
+// Destroy will delete the FoundationDBRestore for the associated FdbBackup if it exists.
+func (restore *FdbRestore) Destroy() {
+	gomega.Eventually(func(g gomega.Gomega) {
+		err := restore.fdbCluster.factory.GetControllerRuntimeClient().
+			Delete(context.Background(), restore.restore)
+		if k8serrors.IsNotFound(err) {
+			return
+		}
+
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+	}).WithTimeout(4 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Succeed())
+
+	// Ensure that the resource is removed.
+	gomega.Eventually(func(g gomega.Gomega) {
+		currentRestore := &fdbv1beta2.FoundationDBRestore{}
+		err := restore.fdbCluster.getClient().
+			Get(context.Background(), client.ObjectKeyFromObject(restore.restore), currentRestore)
+		g.Expect(k8serrors.IsNotFound(err)).To(gomega.BeTrue())
+	}).WithTimeout(10 * time.Minute).WithPolling(1 * time.Second).To(gomega.Succeed())
+}

e2e/fixtures/kubernetes_fixtures.go

@@ -21,21 +21,21 @@
 package fixtures
 
 import (
-	ctx "context"
+	"context"
 	"fmt"
 	"log"
 	"time"
 
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-
-	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/config"
-
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/v2/api/v1beta2"
 	"github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
 )
 
 const (
@@ -108,7 +108,7 @@ func (factory *Factory) createNamespace(suffix string) string {
 
 		namespaceResource := &corev1.Namespace{}
 		err = factory.controllerRuntimeClient.Get(
-			ctx.Background(),
+			context.Background(),
 			client.ObjectKey{Namespace: "", Name: namespace},
 			namespaceResource,
 		)
@@ -162,16 +162,14 @@ func (factory *Factory) createNamespace(suffix string) string {
 	factory.AddShutdownHook(func() error {
 		log.Printf("finished all tests, start deleting namespace %s\n", namespace)
 
-		gomega.Eventually(func() error {
+		gomega.Eventually(func(g gomega.Gomega) {
 			podList := &corev1.PodList{}
 			err := factory.controllerRuntimeClient.List(
-				ctx.Background(),
+				context.Background(),
 				podList,
 				client.InNamespace(namespace),
 			)
-			if err != nil {
-				return err
-			}
+			g.Expect(err).NotTo(gomega.HaveOccurred())
 
 			for _, pod := range podList.Items {
 				if len(pod.Finalizers) > 0 {
@@ -180,8 +178,40 @@ func (factory *Factory) createNamespace(suffix string) string {
 				}
 			}
 
-			return nil
-		}).WithTimeout(2 * time.Minute).WithPolling(1 * time.Second).ShouldNot(gomega.HaveOccurred())
+			backupList := &fdbv1beta2.FoundationDBBackupList{}
+			err = factory.controllerRuntimeClient.List(
+				context.Background(),
+				backupList,
+				client.InNamespace(namespace),
+			)
+			g.Expect(err).NotTo(gomega.HaveOccurred())
+
+			for _, backup := range backupList.Items {
+				if len(backup.Finalizers) > 0 {
+					log.Printf("Removing finalizer from backup %s/%s\n", namespace, backup.Name)
+					gomega.Eventually(func(g gomega.Gomega) {
+						fetchedBackup := &fdbv1beta2.FoundationDBBackup{}
+						err = factory.controllerRuntimeClient.Get(
+							context.Background(),
+							client.ObjectKeyFromObject(ptr.To(backup)),
+							fetchedBackup,
+						)
+						if k8serrors.IsNotFound(err) {
+							return
+						}
+
+						g.Expect(err).NotTo(gomega.HaveOccurred())
+						if len(fetchedBackup.Finalizers) > 0 {
+							fetchedBackup.SetFinalizers([]string{})
+							g.Expect(factory.controllerRuntimeClient.Update(context.Background(), fetchedBackup)).
+								NotTo(gomega.HaveOccurred())
+						}
+
+						g.Expect(fetchedBackup.Finalizers).To(gomega.BeEmpty())
+					}).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Succeed())
+				}
+			}
+		}).WithTimeout(2 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Succeed())
 
 		factory.Delete(&corev1.Namespace{
 			ObjectMeta: metav1.ObjectMeta{
@@ -201,7 +231,7 @@ func (factory *Factory) checkIfNamespaceIsTerminating(name string) error {
 	controllerClient := factory.GetControllerRuntimeClient()
 
 	namespace := &corev1.Namespace{}
-	err := controllerClient.Get(ctx.Background(), client.ObjectKey{Name: name}, namespace)
+	err := controllerClient.Get(context.Background(), client.ObjectKey{Name: name}, namespace)
 	if err != nil {
 		if k8serrors.IsNotFound(err) {
 			return nil
@@ -229,7 +259,7 @@ func (factory *Factory) checkIfNamespaceIsTerminating(name string) error {
 		deletionTimestamp.String(),
 	)
 	podList := &corev1.PodList{}
-	err = controllerClient.List(ctx.Background(), podList, client.InNamespace(name))
+	err = controllerClient.List(context.Background(), podList, client.InNamespace(name))
 	if err != nil {
 		return err
 	}

e2e/fixtures/pods.go

@@ -132,7 +132,7 @@ func (factory *Factory) SetFinalizerForPod(pod *corev1.Pod, finalizers []string)
 	}
 
 	controllerClient := factory.GetControllerRuntimeClient()
-	gomega.Eventually(func(g gomega.Gomega) bool {
+	gomega.Eventually(func(g gomega.Gomega) {
 		fetchedPod := &corev1.Pod{}
 		g.Expect(controllerClient.Get(context.Background(), client.ObjectKeyFromObject(pod), fetchedPod)).
 			NotTo(gomega.HaveOccurred())
@@ -144,9 +144,7 @@ func (factory *Factory) SetFinalizerForPod(pod *corev1.Pod, finalizers []string)
 		}
 
 		g.Expect(fetchedPod.Finalizers).To(gomega.ConsistOf(finalizers))
-
-		return true
-	}).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).Should(gomega.BeTrue())
+	}).WithTimeout(1 * time.Minute).WithPolling(1 * time.Second).Should(gomega.Succeed())
 }
 
 // GetProcessClass returns the Process class of this Pod.

e2e/scripts/remove_namespaces

@@ -21,6 +21,11 @@ do
     fi
 
     echo "start deleting namespace ${ns}"
+    for backup in $(kubectl -n ${ns} get fdbbackup --no-headers -o name);
+    do
+      echo "Removing finalizer from ${backup} in namespace ${ns}"
+      kubectl -n ${ns} patch ${backup} -p '{"metadata":{"finalizers":null}}' --type=merge
+    done
 
     kubectl delete ns --ignore-not-found "${ns}"
   ) &