fix(e2e): increase leader election lease duration and e2e timeouts

The controller manager was crashing after ~2 minutes in the GitHub
Actions Kind environment (exit code 1). The most likely cause is leader
election lease renewal failing under heavy CPU load from cert-manager
and prometheus-operator running concurrently.

Increased lease durations (15s→40s, 10s→25s, 2s→5s) to tolerate
slow API server responses in resource-constrained CI environments.
Also increased e2e default timeout from 2 to 5 minutes, added a
BeforeEach to wait for controller readiness before each CR test, and
added --previous log capture to diagnose future pod crashes.

Spent ~30min analyzing crash timing in pod events and workqueue metrics.

Author: tazhate

cmd/main.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -336,13 +337,19 @@ func main() {
 	}
 
 	// --- Controller manager --------------------------------------------
+	leaseDuration := 40 * time.Second
+	renewDeadline := 25 * time.Second
+	retryPeriod := 5 * time.Second
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:                 scheme,
-		Metrics:                metricsServerOptions,
-		WebhookServer:          webhookServer,
-		HealthProbeBindAddress: cfg.ProbeAddr,
-		LeaderElection:         cfg.LeaderElect,
-		LeaderElectionID:       defaultLeaderElectID,
+		Scheme:                  scheme,
+		Metrics:                 metricsServerOptions,
+		WebhookServer:           webhookServer,
+		HealthProbeBindAddress:  cfg.ProbeAddr,
+		LeaderElection:          cfg.LeaderElect,
+		LeaderElectionID:        defaultLeaderElectID,
+		LeaseDuration:           &leaseDuration,
+		RenewDeadline:           &renewDeadline,
+		RetryPeriod:             &retryPeriod,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

test/e2e/e2e_test.go

@@ -106,6 +106,13 @@ var _ = Describe("Manager", Ordered, func() {
 				_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err)
 			}
 
+			By("Fetching previous controller manager pod logs (crash logs)")
+			prevCmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace, "--previous")
+			prevLogs, prevErr := utils.Run(prevCmd)
+			if prevErr == nil {
+				_, _ = fmt.Fprintf(GinkgoWriter, "Previous controller logs (crash):\n %s", prevLogs)
+			}
+
 			By("Fetching Kubernetes events")
 			cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp")
 			eventsOutput, err := utils.Run(cmd)
@@ -135,7 +142,7 @@ var _ = Describe("Manager", Ordered, func() {
 		}
 	})
 
-	SetDefaultEventuallyTimeout(2 * time.Minute)
+	SetDefaultEventuallyTimeout(5 * time.Minute)
 	SetDefaultEventuallyPollingInterval(time.Second)
 
 	Context("Manager", func() {
@@ -275,6 +282,39 @@ var _ = Describe("Manager", Ordered, func() {
 	Context("BlockchainNode CR lifecycle", func() {
 		const testNS = namespace
 
+		// waitForControllerReady waits until the controller-manager pod is Running.
+		// This handles the case where the pod restarts (e.g., leader election loss)
+		// and ensures the controller is ready before applying CRs.
+		waitForControllerReady := func() {
+			Eventually(func(g Gomega) {
+				cmd := exec.Command("kubectl", "get",
+					"pods", "-l", "control-plane=controller-manager",
+					"-o", "go-template={{ range .items }}"+
+						"{{ if not .metadata.deletionTimestamp }}"+
+						"{{ .metadata.name }}{{ \"\\n\" }}"+
+						"{{ end }}{{ end }}",
+					"-n", namespace,
+				)
+				podOutput, err := utils.Run(cmd)
+				g.Expect(err).NotTo(HaveOccurred())
+				podNames := utils.GetNonEmptyLines(podOutput)
+				g.Expect(podNames).To(HaveLen(1))
+				controllerPodName = podNames[0]
+
+				cmd = exec.Command("kubectl", "get",
+					"pods", controllerPodName, "-o", "jsonpath={.status.phase}",
+					"-n", namespace,
+				)
+				output, err := utils.Run(cmd)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(output).To(Equal("Running"))
+			}).Should(Succeed())
+		}
+
+		BeforeEach(func() {
+			waitForControllerReady()
+		})
+
 		// cleanupCR deletes a BlockchainNode CR by name and waits for it to disappear.
 		cleanupCR := func(name string) {
 			cmd := exec.Command("kubectl", "delete", "blockchainnode", name,