MLE-22135 Port Helm change to Operator (#93)

* MLE-22135: change liveness probe

* add retry logic to group configuration

* update haproxy image

* adjust testing time

* update marklogic image version

* remove image

* update image version in Jenkins

* fix makefile for pipeline

* fix validate rule and image load

* push changes from build

* MLE-22902: fix operator pipeline bug

* ignore docker cache

* fix validating rule

* add debug message for Jenkins Pipeline

* fix test with bug: index out of range [0] with length 0

* fix test bug with index out of range [1] with length 1

* add debug info

* revert changes back to makefile

* uncomment changes

---------

Co-authored-by: Peng Zhou <[email protected]>

Author: pengzhouml

Jenkinsfile

@@ -164,7 +164,8 @@ pipeline {
     // }
 
     parameters {
-        string(name: 'dockerImage', defaultValue: 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11', description: 'Docker image to use for tests.', trim: true)
+        string(name: 'E2E_MARKLOGIC_IMAGE_VERSION', defaultValue: 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless:latest-12', description: 'Docker image to use for tests.', trim: true)
+        string(name: 'IMG', defaultValue: 'testrepo/marklogic-operator-image-dev:internal', description: 'Docker image for Running Operator Container', trim: true)
         string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
     }
 

Makefile

@@ -11,7 +11,7 @@ VERIFY_HUGE_PAGES ?= false
 export E2E_DOCKER_IMAGE ?= $(IMG)
 export E2E_KUSTOMIZE_VERSION ?= $(KUSTOMIZE_VERSION)
 export E2E_CONTROLLER_TOOLS_VERSION ?= $(CONTROLLER_TOOLS_VERSION)
-export E2E_MARKLOGIC_IMAGE_VERSION ?= progressofficial/marklogic-db:11.3.1-ubi-rootless-2.1.0
+export E2E_MARKLOGIC_IMAGE_VERSION ?= progressofficial/marklogic-db:11.3.1-ubi-rootless-2.1.3
 export E2E_KUBERNETES_VERSION ?= v1.31.0
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
@@ -65,7 +65,8 @@ OPERATOR_SDK_VERSION ?= v1.34.2
 
 # Image URL to use all building/pushing image targets
 # Image for dev: ml-marklogic-operator-dev.bed-artifactory.bedford.progress.com/marklogic-operator-kubernetes
-IMG ?= progressofficial/marklogic-operator-kubernetes:$(VERSION)
+# IMG ?= progressofficial/marklogic-operator-kubernetes:$(VERSION)
+IMG = "testrepo/marklogic-operator-image-dev:1.0.0"
 
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
@@ -162,6 +163,9 @@ e2e-setup-minikube: kustomize controller-gen build docker-build
 	minikube start --driver=docker --kubernetes-version=$(E2E_KUBERNETES_VERSION) --memory=8192 --cpus=2
 	minikube addons enable ingress
 	minikube image load $(IMG)
+	minikube image load $(E2E_MARKLOGIC_IMAGE_VERSION)
+	minikube image load "docker.io/haproxytech/haproxy-alpine:3.2"
+	minikube image ls
 
 .PHONY: e2e-cleanup-minikube
 e2e-cleanup-minikube:

api/v1/marklogiccluster_types.go

@@ -27,7 +27,7 @@ import (
 
 // MarklogicClusterSpec defines the desired state of MarklogicCluster
 
-// +kubebuilder:validation:XValidation:rule="!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting == true) || int(self.image.split(':')[1].split('.')[0] + self.image.split(':')[1].split('.')[1]) >= 111", message="HAProxy and Pathbased Routing is enabled. PathBasedRouting is only supported for MarkLogic 11.1 and above"
+// +kubebuilder:validation:XValidation:rule="!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting == true) || self.image.split(':')[1].matches('.*latest.*') || int(self.image.split(':')[1].split('.')[0] + self.image.split(':')[1].split('.')[1]) >= 111", message="HAProxy and Pathbased Routing is enabled. PathBasedRouting is only supported for MarkLogic 11.1 and above"
 type MarklogicClusterSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file

config/crd/bases/marklogic.progress.com_marklogicclusters.yaml

@@ -11037,7 +11037,8 @@ spec:
             - message: HAProxy and Pathbased Routing is enabled. PathBasedRouting
                 is only supported for MarkLogic 11.1 and above
               rule: '!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting
-                == true) || int(self.image.split('':'')[1].split(''.'')[0] + self.image.split('':'')[1].split(''.'')[1])
+                == true) || self.image.split('':'')[1].matches(''.*latest.*'') ||
+                int(self.image.split('':'')[1].split(''.'')[0] + self.image.split('':'')[1].split(''.'')[1])
                 >= 111'
           status:
             description: MarklogicClusterStatus defines the observed state of MarklogicCluster

pkg/k8sutil/scripts/liveness-probe.sh

@@ -53,6 +53,25 @@ log () {
     echo $message >> /tmp/script.log
 }
 
+# Function to retry a command based on the return code
+# $1: The number of retries
+# $2: The command to run
+retry() {
+    local retries=$1
+    shift
+    local count=0
+    until "$@"; do
+        exit_code=$?
+        count=$((count + 1))
+        if [ $count -ge $retries ]; then
+        echo "Command failed after $retries attempts."
+        return $exit_code
+        fi
+        echo "Attempt $count failed. Retrying..."
+        sleep 5
+    done
+}
+
 ###############################################################
 # Function to get the current host protocol
 # $1: The host name
@@ -699,10 +718,10 @@ if [[ "$IS_BOOTSTRAP_HOST" == "true" ]]; then
     if [[ "${MARKLOGIC_CLUSTER_TYPE}" == "bootstrap" ]]; then
         log "Info:  bootstrap host is ready"
         init_security_db
-        configure_group
+        retry 5 configure_group
     else 
         log "Info:  bootstrap host is ready"
-        configure_group
+        retry 5 configure_group
         join_cluster $HOST_FQDN
     fi
     configure_path_based_routing

pkg/k8sutil/scripts/poststart-hook.sh

@@ -696,8 +696,11 @@ func getLivenessProbe(probe marklogicv1.ContainerProbe) *corev1.Probe {
 		TimeoutSeconds:      probe.TimeoutSeconds,
 		SuccessThreshold:    probe.SuccessThreshold,
 		ProbeHandler: corev1.ProbeHandler{
-			Exec: &corev1.ExecAction{
-				Command: []string{"/bin/bash", "/tmp/helm-scripts/liveness-probe.sh"},
+			TCPSocket: &corev1.TCPSocketAction{
+				Port: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: 8001,
+				},
 			},
 		},
 	}

pkg/k8sutil/statefulset.go

@@ -134,14 +134,17 @@ func TestMarklogicCluster(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to install grafana helm chart: %v", err)
 		}
-
+		// Wait for Grafana pod to be ready
+		time.Sleep(5 * time.Second) // Give some time for Grafana to start
 		podList := &corev1.PodList{}
 		if err := client.Resources().List(ctx, podList, func(lo *metav1.ListOptions) {
 			lo.FieldSelector = "metadata.namespace=" + "grafana"
 		}); err != nil {
 			t.Fatal(err)
 		}
-
+		if len(podList.Items) == 0 {
+			t.Fatal("No Grafana pods found")
+		}
 		grafanaPodName := podList.Items[0].Name
 		err = utils.WaitForPod(ctx, t, client, "grafana", grafanaPodName, 120*time.Second)
 		if err != nil {
@@ -228,7 +231,7 @@ func TestMarklogicCluster(t *testing.T) {
 		client := c.Client()
 
 		podName := "node-0"
-		err := utils.WaitForPod(ctx, t, client, mlNamespace, podName, 180*time.Second)
+		err := utils.WaitForPod(ctx, t, client, mlNamespace, podName, 240*time.Second)
 		if err != nil {
 			t.Fatalf("Failed to wait for pod creation: %v", err)
 		}
@@ -245,6 +248,10 @@ func TestMarklogicCluster(t *testing.T) {
 		}); err != nil {
 			t.Fatal(err)
 		}
+		time.Sleep(5 * time.Second) // Wait for Grafana to be fully ready
+		if len(podList.Items) == 0 {
+			t.Fatal("No Grafana pods found")
+		}
 		grafanaPodName := podList.Items[0].Name
 		grafanaAdminUser, grafanaAdminPassword, err := utils.GetSecretData(ctx, client, "grafana", "grafana", "admin-user", "admin-password")
 		if err != nil {
@@ -337,7 +344,7 @@ func TestMarklogicCluster(t *testing.T) {
 			if err := client.Resources().Get(ctx, "marklogicclusters", mlNamespace, &mlcluster); err != nil {
 				t.Fatal(err)
 			}
-
+			
 			mlcluster.Spec.MarkLogicGroups[0].Resources = &resources
 			if err := client.Resources().Update(ctx, &mlcluster); err != nil {
 				t.Log("Failed to update MarkLogic group resources")

test/e2e/2_marklogic_cluster_test.go

@@ -240,6 +240,9 @@ func TestTlsWithNamedCert(t *testing.T) {
 		}
 		certURIs := gjson.Get(certs, `certificate-default-list.list-items.list-item.#.uriref`).Array()
 		t.Log("Certificates URL list", certURIs)
+		if len(certURIs) < 2 {
+			t.Fatalf("Expected at least 2 certificates, found %d", len(certURIs))
+		}
 		cert0Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[0])
 		cert1Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[1])
 		command = fmt.Sprintf("curl -k --anyauth -u %s:%s %s", adminUsername, adminPassword, cert0Url)
@@ -404,8 +407,12 @@ func TestTlsWithMultiNode(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to get certificates list: %v", err)
 		}
+		t.Log("Certificates list", certs)
 		certURIs := gjson.Get(certs, `certificate-default-list.list-items.list-item.#.uriref`).Array()
 		t.Log("Dnode Cert Url", certURIs)
+		if len(certURIs) < 2 {
+			t.Fatalf("Expected at least 2 certificates, found %d", len(certURIs))
+		}
 		cert0Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[0])
 		cert1Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[1])
 		command = fmt.Sprintf("curl -k --anyauth -u %s:%s %s", adminUsername, adminPassword, cert0Url)

test/e2e/4_tls_test.go

@@ -94,8 +94,8 @@ func TestMain(m *testing.M) {
 			log.Println("Deploying controller-manager resources...")
 			p := utils.RunCommand(`kubectl version`)
 			log.Printf("Output of kubectl: %s", p.Result())
-			p = utils.RunCommand(`bash -c "kustomize build config/default | kubectl apply --server-side -f -"`)
-			log.Printf("Output: %s", p.Result())
+			p = utils.RunCommand(`make deploy`)
+			log.Printf("Output of make deploy: %s", p.Result())
 			if p.Err() != nil {
 				log.Printf("Failed to deploy resource configurations: %s: %s", p.Err(), p.Result())
 				return ctx, p.Err()