diff --git a/Jenkinsfile b/Jenkinsfile index 681dbd3..41b46ce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -164,7 +164,8 @@ pipeline { // } parameters { - string(name: 'dockerImage', defaultValue: 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11', description: 'Docker image to use for tests.', trim: true) + string(name: 'E2E_MARKLOGIC_IMAGE_VERSION', defaultValue: 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless:latest-12', description: 'Docker image to use for tests.', trim: true) + string(name: 'IMG', defaultValue: 'testrepo/marklogic-operator-image-dev:internal', description: 'Docker image for Running Operator Container', trim: true) string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true) } diff --git a/Makefile b/Makefile index 156b14e..de1ee0c 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ VERIFY_HUGE_PAGES ?= false export E2E_DOCKER_IMAGE ?= $(IMG) export E2E_KUSTOMIZE_VERSION ?= $(KUSTOMIZE_VERSION) export E2E_CONTROLLER_TOOLS_VERSION ?= $(CONTROLLER_TOOLS_VERSION) -export E2E_MARKLOGIC_IMAGE_VERSION ?= progressofficial/marklogic-db:11.3.1-ubi-rootless-2.1.0 +export E2E_MARKLOGIC_IMAGE_VERSION ?= progressofficial/marklogic-db:11.3.1-ubi-rootless-2.1.3 export E2E_KUBERNETES_VERSION ?= v1.31.0 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. @@ -65,7 +65,8 @@ OPERATOR_SDK_VERSION ?= v1.34.2 # Image URL to use all building/pushing image targets # Image for dev: ml-marklogic-operator-dev.bed-artifactory.bedford.progress.com/marklogic-operator-kubernetes -IMG ?= progressofficial/marklogic-operator-kubernetes:$(VERSION) +# IMG ?= progressofficial/marklogic-operator-kubernetes:$(VERSION) +IMG = "testrepo/marklogic-operator-image-dev:1.0.0" # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) @@ -162,6 +163,9 @@ e2e-setup-minikube: kustomize controller-gen build docker-build minikube start --driver=docker --kubernetes-version=$(E2E_KUBERNETES_VERSION) --memory=8192 --cpus=2 minikube addons enable ingress minikube image load $(IMG) + minikube image load $(E2E_MARKLOGIC_IMAGE_VERSION) + minikube image load "docker.io/haproxytech/haproxy-alpine:3.2" + minikube image ls .PHONY: e2e-cleanup-minikube e2e-cleanup-minikube: diff --git a/api/v1/marklogiccluster_types.go b/api/v1/marklogiccluster_types.go index d213666..5453f7b 100644 --- a/api/v1/marklogiccluster_types.go +++ b/api/v1/marklogiccluster_types.go @@ -27,7 +27,7 @@ import ( // MarklogicClusterSpec defines the desired state of MarklogicCluster -// +kubebuilder:validation:XValidation:rule="!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting == true) || int(self.image.split(':')[1].split('.')[0] + self.image.split(':')[1].split('.')[1]) >= 111", message="HAProxy and Pathbased Routing is enabled. PathBasedRouting is only supported for MarkLogic 11.1 and above" +// +kubebuilder:validation:XValidation:rule="!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting == true) || self.image.split(':')[1].matches('.*latest.*') || int(self.image.split(':')[1].split('.')[0] + self.image.split(':')[1].split('.')[1]) >= 111", message="HAProxy and Pathbased Routing is enabled. PathBasedRouting is only supported for MarkLogic 11.1 and above" type MarklogicClusterSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file diff --git a/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml b/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml index c5b9d03..8eee952 100644 --- a/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml +++ b/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml @@ -11037,7 +11037,8 @@ spec: - message: HAProxy and Pathbased Routing is enabled. PathBasedRouting is only supported for MarkLogic 11.1 and above rule: '!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting - == true) || int(self.image.split('':'')[1].split(''.'')[0] + self.image.split('':'')[1].split(''.'')[1]) + == true) || self.image.split('':'')[1].matches(''.*latest.*'') || + int(self.image.split('':'')[1].split(''.'')[0] + self.image.split('':'')[1].split(''.'')[1]) >= 111' status: description: MarklogicClusterStatus defines the observed state of MarklogicCluster diff --git a/pkg/k8sutil/scripts/liveness-probe.sh b/pkg/k8sutil/scripts/liveness-probe.sh deleted file mode 100644 index 388d426..0000000 --- a/pkg/k8sutil/scripts/liveness-probe.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -log () { - local TIMESTAMP=$(date +"%Y-%m-%d %T.%3N") - # Check to make sure pod doesn't terminate if PID value is empty for any reason - if [ -n "$pid" ]; then - echo "${TIMESTAMP} $@" > /proc/$pid/fd/1 - fi -} - -pid=$(pgrep -fn start.marklogic) - -# Check if ML service is running. Exit with 1 if it is other than running -ml_status=$(/etc/init.d/MarkLogic status) - -if [[ "$ml_status" =~ "running" ]]; then - http_code=$(curl -o /tmp/probe_response.txt -s -w "%{http_code}" "http://${HOSTNAME}:8001/admin/v1/timestamp") - curl_code=$? - http_resp=$(cat /tmp/probe_response.txt) - - if [[ $curl_code -ne 0 && $http_code -ne 401 ]]; then - log "Info: [Liveness Probe] Error with MarkLogic" - log "Info: [Liveness Probe] Curl response code: "$curl_code - log "Info: [Liveness Probe] Http response code: "$http_code - log "Info: [Liveness Probe] Http response message: "$http_resp - fi - rm -f /tmp/probe_response.txt - exit 0 -else - exit 1 -fi \ No newline at end of file diff --git a/pkg/k8sutil/scripts/poststart-hook.sh b/pkg/k8sutil/scripts/poststart-hook.sh index 0cbac3d..3abedba 100644 --- a/pkg/k8sutil/scripts/poststart-hook.sh +++ b/pkg/k8sutil/scripts/poststart-hook.sh @@ -53,6 +53,25 @@ log () { echo $message >> /tmp/script.log } +# Function to retry a command based on the return code +# $1: The number of retries +# $2: The command to run +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit_code=$? + count=$((count + 1)) + if [ $count -ge $retries ]; then + echo "Command failed after $retries attempts." + return $exit_code + fi + echo "Attempt $count failed. Retrying..." + sleep 5 + done +} + ############################################################### # Function to get the current host protocol # $1: The host name @@ -699,10 +718,10 @@ if [[ "$IS_BOOTSTRAP_HOST" == "true" ]]; then if [[ "${MARKLOGIC_CLUSTER_TYPE}" == "bootstrap" ]]; then log "Info: bootstrap host is ready" init_security_db - configure_group + retry 5 configure_group else log "Info: bootstrap host is ready" - configure_group + retry 5 configure_group join_cluster $HOST_FQDN fi configure_path_based_routing diff --git a/pkg/k8sutil/statefulset.go b/pkg/k8sutil/statefulset.go index 8c690f3..080b805 100644 --- a/pkg/k8sutil/statefulset.go +++ b/pkg/k8sutil/statefulset.go @@ -696,8 +696,11 @@ func getLivenessProbe(probe marklogicv1.ContainerProbe) *corev1.Probe { TimeoutSeconds: probe.TimeoutSeconds, SuccessThreshold: probe.SuccessThreshold, ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{ - Command: []string{"/bin/bash", "/tmp/helm-scripts/liveness-probe.sh"}, + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 8001, + }, }, }, } diff --git a/test/e2e/2_marklogic_cluster_test.go b/test/e2e/2_marklogic_cluster_test.go index a8e34fc..4e3986e 100644 --- a/test/e2e/2_marklogic_cluster_test.go +++ b/test/e2e/2_marklogic_cluster_test.go @@ -134,14 +134,17 @@ func TestMarklogicCluster(t *testing.T) { if err != nil { t.Fatalf("Failed to install grafana helm chart: %v", err) } - + // Wait for Grafana pod to be ready + time.Sleep(5 * time.Second) // Give some time for Grafana to start podList := &corev1.PodList{} if err := client.Resources().List(ctx, podList, func(lo *metav1.ListOptions) { lo.FieldSelector = "metadata.namespace=" + "grafana" }); err != nil { t.Fatal(err) } - + if len(podList.Items) == 0 { + t.Fatal("No Grafana pods found") + } grafanaPodName := podList.Items[0].Name err = utils.WaitForPod(ctx, t, client, "grafana", grafanaPodName, 120*time.Second) if err != nil { @@ -228,7 +231,7 @@ func TestMarklogicCluster(t *testing.T) { client := c.Client() podName := "node-0" - err := utils.WaitForPod(ctx, t, client, mlNamespace, podName, 180*time.Second) + err := utils.WaitForPod(ctx, t, client, mlNamespace, podName, 240*time.Second) if err != nil { t.Fatalf("Failed to wait for pod creation: %v", err) } @@ -245,6 +248,10 @@ func TestMarklogicCluster(t *testing.T) { }); err != nil { t.Fatal(err) } + time.Sleep(5 * time.Second) // Wait for Grafana to be fully ready + if len(podList.Items) == 0 { + t.Fatal("No Grafana pods found") + } grafanaPodName := podList.Items[0].Name grafanaAdminUser, grafanaAdminPassword, err := utils.GetSecretData(ctx, client, "grafana", "grafana", "admin-user", "admin-password") if err != nil { @@ -337,7 +344,7 @@ func TestMarklogicCluster(t *testing.T) { if err := client.Resources().Get(ctx, "marklogicclusters", mlNamespace, &mlcluster); err != nil { t.Fatal(err) } - + mlcluster.Spec.MarkLogicGroups[0].Resources = &resources if err := client.Resources().Update(ctx, &mlcluster); err != nil { t.Log("Failed to update MarkLogic group resources") diff --git a/test/e2e/4_tls_test.go b/test/e2e/4_tls_test.go index a645313..b1c564e 100644 --- a/test/e2e/4_tls_test.go +++ b/test/e2e/4_tls_test.go @@ -240,6 +240,9 @@ func TestTlsWithNamedCert(t *testing.T) { } certURIs := gjson.Get(certs, `certificate-default-list.list-items.list-item.#.uriref`).Array() t.Log("Certificates URL list", certURIs) + if len(certURIs) < 2 { + t.Fatalf("Expected at least 2 certificates, found %d", len(certURIs)) + } cert0Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[0]) cert1Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[1]) command = fmt.Sprintf("curl -k --anyauth -u %s:%s %s", adminUsername, adminPassword, cert0Url) @@ -404,8 +407,12 @@ func TestTlsWithMultiNode(t *testing.T) { if err != nil { t.Fatalf("Failed to get certificates list: %v", err) } + t.Log("Certificates list", certs) certURIs := gjson.Get(certs, `certificate-default-list.list-items.list-item.#.uriref`).Array() t.Log("Dnode Cert Url", certURIs) + if len(certURIs) < 2 { + t.Fatalf("Expected at least 2 certificates, found %d", len(certURIs)) + } cert0Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[0]) cert1Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[1]) command = fmt.Sprintf("curl -k --anyauth -u %s:%s %s", adminUsername, adminPassword, cert0Url) diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 4af29f9..29be6b7 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -94,8 +94,8 @@ func TestMain(m *testing.M) { log.Println("Deploying controller-manager resources...") p := utils.RunCommand(`kubectl version`) log.Printf("Output of kubectl: %s", p.Result()) - p = utils.RunCommand(`bash -c "kustomize build config/default | kubectl apply --server-side -f -"`) - log.Printf("Output: %s", p.Result()) + p = utils.RunCommand(`make deploy`) + log.Printf("Output of make deploy: %s", p.Result()) if p.Err() != nil { log.Printf("Failed to deploy resource configurations: %s: %s", p.Err(), p.Result()) return ctx, p.Err() diff --git a/test/utils/utils.go b/test/utils/utils.go index f01e627..787c624 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -154,6 +154,9 @@ func WaitForPod(ctx context.Context, t *testing.T, client klient.Client, namespa pod := &corev1.Pod{} p := utils.RunCommand(`kubectl get ns`) t.Logf("Kubernetes namespace: %s", p.Result()) + p = utils.RunCommand("kubectl get pods --namespace " + "marklogic-operator-system" + " -o wide") + t.Logf("Kubernetes Operator Running Status: %s", p.Result()) + for { t.Logf("Waiting for pod %s in namespace %s ", podName, namespace) p := utils.RunCommand("kubectl get pods --namespace " + namespace)