[CLOUDP-352109] Run MCK E2E tests against OCI published helm chart

.evergreen.yml

@@ -37,6 +37,8 @@ variables:
         variant: init_test_run
       - name: build_init_om_images_ubi
         variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
 
   - &base_no_om_image_dependency
     depends_on:
@@ -54,6 +56,8 @@ variables:
         variant: init_test_run
       - name: build_init_appdb_images_ubi
         variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
 
   - &community_dependency
     depends_on:

docker/mongodb-kubernetes-tests/Dockerfile

@@ -45,7 +45,8 @@ RUN apt-get -qq update \
     libldap2-dev \
     libsasl2-dev \
     git \
-    openssl
+    openssl \
+    unzip
 
 RUN mkdir -p /tmp/mongodb-tools && \
     tar xfz /tmp/mongodb-tools.tgz -C /tmp/mongodb-tools && \
@@ -66,6 +67,11 @@ RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -
     && chmod +x ./kubectl \
     && mv ./kubectl /usr/local/bin/kubectl
 
+# install aws, required to run helm registry login while running the tests
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
+    && unzip -q awscliv2.zip \
+    && ./aws/install
+
 COPY --from=builder /venv /venv
 
 ENV PATH="/venv/bin:${PATH}"
@@ -75,8 +81,9 @@ WORKDIR /tests
 
 # copying the test files after python build, otherwise pip install will be called each time the tests change
 COPY . /tests
-# copying the helm_chart directory as well to support installation of the Operator from the test application
-COPY helm_chart /helm_chart
+# copying the helm_chart/crds directory so that we can install CRDs before installing the operator helm chart to run a test.
+# operator is installed via published OCI helm repo and not the local helm repo.
+COPY helm_chart/crds /helm_chart/crds
 COPY release.json /release.json
 # we use the public directory to automatically test resources samples
 COPY public /mongodb-kubernetes/public

docker/mongodb-kubernetes-tests/kubetester/consts.py

@@ -0,0 +1,8 @@
+OPERATOR_VERSION_ENV_VAR_NAME = "OPERATOR_VERSION"
+
+OCI_HELM_REGISTRY_ENV_VAR_NAME = "OCI_HELM_REGISTRY"
+OCI_HELM_REPOSITORY_ENV_VAR_NAME = "OCI_HELM_REPOSITORY"
+OCI_HELM_REGION_ENV_VAR_NAME = "OCI_HELM_REGION"
+
+LEGACY_OPERATOR_CHART = "mongodb/enterprise-operator"
+MCK_HELM_CHART = "mongodb/mongodb-kubernetes"

docker/mongodb-kubernetes-tests/kubetester/helm.py

@@ -6,10 +6,14 @@
 import uuid
 from typing import Dict, List, Optional, Tuple
 
+from kubetester.consts import *
 from tests import test_logger
 
 logger = test_logger.get_test_logger(__name__)
 
+# LOCAL_CRDs_DIR is the dir where local helm chart's CRDs are copied in tests image
+LOCAL_CRDs_DIR = "helm_chart/crds"
+
 
 def helm_template(
     helm_args: Dict,
@@ -145,6 +149,55 @@ def process_run_and_check(args, **kwargs):
         raise
 
 
+def oci_helm_registry_login(helm_registry: str, region: str):
+    logger.info(f"Attempting to log into ECR registry: {helm_registry}, using helm registry login.")
+
+    aws_command = ["aws", "ecr", "get-login-password", "--region", region]
+
+    # as we can see the password is being provided by stdin, that would mean we will have to
+    # pipe the aws_command (it figures out the password) into helm_command.
+    helm_command = ["helm", "registry", "login", "--username", "AWS", "--password-stdin", helm_registry]
+
+    try:
+        logger.info("Starting AWS ECR credential retrieval.")
+        aws_proc = subprocess.Popen(
+            aws_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True  # Treat input/output as text strings
+        )
+
+        logger.info("Starting Helm registry login.")
+        helm_proc = subprocess.Popen(
+            helm_command, stdin=aws_proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+        )
+
+        # Close the stdout stream of aws_proc in the parent process
+        # to prevent resource leakage (only needed if you plan to do more processing)
+        aws_proc.stdout.close()
+
+        # Wait for the Helm command (helm_proc) to finish and capture its output
+        helm_stdout, helm_stderr = helm_proc.communicate()
+
+        # Wait for the AWS process to finish as well
+        aws_proc.wait()
+
+        if aws_proc.returncode != 0:
+            _, aws_stderr = aws_proc.communicate()
+            raise Exception(f"aws command to get password failed. Error: {aws_stderr}")
+
+        if helm_proc.returncode == 0:
+            logger.info("Login to helm registry was successful.")
+            logger.info(helm_stdout.strip())
+        else:
+            raise Exception(
+                f"Login to helm registry failed, Exit code: {helm_proc.returncode}, Error: {helm_stderr.strip()}"
+            )
+
+    except FileNotFoundError as e:
+        # This catches errors if 'aws' or 'helm' are not in the PATH
+        raise Exception(f"Command not found. Please ensure '{e.filename}' is installed and in your system's PATH.")
+    except Exception as e:
+        raise Exception(f"An unexpected error occurred: {e}.")
+
+
 def helm_upgrade(
     name: str,
     namespace: str,
@@ -162,7 +215,7 @@ def helm_upgrade(
     chart_dir = helm_chart_path if helm_override_path else _helm_chart_dir(helm_chart_path)
 
     if apply_crds_first:
-        apply_crds_from_chart(chart_dir)
+        apply_crds_from_chart(LOCAL_CRDs_DIR)
 
     command_args = _create_helm_args(helm_args, helm_options)
     args = [
@@ -183,11 +236,24 @@ def helm_upgrade(
     process_run_and_check(command, check=True, capture_output=True, shell=True)
 
 
-def apply_crds_from_chart(chart_dir: str):
-    crd_files = glob.glob(os.path.join(chart_dir, "crds", "*.yaml"))
+# oci_chart_info returns the respective registry/repo and region information
+# based on the build scenario (dev/staging) tests are being run in. These are
+# read from build_info.json and then set to the tests image as env vars.
+def oci_chart_info():
+    registry = os.environ.get(OCI_HELM_REGISTRY_ENV_VAR_NAME)
+    repository = os.environ.get(OCI_HELM_REPOSITORY_ENV_VAR_NAME)
+    region = os.environ.get(OCI_HELM_REGION_ENV_VAR_NAME)
+
+    logger.info(f"oci chart details in test image is registry {registry}, repo {repository}, region {region}")
+
+    return registry, f"{repository}/mongodb-kubernetes", region
+
+
+def apply_crds_from_chart(crds_dir: str):
+    crd_files = glob.glob(os.path.join(crds_dir, "*.yaml"))
 
     if not crd_files:
-        raise Exception(f"No CRD files found in chart directory: {chart_dir}")
+        raise Exception(f"No CRD files found in chart directory: {crds_dir}")
 
     logger.info(f"Found {len(crd_files)} CRD files to apply:")
 
@@ -234,3 +300,43 @@ def _create_helm_args(helm_args: Dict[str, str], helm_options: Optional[List[str
 
 def _helm_chart_dir(default: Optional[str] = "helm_chart") -> str:
     return os.environ.get("HELM_CHART_DIR", default)
+
+
+# helm_chart_path_and_version returns the chart path and version that we would like to install to run the E2E tests.
+# for local tests it returns early with local helm chart dir and for other scenarios it figure out the chart and version
+# based on the caller. In most of the cases we will install chart from OCI registry but for the tests where we would like
+# to install MEKO's specific version or MCK's specific version, we would expec `helm_chart_path` to set already.
+def helm_chart_path_and_version(helm_chart_path: str, operator_version: str) -> tuple[str, str]:
+    # these are imported here to resolve import cycle issue
+    from tests.conftest import LOCAL_HELM_CHART_DIR, local_operator
+
+    if local_operator():
+        return LOCAL_HELM_CHART_DIR, ""
+
+    # if operator_version is not specified and we are not installing the MCK or MEKO chart
+    # it would mean we want to install OCI published helm chart. Figure out respective version,
+    # it is set in env var `OPERATOR_VERSION` based on build_scenario.
+    if not operator_version and helm_chart_path not in (
+        MCK_HELM_CHART,
+        LEGACY_OPERATOR_CHART,
+    ):
+        non_semver_operator_version = os.environ.get(OPERATOR_VERSION_ENV_VAR_NAME)
+        # when we publish the helm chart we append `0.0.0+` in the chart version, details are
+        # here https://docs.google.com/document/d/1eJ8iKsI0libbpcJakGjxcPfbrTn8lmcZDbQH1UqMR_g/edit?tab=t.gg5ble8qlesq
+        operator_version = f"0.0.0+{non_semver_operator_version}"
+
+    # helm_chart_path not being passed would mean we are on evg env and would like to
+    # install helm chart from OCI registry.
+    if not helm_chart_path:
+        # login to the OCI container registry
+        registry, repository, region = oci_chart_info()
+        try:
+            oci_helm_registry_login(registry, region)
+        except Exception as e:
+            raise e
+
+        # figure out the registry URI, based on dev/staging scenario
+        chart_uri = f"oci://{registry}/{repository}"
+        helm_chart_path = chart_uri
+
+    return helm_chart_path, operator_version

docker/mongodb-kubernetes-tests/kubetester/operator.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import time
 from typing import Dict, List, Optional
 
@@ -9,13 +10,17 @@
 from kubernetes.client import V1CustomResourceDefinition, V1Deployment, V1Pod
 from kubernetes.client.rest import ApiException
 from kubetester import wait_for_webhook
+from kubetester.consts import *
 from kubetester.create_or_replace_from_yaml import create_or_replace_from_yaml
 from kubetester.helm import (
+    helm_chart_path_and_version,
     helm_install,
     helm_repo_add,
     helm_template,
     helm_uninstall,
     helm_upgrade,
+    oci_chart_info,
+    oci_helm_registry_login,
 )
 from tests import test_logger
 
@@ -25,6 +30,7 @@
     "opsmanagers.mongodb.com",
 )
 
+
 logger = test_logger.get_test_logger(__name__)
 
 
@@ -43,14 +49,16 @@ def __init__(
         namespace: str,
         helm_args: Optional[Dict] = None,
         helm_options: Optional[List[str]] = None,
-        helm_chart_path: Optional[str] = "helm_chart",
+        helm_chart_path: Optional[str] = None,
         name: Optional[str] = "mongodb-kubernetes-operator",
         api_client: Optional[client.api_client.ApiClient] = None,
+        operator_version: Optional[str] = None,
     ):
-
         # The Operator will be installed from the following repo, so adding it first
         helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts")
 
+        helm_chart_path, operator_version = helm_chart_path_and_version(helm_chart_path, operator_version)
+
         if helm_args is None:
             helm_args = {}
 
@@ -69,6 +77,7 @@ def __init__(
         self.helm_chart_path = helm_chart_path
         self.name = name
         self.api_client = api_client
+        self.operator_version = operator_version
 
     def install_from_template(self):
         """Uses helm to generate yaml specification and then uses python K8s client to apply them to the cluster
@@ -82,6 +91,9 @@ def install_from_template(self):
 
     def install(self, custom_operator_version: Optional[str] = None) -> Operator:
         """Installs the Operator to Kubernetes cluster using 'helm install', waits until it's running"""
+        if not custom_operator_version:
+            custom_operator_version = self.operator_version
+
         helm_install(
             self.name,
             self.namespace,
@@ -99,6 +111,9 @@ def upgrade(
         self, multi_cluster: bool = False, custom_operator_version: Optional[str] = None, apply_crds_first: bool = False
     ) -> Operator:
         """Upgrades the Operator in Kubernetes cluster using 'helm upgrade', waits until it's running"""
+        if not custom_operator_version:
+            custom_operator_version = self.operator_version
+
         helm_upgrade(
             self.name,
             self.namespace,

docker/mongodb-kubernetes-tests/tests/conftest.py

@@ -20,7 +20,14 @@
     update_configmap,
 )
 from kubetester.awss3client import AwsS3Client
-from kubetester.helm import helm_install_from_chart, helm_repo_add
+from kubetester.consts import *
+from kubetester.helm import (
+    helm_chart_path_and_version,
+    helm_install_from_chart,
+    helm_repo_add,
+    oci_chart_info,
+    oci_helm_registry_login,
+)
 from kubetester.kubetester import KubernetesTester
 from kubetester.kubetester import fixture as _fixture
 from kubetester.kubetester import running_locally
@@ -55,9 +62,6 @@
 LEGACY_CENTRAL_CLUSTER_NAME: str = "__default"
 LEGACY_DEPLOYMENT_STATE_VERSION: str = "1.27.0"
 
-# Helm charts
-LEGACY_OPERATOR_CHART = "mongodb/enterprise-operator"
-MCK_HELM_CHART = "mongodb/mongodb-kubernetes"
 LOCAL_HELM_CHART_DIR = "helm_chart"
 
 OFFICIAL_OPERATOR_IMAGE_NAME = "mongodb-kubernetes"
@@ -827,7 +831,7 @@ def _install_multi_cluster_operator(
     helm_opts: dict[str, str],
     central_cluster_name: str,
     operator_name: Optional[str] = MULTI_CLUSTER_OPERATOR_NAME,
-    helm_chart_path: Optional[str] = LOCAL_HELM_CHART_DIR,
+    helm_chart_path: Optional[str] = None,
     custom_operator_version: Optional[str] = None,
     apply_crds_first: bool = False,
 ) -> Operator:
@@ -836,6 +840,8 @@ def _install_multi_cluster_operator(
     # The Operator will be installed from the following repo, so adding it first
     helm_repo_add("mongodb", "https://mongodb.github.io/helm-charts")
 
+    helm_chart_path, custom_operator_version = helm_chart_path_and_version(helm_chart_path, custom_operator_version)
+
     prepare_multi_cluster_namespaces(
         namespace,
         multi_cluster_operator_installation_config,

scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml

@@ -174,6 +174,22 @@ spec:
     - name: OM_DEBUG_HTTP
       value: "{{ .Values.omDebugHttp }}"
     {{ end }}
+    {{ if .Values.helm.oci.registry }}
+    - name: OCI_HELM_REGISTRY
+      value: "{{ .Values.helm.oci.registry }}"
+    {{ end }}
+    {{ if .Values.operator.version }}
+    - name: OPERATOR_VERSION
+      value: "{{ .Values.operator.version }}"
+    {{ end }}
+    {{ if .Values.helm.oci.repository }}
+    - name: OCI_HELM_REPOSITORY
+      value: "{{ .Values.helm.oci.repository }}"
+    {{ end }}
+    {{ if .Values.helm.oci.region }}
+    - name: OCI_HELM_REGION
+      value: "{{ .Values.helm.oci.region }}"
+    {{ end }}
     - name: ops_manager_version
       value: "{{ .Values.opsManagerVersion }}"
     - name: cognito_user_pool_id

scripts/evergreen/deployments/test-app/values.yaml

@@ -43,3 +43,14 @@ mdbImageType: "ubi8"
 
 # set to "true" to set OM_DEBUG_HTTP=true for the operator
 omDebugHttp:
+
+# sets the configuration using which we can figure out the helm OCI repo of the MCK operator while deploying it
+# to run a test.
+helm:
+  oci:
+    registry: ""
+    repository: ""
+    region: ""
+
+operator:
+  version: ""