CLOUDP-230405: Add appDB multi-cluster member config (#3725)

# Summary

Based on 10gen/ops-manager-kubernetes#3447 ,
this patch adds multi-cluster support for appdb member configuration.

This also requires a patch in the community operator:
mongodb/mongodb-kubernetes-operator#1610

## Documentation changes

* [ ] Add an entry to [release notes](.../RELEASE_NOTES.md).
* [ ] When needed, make sure you create a new [DOCSP
ticket](https://jira.mongodb.org/projects/DOCSP) that documents your
change.

## Changes to CRDs

* [ ] Add `slaskawi`(Sebastian) and `@giohan` (George) as reviewers.
* [ ] Make sure any changes are reflected on `/public/samples`
directory.

Author: mircea-cosbuc

.evergreen.yml

@@ -1534,7 +1534,7 @@ tasks:
     - func: "e2e_test"
 
 # E2E tests for Ops Manager (sorted alphabetically):
-- name: e2e_om_appdb_agent_flags
+- name: e2e_om_appdb_flags_and_config
   tags: ["patch-run"]
   commands:
     - func: "e2e_test"
@@ -2293,7 +2293,7 @@ task_groups:
     - e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure
     - e2e_multi_cluster_om_networking_clusterwide
     # Reused OM tests with AppDB Multi-Cluster topology
-    - e2e_om_appdb_agent_flags
+    - e2e_om_appdb_flags_and_config
     - e2e_om_appdb_upgrade
     - e2e_om_appdb_monitoring_tls
     - e2e_om_ops_manager_backup
@@ -2343,7 +2343,7 @@ task_groups:
     - e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure
     - e2e_multi_cluster_om_networking_clusterwide
     # Reused OM tests with AppDB Multi-Cluster topology
-    - e2e_om_appdb_agent_flags
+    - e2e_om_appdb_flags_and_config
     - e2e_om_appdb_upgrade
     - e2e_om_appdb_monitoring_tls
     - e2e_om_ops_manager_backup
@@ -2395,7 +2395,7 @@ task_groups:
   <<: *setup_group
   <<: *setup_and_teardown_task_cloudqa
   tasks:
-    - e2e_om_appdb_agent_flags
+    - e2e_om_appdb_flags_and_config
     - e2e_om_appdb_monitoring_tls
     - e2e_om_appdb_multi_change
     - e2e_om_appdb_scale_up_down
@@ -2439,7 +2439,7 @@ task_groups:
   <<: *setup_group
   <<: *setup_and_teardown_task_cloudqa
   tasks:
-    - e2e_om_appdb_agent_flags
+    - e2e_om_appdb_flags_and_config
     - e2e_om_appdb_monitoring_tls
     - e2e_om_appdb_multi_change
     - e2e_om_appdb_scale_up_down

api/v1/om/appdb_types.go

@@ -530,8 +530,9 @@ func (m *AppDBSpec) GetClusterSpecList() []mdbv1.ClusterSpecItem {
 	} else {
 		return []mdbv1.ClusterSpecItem{
 			{
-				ClusterName: multicluster.LegacyCentralClusterName,
-				Members:     m.Members,
+				ClusterName:  multicluster.LegacyCentralClusterName,
+				Members:      m.Members,
+				MemberConfig: m.GetMemberOptions(),
 			},
 		}
 	}

controllers/operator/appdbreplicaset_controller.go

@@ -46,6 +46,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
+
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
@@ -73,6 +74,7 @@ import (
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset"
 	"go.uber.org/zap"
+	"k8s.io/utils/ptr"
 )
 
 type agentType string
@@ -647,7 +649,7 @@ func (r *ReconcileAppDbReplicaSet) ReconcileAppDB(ctx context.Context, opsManage
 		// this doesn't requeue the reconciliation immediately, the calling OM controller
 		// requeues after Ops Manager has been fully configured.
 		log.Infof("Requeuing reconciliation to configure Monitoring in Ops Manager.")
-		// FIXME: use correct MembersOption for scaler
+
 		return r.updateStatus(ctx, opsManager, workflow.Pending("Enabling monitoring").Requeue(), log, appDbStatusOption, status.AppDBMemberOptions(appDBScalers...))
 	}
 
@@ -990,12 +992,9 @@ func (r *ReconcileAppDbReplicaSet) buildAppDbAutomationConfig(ctx context.Contex
 
 	}
 
-	// get member options from appDB spec
-	appDBSpec := opsManager.Spec.AppDB
-	memberOptions := appDBSpec.GetMemberOptions()
-
 	processList := r.generateProcessList(opsManager)
-	existingAutomationMemberIds, nextId := getExistingAutomationMemberIds(existingAutomationConfig)
+	existingAutomationMembers, nextId := getExistingAutomationReplicaSetMembers(existingAutomationConfig)
+	memberOptions := r.generateMemberOptions(opsManager, existingAutomationMembers)
 	replicasThisReconciliation := 0
 	// we want to use all member clusters to maintain the same process list despite having some clusters down
 	for _, memberCluster := range r.getAllMemberClusters() {
@@ -1075,8 +1074,8 @@ func (r *ReconcileAppDbReplicaSet) buildAppDbAutomationConfig(ctx context.Contex
 		AddModifications(func(automationConfig *automationconfig.AutomationConfig) {
 			if len(automationConfig.ReplicaSets) == 1 {
 				for idx, member := range automationConfig.ReplicaSets[0].Members {
-					if existingId, ok := existingAutomationMemberIds[member.Host]; ok {
-						automationConfig.ReplicaSets[0].Members[idx].Id = existingId
+					if existingMember, ok := existingAutomationMembers[member.Host]; ok {
+						automationConfig.ReplicaSets[0].Members[idx].Id = existingMember.Id
 					} else {
 						automationConfig.ReplicaSets[0].Members[idx].Id = nextId
 						nextId = nextId + 1
@@ -1145,48 +1144,95 @@ func shouldPerformForcedReconfigure(annotations map[string]string) bool {
 	return false
 }
 
-func getExistingAutomationMemberIds(automationConfig automationconfig.AutomationConfig) (map[string]int, int) {
+func getExistingAutomationReplicaSetMembers(automationConfig automationconfig.AutomationConfig) (map[string]automationconfig.ReplicaSetMember, int) {
 	nextId := 0
-	existingIds := map[string]int{}
+	existingMembers := map[string]automationconfig.ReplicaSetMember{}
 	if len(automationConfig.ReplicaSets) != 1 {
-		return existingIds, nextId
+		return existingMembers, nextId
 	}
 	for _, member := range automationConfig.ReplicaSets[0].Members {
-		existingIds[member.Host] = member.Id
+		existingMembers[member.Host] = member
 		if member.Id >= nextId {
 			nextId = member.Id + 1
 		}
 	}
-	return existingIds, nextId
+	return existingMembers, nextId
+}
+
+func (r *ReconcileAppDbReplicaSet) generateProcessHostnames(opsManager *omv1.MongoDBOpsManager, memberCluster multicluster.MemberCluster) []string {
+	members := scale.ReplicasThisReconciliation(scalers.GetAppDBScaler(opsManager, memberCluster.Name, r.getMemberClusterIndex(memberCluster.Name), r.memberClusters))
+	var hostnames []string
+	if opsManager.Spec.AppDB.IsMultiCluster() {
+		hostnames = dns.GetMultiClusterProcessHostnames(opsManager.Spec.AppDB.GetName(), opsManager.GetNamespace(), memberCluster.Index, members, opsManager.Spec.GetClusterDomain(), nil)
+	} else {
+		hostnames, _ = dns.GetDNSNames(opsManager.Spec.AppDB.GetName(), opsManager.Spec.AppDB.ServiceName(), opsManager.GetNamespace(), opsManager.Spec.GetClusterDomain(), members, nil)
+	}
+	return hostnames
 }
 
 func (r *ReconcileAppDbReplicaSet) generateProcessList(opsManager *omv1.MongoDBOpsManager) []automationconfig.Process {
 	var processList []automationconfig.Process
 	// We want all clusters to generate stable process list in case of some clusters being down. Process list cannot change regardless of the cluster health.
 	for _, memberCluster := range r.getAllMemberClusters() {
-		members := scale.ReplicasThisReconciliation(scalers.GetAppDBScaler(opsManager, memberCluster.Name, r.getMemberClusterIndex(memberCluster.Name), r.memberClusters))
-		var hostnames []string
-		if opsManager.Spec.AppDB.IsMultiCluster() {
-			hostnames = dns.GetMultiClusterProcessHostnames(opsManager.Spec.AppDB.GetName(), opsManager.GetNamespace(), memberCluster.Index, members, opsManager.Spec.GetClusterDomain(), nil)
-		} else {
-			hostnames, _ = dns.GetDNSNames(opsManager.Spec.AppDB.GetName(), opsManager.Spec.AppDB.ServiceName(), opsManager.GetNamespace(), opsManager.Spec.GetClusterDomain(), members, nil)
-		}
-
+		hostnames := r.generateProcessHostnames(opsManager, memberCluster)
 		for idx, hostname := range hostnames {
-			processList = append(processList, automationconfig.Process{
+			process := automationconfig.Process{
 				Name:     fmt.Sprintf("%s-%d", opsManager.Spec.AppDB.NameForCluster(memberCluster.Index), idx),
 				HostName: hostname,
-			})
+			}
+			processList = append(processList, process)
 		}
 	}
 	return processList
 }
 
+func (r *ReconcileAppDbReplicaSet) generateMemberOptions(opsManager *omv1.MongoDBOpsManager, previousMembers map[string]automationconfig.ReplicaSetMember) []automationconfig.MemberOptions {
+	var memberOptionsList []automationconfig.MemberOptions
+	for _, memberCluster := range r.getAllMemberClusters() {
+		hostnames := r.generateProcessHostnames(opsManager, memberCluster)
+		memberConfig := make([]automationconfig.MemberOptions, 0)
+		if memberCluster.Active {
+			memberConfigForCluster := opsManager.Spec.AppDB.GetMemberClusterSpecByName(memberCluster.Name).MemberConfig
+			if memberConfigForCluster != nil {
+				memberConfig = append(memberConfig, memberConfigForCluster...)
+			}
+		}
+		for idx, hostname := range hostnames {
+			memberOptions := automationconfig.MemberOptions{}
+			if idx < len(memberConfig) { // There are member options configured in the spec
+				memberOptions.Votes = memberConfig[idx].Votes
+				memberOptions.Priority = memberConfig[idx].Priority
+				memberOptions.Tags = memberConfig[idx].Tags
+			} else {
+				// There are three cases we might not have memberOptions in spec:
+				//   1. user never specified member config in the spec
+				//   2. user scaled down members e.g. from 5 to 2 removing memberConfig elements at the same time
+				//   3. user removed whole clusterSpecItem from the list (removing cluster entirely)
+				// For 2. and 3. we should have those members in existing AC
+				if replicaSetMember, ok := previousMembers[hostname]; ok {
+					memberOptions.Votes = replicaSetMember.Votes
+					if replicaSetMember.Priority != nil {
+						memberOptions.Priority = ptr.To(fmt.Sprintf("%f", *replicaSetMember.Priority))
+					}
+					memberOptions.Tags = replicaSetMember.Tags
+
+				} else {
+					// If the member does not exist in the previous automation config, we populate the member options with defaults
+					memberOptions.Votes = ptr.To(1)
+					memberOptions.Priority = ptr.To("1.0")
+				}
+			}
+			memberOptionsList = append(memberOptionsList, memberOptions)
+		}
+
+	}
+	return memberOptionsList
+}
+
 func (r *ReconcileAppDbReplicaSet) generateHeadlessHostnamesForMonitoring(opsManager *omv1.MongoDBOpsManager) []string {
 	var hostnames []string
 	// We want all clusters to generate stable process list in case of some clusters being down. Process list cannot change regardless of the cluster health.
 	for _, memberCluster := range r.getAllMemberClusters() {
-		// TODO for now scaling is disabled - we create all desired processes
 		members := scale.ReplicasThisReconciliation(scalers.GetAppDBScaler(opsManager, memberCluster.Name, r.getMemberClusterIndex(memberCluster.Name), r.memberClusters))
 		if opsManager.Spec.AppDB.IsMultiCluster() {
 			hostnames = append(hostnames, dns.GetMultiClusterHostnamesForMonitoring(opsManager.Spec.AppDB.GetName(), opsManager.GetNamespace(), memberCluster.Index, members)...)

docker/mongodb-enterprise-tests/kubetester/automation_config_tester.py

@@ -26,7 +26,7 @@ def get_replica_set_processes(self, rs_name: str) -> List[Dict]:
 
     def get_replica_set_members(self, rs_name: str) -> List[Dict]:
         replica_set = ([rs for rs in self.automation_config["replicaSets"] if rs["_id"] == rs_name])[0]
-        return replica_set["members"]
+        return sorted(replica_set["members"], key=lambda member: member["_id"])
 
     def get_mongos_processes(self):
         """ " Returns all mongos processes in deployment. We don't need to filter by sharded cluster name as

docker/mongodb-enterprise-tests/tests/multicluster/conftest.py

@@ -256,7 +256,7 @@ def create_service_entries_objects(
 def cluster_spec_list(
     member_cluster_names: List[str],
     members: List[int],
-    member_configs: Optional[List[Dict]] = None,
+    member_configs: Optional[List[List[Dict]]] = None,
     backup_configs: Optional[List[Dict]] = None,
 ):
     if member_configs is None and backup_configs is None:

docker/mongodb-enterprise-tests/tests/opsmanager/withMonitoredAppDB/conftest.py

@@ -34,6 +34,7 @@ def enable_multi_cluster_deployment(
     resource: MongoDBOpsManager,
     om_cluster_spec_list: Optional[list[int]] = None,
     appdb_cluster_spec_list: Optional[list[int]] = None,
+    appdb_member_configs: Optional[list[list[dict]]] = None,
 ):
     resource["spec"]["topology"] = "MultiCluster"
     backup_configs = None
@@ -75,6 +76,6 @@ def enable_multi_cluster_deployment(
     )
     resource["spec"]["applicationDatabase"]["topology"] = "MultiCluster"
     resource["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list(
-        get_appdb_member_cluster_names(), appdb_cluster_spec_list
+        get_appdb_member_cluster_names(), appdb_cluster_spec_list, appdb_member_configs
     )
     resource.api = kubernetes.client.CustomObjectsApi(api_client=get_central_cluster_client())