diff --git a/cmd/manager/BUILD.bazel b/cmd/manager/BUILD.bazel index 5a100577d..56aefaaac 100644 --- a/cmd/manager/BUILD.bazel +++ b/cmd/manager/BUILD.bazel @@ -20,6 +20,7 @@ go_library( "@com_github_operator_framework_operator_sdk//pkg/metrics:go_default_library", "@com_github_operator_framework_operator_sdk//version:go_default_library", "@com_github_spf13_pflag//:go_default_library", + "@io_k8s_client_go//kubernetes:go_default_library", "@io_k8s_client_go//plugin/pkg/client/auth:go_default_library", "@io_k8s_sigs_controller_runtime//pkg/client/config:go_default_library", "@io_k8s_sigs_controller_runtime//pkg/log:go_default_library", diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 2634281cf..34a0e450f 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -11,6 +11,7 @@ import ( _ "github.com/operator-framework/operator-sdk/pkg/metrics" sdkVersion "github.com/operator-framework/operator-sdk/version" "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -91,7 +92,7 @@ func main() { os.Exit(1) } - clientset, err := v1alpha1.GetClientset() + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { log.Error(err, "") os.Exit(1) diff --git a/contrail-provisioner/main.go b/contrail-provisioner/main.go index 70b2a19d9..aff920bd5 100644 --- a/contrail-provisioner/main.go +++ b/contrail-provisioner/main.go @@ -351,31 +351,37 @@ func getAPIClient(apiServerObj *APIServer, keystoneAuthParameters *KeystoneAuthP } func setupAuthKeystone(client *contrail.Client, keystoneAuthParameters *KeystoneAuthParameters) { - // AddEncryption expected http url in older versions of contrail-go-api - // https://github.com/Juniper/contrail-go-api/commit/4c876ba038a8ecec211376133375d467b6098202 - var authUrl string + var keystone *contrail.KeepaliveKeystoneClient if strings.HasPrefix(keystoneAuthParameters.AuthUrl, "https") { - authUrl = strings.Replace(keystoneAuthParameters.AuthUrl, "https", "http", 1) + // AddEncryption expected http url in older versions of contrail-go-api + // https://github.com/Juniper/contrail-go-api/commit/4c876ba038a8ecec211376133375d467b6098202 + keystone = contrail.NewKeepaliveKeystoneClient( + strings.Replace(keystoneAuthParameters.AuthUrl, "https", "http", 1), + keystoneAuthParameters.TenantName, + keystoneAuthParameters.AdminUsername, + keystoneAuthParameters.AdminPassword, + "", + ) + err := keystone.AddEncryption( + keystoneAuthParameters.Encryption.CA, + keystoneAuthParameters.Encryption.Key, + keystoneAuthParameters.Encryption.Cert, + keystoneAuthParameters.Encryption.Insecure) + + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } } else { - authUrl = keystoneAuthParameters.AuthUrl - } - keystone := contrail.NewKeepaliveKeystoneClient( - authUrl, - keystoneAuthParameters.TenantName, - keystoneAuthParameters.AdminUsername, - keystoneAuthParameters.AdminPassword, - "", - ) - err := keystone.AddEncryption( - keystoneAuthParameters.Encryption.CA, - keystoneAuthParameters.Encryption.Key, - keystoneAuthParameters.Encryption.Cert, - keystoneAuthParameters.Encryption.Insecure) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + keystone = contrail.NewKeepaliveKeystoneClient( + keystoneAuthParameters.AuthUrl, + keystoneAuthParameters.TenantName, + keystoneAuthParameters.AdminUsername, + keystoneAuthParameters.AdminPassword, + "", + ) } - err = keystone.AuthenticateV3() + err := keystone.AuthenticateV3() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/deploy/1-create-operator.yaml b/deploy/1-create-operator.yaml index a29963ce0..f93313331 100644 --- a/deploy/1-create-operator.yaml +++ b/deploy/1-create-operator.yaml @@ -68,6 +68,7 @@ rules: - fernetkeymanagers - contrailmonitors - contrailstatusmonitors + - devicemanagers verbs: - '*' - apiGroups: @@ -196,12 +197,12 @@ spec: - name: init # Replace this with the built image name image: registry:5000/contrail-operator/engprod-269421/contrail-operator-crdsloader:master.latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent containers: - name: contrail-operator # Replace this with the built image name image: registry:5000/contrail-operator/engprod-269421/contrail-operator:master.latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent env: - name: WATCH_NAMESPACE valueFrom: diff --git a/deploy/crds/contrail.juniper.net_commands_crd.yaml b/deploy/crds/contrail.juniper.net_commands_crd.yaml index 0335636c9..4d0bd58d4 100644 --- a/deploy/crds/contrail.juniper.net_commands_crd.yaml +++ b/deploy/crds/contrail.juniper.net_commands_crd.yaml @@ -155,6 +155,19 @@ spec: type: array contrailVersion: type: string + endpoints: + items: + description: CommandEndpoint is used to register extra endpoints + in Command + properties: + name: + type: string + privateURL: + type: string + publicURL: + type: string + type: object + type: array keystoneInstance: type: string keystoneSecretName: @@ -184,6 +197,8 @@ spec: replicas: format: int32 type: integer + targetContainerImage: + type: string upgradeState: enum: - "" @@ -191,6 +206,7 @@ spec: - not upgrading - shutting down before upgrade - starting upgraded deployment + - upgrade failed type: string type: object type: object diff --git a/deploy/crds/contrail.juniper.net_devicemanagers_crd.yaml b/deploy/crds/contrail.juniper.net_devicemanagers_crd.yaml new file mode 100644 index 000000000..b3279d5f3 --- /dev/null +++ b/deploy/crds/contrail.juniper.net_devicemanagers_crd.yaml @@ -0,0 +1,42 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: devicemanagers.contrail.juniper.net +spec: + group: contrail.juniper.net + names: + kind: Devicemanager + listKind: DevicemanagerList + plural: devicemanagers + singular: devicemanager + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Devicemanager is the Schema for the devicemanagers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DevicemanagerSpec defines the desired state of Devicemanager + type: object + status: + description: DevicemanagerStatus defines the observed state of Devicemanager + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true diff --git a/deploy/crds/contrail.juniper.net_kubemanagers_crd.yaml b/deploy/crds/contrail.juniper.net_kubemanagers_crd.yaml index da5efca22..8d1277d27 100644 --- a/deploy/crds/contrail.juniper.net_kubemanagers_crd.yaml +++ b/deploy/crds/contrail.juniper.net_kubemanagers_crd.yaml @@ -120,6 +120,11 @@ spec: description: KubemanagerServiceConfiguration is the Spec for the kubemanagers API. properties: + authMode: + enum: + - noauth + - keystone + type: string cassandraNodesConfiguration: description: CassandraClusterConfiguration stores all information about Cassandra's endpoints. @@ -195,6 +200,19 @@ spec: type: boolean ipFabricSubnets: type: string + keystoneNodesConfiguration: + description: KeystoneClusterConfiguration defines all information + about Keystone's endpoints. + properties: + authProtocol: + type: string + endpoint: + type: string + port: + type: integer + userDomainName: + type: string + type: object kubernetesAPIPort: type: integer kubernetesAPISSLPort: diff --git a/deploy/crds/contrail.juniper.net_managers_crd.yaml b/deploy/crds/contrail.juniper.net_managers_crd.yaml index 1e424c9dd..494017d8f 100644 --- a/deploy/crds/contrail.juniper.net_managers_crd.yaml +++ b/deploy/crds/contrail.juniper.net_managers_crd.yaml @@ -435,6 +435,19 @@ spec: type: array contrailVersion: type: string + endpoints: + items: + description: CommandEndpoint is used to register extra + endpoints in Command + properties: + name: + type: string + privateURL: + type: string + publicURL: + type: string + type: object + type: array keystoneInstance: type: string keystoneSecretName: @@ -464,6 +477,8 @@ spec: replicas: format: int32 type: integer + targetContainerImage: + type: string upgradeState: enum: - "" @@ -471,6 +486,7 @@ spec: - not upgrading - shutting down before upgrade - starting upgraded deployment + - upgrade failed type: string type: object type: object @@ -1378,13 +1394,13 @@ spec: kubemanagers: items: description: KubemanagerService defines desired configuration - of vRouter + of Kubemanager properties: metadata: type: object spec: description: KubemanagerServiceSpec defines desired spec configuration - of vRouter + of Kubemanager properties: commonConfiguration: description: PodConfiguration is the common services struct. @@ -1487,6 +1503,11 @@ spec: description: KubemanagerManagerServiceConfiguration defines service configuration of Kubemanager properties: + authMode: + enum: + - noauth + - keystone + type: string cassandraInstance: type: string cloudOrchestrator: @@ -1517,6 +1538,8 @@ spec: type: boolean ipFabricSubnets: type: string + keystoneInstance: + type: string kubernetesAPIPort: type: integer kubernetesAPISSLPort: @@ -1873,26 +1896,14 @@ spec: type: object type: object provisionManager: - description: ProvisionManager is the Schema for the provisionmanagers - API + description: ProvisionManagerService defines desired configuration + of ProvisionManager properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string metadata: type: object spec: - description: ProvisionManagerSpec defines the desired state - of ProvisionManager + description: ProvisionManagerServiceSpec defines desired spec + configuration of ProvisionManager properties: commonConfiguration: description: PodConfiguration is the common services struct. @@ -2036,25 +2047,6 @@ spec: required: - serviceConfiguration type: object - status: - description: ProvisionManagerStatus defines the observed state - of ProvisionManager - properties: - active: - description: 'INSERT ADDITIONAL STATUS FIELD - define observed - state of cluster Important: Run "operator-sdk generate - k8s" to regenerate code after modifying this file Add - custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html' - type: boolean - globalConfiguration: - additionalProperties: - type: string - type: object - nodes: - additionalProperties: - type: string - type: object - type: object type: object rabbitmq: description: Rabbitmq is the Schema for the rabbitmqs API. @@ -2402,6 +2394,23 @@ spec: type: string ringConfigMapName: type: string + service: + description: Service is the configuration of the + service that exposes a workload + properties: + annotations: + additionalProperties: + type: string + type: object + serviceType: + enum: + - "" + - ClusterIP + - NodePort + - LoadBalancer + - ExternalName + type: string + type: object swiftConfSecretName: type: string swiftServiceName: @@ -2471,12 +2480,12 @@ spec: type: object vrouters: items: - description: VrouterService defines desired confgiuration of vRouter + description: VrouterService defines desired configuration of vRouter properties: metadata: type: object spec: - description: VrouterServiceSpec defines desired spec confgiuration + description: VrouterServiceSpec defines desired spec configuration of vRouter properties: commonConfiguration: diff --git a/deploy/crds/contrail.juniper.net_provisionmanagers_crd.yaml b/deploy/crds/contrail.juniper.net_provisionmanagers_crd.yaml index 89dfd2645..9bd085a44 100644 --- a/deploy/crds/contrail.juniper.net_provisionmanagers_crd.yaml +++ b/deploy/crds/contrail.juniper.net_provisionmanagers_crd.yaml @@ -117,9 +117,39 @@ spec: type: array type: object serviceConfiguration: - description: ProvisionManagerConfiguration defines the provision manager - configuration + description: ProvisionManagerServiceConfiguration is the Spec for the + provisionmanagers API. properties: + configNodesConfiguration: + description: ConfigClusterConfiguration stores all information + about service's endpoints under the Contrail Config + properties: + analyticsServerIPList: + items: + type: string + type: array + analyticsServerPort: + type: integer + apiServerIPList: + items: + type: string + type: array + apiServerPort: + type: integer + authMode: + enum: + - noauth + - keystone + type: string + collectorPort: + type: integer + collectorServerIPList: + items: + type: string + type: array + redisPort: + type: integer + type: object containers: items: description: Container defines name, image and command. diff --git a/deploy/crds/contrail.juniper.net_swiftproxies_crd.yaml b/deploy/crds/contrail.juniper.net_swiftproxies_crd.yaml index 7dd176ce5..29de33397 100644 --- a/deploy/crds/contrail.juniper.net_swiftproxies_crd.yaml +++ b/deploy/crds/contrail.juniper.net_swiftproxies_crd.yaml @@ -161,6 +161,23 @@ spec: type: string ringConfigMapName: type: string + service: + description: Service is the configuration of the service that exposes + a workload + properties: + annotations: + additionalProperties: + type: string + type: object + serviceType: + enum: + - "" + - ClusterIP + - NodePort + - LoadBalancer + - ExternalName + type: string + type: object swiftConfSecretName: type: string swiftServiceName: diff --git a/deploy/crds/contrail.juniper.net_swifts_crd.yaml b/deploy/crds/contrail.juniper.net_swifts_crd.yaml index e206978f1..28412ef34 100644 --- a/deploy/crds/contrail.juniper.net_swifts_crd.yaml +++ b/deploy/crds/contrail.juniper.net_swifts_crd.yaml @@ -173,6 +173,23 @@ spec: type: string ringConfigMapName: type: string + service: + description: Service is the configuration of the service that + exposes a workload + properties: + annotations: + additionalProperties: + type: string + type: object + serviceType: + enum: + - "" + - ClusterIP + - NodePort + - LoadBalancer + - ExternalName + type: string + type: object swiftConfSecretName: type: string swiftServiceName: diff --git a/deploy/crds/contrail.juniper.net_v1alpha1_devicemanager_cr.yaml b/deploy/crds/contrail.juniper.net_v1alpha1_devicemanager_cr.yaml new file mode 100644 index 000000000..30478c9f2 --- /dev/null +++ b/deploy/crds/contrail.juniper.net_v1alpha1_devicemanager_cr.yaml @@ -0,0 +1,31 @@ +apiVersion: contrail.juniper.net/v1alpha1 +kind: Devicemanager +metadata: + labels: + contrail_cluster: cluster1 + name: devicemanager1 + namespace: contrail +spec: + commonConfiguration: + replicas: 1 + hostNetwork: true + nodeSelector: + node-role.juniper.net/contrail: "" + serviceConfiguration: + cassandraInstance: cassandra1 + keystoneInstance: keystone + containers: + - name: devicemanager + image: registry:5000/contrail-nightly/contrail-controller-config-devicemgr:master.1417-ubi + - name: dnsmasq + image: registry:5000/contrail-nightly/contrail-controller-config-dnsmasq:master.1417-ubi + - name: init + image: registry:5000/common-docker-third-party/contrail/python:3.8.2-alpine + - name: init2 + image: registry:5000/common-docker-third-party/contrail/busybox:1.31 + - name: statusmonitor + image: registry:5000/contrail-operator/engprod-269421/contrail-statusmonitor:master.latest + logLevel: SYS_DEBUG + zookeeperInstance: zookeeper1 + authMode: keystone + keystoneSecretName: cluster1-admin-password \ No newline at end of file diff --git a/deploy/kustomize/base/operator/operator.yaml b/deploy/kustomize/base/operator/operator.yaml index 2a8bd6a17..fba2aa4a3 100755 --- a/deploy/kustomize/base/operator/operator.yaml +++ b/deploy/kustomize/base/operator/operator.yaml @@ -195,13 +195,13 @@ spec: initContainers: - name: init image: crdsloader:latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent containers: - name: contrail-operator # Replace this with the built image name image: contrail-operator:latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent env: - name: WATCH_NAMESPACE valueFrom: diff --git a/deploy/openshift/manifests/00-contrail-08-operator.yaml b/deploy/openshift/manifests/00-contrail-08-operator.yaml index a801bd435..fde4346b4 100644 --- a/deploy/openshift/manifests/00-contrail-08-operator.yaml +++ b/deploy/openshift/manifests/00-contrail-08-operator.yaml @@ -38,7 +38,7 @@ spec: - name: contrail-operator # Replace this with the built image name image: hub.juniper.net/contrail-nightly/contrail-operator: - imagePullPolicy: Always + imagePullPolicy: IfNotPresent env: - name: WATCH_NAMESPACE valueFrom: diff --git a/deploy/openshift/manifests/00-contrail-09-manager.yaml b/deploy/openshift/manifests/00-contrail-09-manager.yaml index 9861c99a3..55619c514 100644 --- a/deploy/openshift/manifests/00-contrail-09-manager.yaml +++ b/deploy/openshift/manifests/00-contrail-09-manager.yaml @@ -21,11 +21,11 @@ spec: serviceConfiguration: containers: - name: cassandra - image: cassandra:3.11.4 + image: /common-docker-third-party/contrail/cassandra:3.11.4 - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: init2 - image: cassandra:3.11.4 + image: /common-docker-third-party/contrail/cassandra:3.11.4 config: metadata: labels: @@ -58,11 +58,11 @@ spec: - "-c" - "tail -f /dev/null" - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: init2 - image: busybox:1.31 + image: /common-docker-third-party/contrail/busybox:1.31 - name: redis - image: redis:4.0.2 + image: /common-docker-third-party/contrail/redis:4.0.2 - name: schematransformer image: /contrail-controller-config-schema: - name: servicemonitor @@ -88,7 +88,7 @@ spec: - name: dns image: /contrail-controller-control-dns: - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: named image: /contrail-controller-control-named: - name: statusmonitor @@ -105,7 +105,7 @@ spec: serviceConfiguration: containers: - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: provisioner image: /contrail-operator-provisioner: rabbitmq: @@ -117,9 +117,9 @@ spec: serviceConfiguration: containers: - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: rabbitmq - image: rabbitmq:3.7 + image: /common-docker-third-party/contrail/rabbitmq:3.7 webui: metadata: labels: @@ -130,9 +130,9 @@ spec: cassandraInstance: cassandra1 containers: - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: redis - image: redis:4.0.2 + image: /common-docker-third-party/contrail/redis:4.0.2 - name: webuijob image: /contrail-controller-webui-job: - name: webuiweb @@ -152,11 +152,11 @@ spec: serviceConfiguration: containers: - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: conf-init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: zookeeper - image: docker.io/zookeeper:3.5.5 + image: /common-docker-third-party/contrail/zookeeper:3.5.5 kubemanagers: - metadata: labels: @@ -168,7 +168,7 @@ spec: zookeeperInstance: zookeeper1 containers: - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: kubemanager image: /contrail-kubernetes-kube-manager: - name: statusmonitor @@ -194,7 +194,7 @@ spec: contrailStatusImage: /contrail-status: containers: - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: nodeinit image: /contrail-node-init: - name: vrouteragent @@ -206,7 +206,7 @@ spec: - name: vrouterkernelinit image: /contrail-vrouter-kernel-init: - name: multusconfig - image: busybox:1.31 + image: /common-docker-third-party/contrail/busybox:1.31 - metadata: labels: contrail_cluster: cluster1 @@ -222,7 +222,7 @@ spec: contrailStatusImage: /contrail-status: containers: - name: init - image: python:3.8.2-alpine + image: /common-docker-third-party/contrail/python:3.8.2-alpine - name: nodeinit image: /contrail-node-init: - name: vrouteragent @@ -234,7 +234,7 @@ spec: - name: vrouterkernelinit image: /contrail-vrouter-kernel-init: - name: multusconfig - image: busybox:1.31 + image: /common-docker-third-party/contrail/busybox:1.31 contrailCNIs: - metadata: @@ -253,7 +253,7 @@ spec: - name: vroutercni image: /contrail-kubernetes-cni-init: - name: multusconfig - image: busybox:1.31 + image: /common-docker-third-party/contrail/busybox:1.31 - metadata: labels: contrail_cluster: cluster1 @@ -270,4 +270,19 @@ spec: - name: vroutercni image: /contrail-kubernetes-cni-init: - name: multusconfig - image: busybox:1.31 + image: /common-docker-third-party/contrail/busybox:1.31 + contrailmonitor: + metadata: + labels: + contrail_cluster: cluster1 + name: contrailmonitor + spec: + serviceConfiguration: + cassandraInstance: cassandra1 + rabbitmqInstance: rabbitmq1 + zookeeperInstance: zookeeper1 + webuiInstance: webui1 + configInstance: config1 + controlInstance: control1 + provisionmanagerInstance: provmanager1 + diff --git a/deploy/openshift/releases/R2011/manifests/00-contrail-08-operator.yaml b/deploy/openshift/releases/R2011/manifests/00-contrail-08-operator.yaml index cec5dac07..8235866c1 100644 --- a/deploy/openshift/releases/R2011/manifests/00-contrail-08-operator.yaml +++ b/deploy/openshift/releases/R2011/manifests/00-contrail-08-operator.yaml @@ -37,7 +37,7 @@ spec: containers: - name: contrail-operator # Replace this with the built image name - image: hub.juniper.net/contrail-nightly/contrail-operator:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-operator:2011.85-ubi imagePullPolicy: Always env: - name: WATCH_NAMESPACE diff --git a/deploy/openshift/releases/R2011/manifests/00-contrail-09-manager.yaml b/deploy/openshift/releases/R2011/manifests/00-contrail-09-manager.yaml index 411ef0212..c25ead643 100644 --- a/deploy/openshift/releases/R2011/manifests/00-contrail-09-manager.yaml +++ b/deploy/openshift/releases/R2011/manifests/00-contrail-09-manager.yaml @@ -24,11 +24,11 @@ spec: serviceConfiguration: containers: - name: cassandra - image: cassandra:3.11.4 + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/cassandra:3.11.4 - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: init2 - image: cassandra:3.11.4 + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/cassandra:3.11.4 config: metadata: labels: @@ -47,37 +47,37 @@ spec: analyticsStatisticsTTL: 2 containers: - name: analyticsapi - image: hub.juniper.net/contrail-nightly/contrail-analytics-api:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-analytics-api:2011.85-ubi - name: api - image: hub.juniper.net/contrail-nightly/contrail-controller-config-api:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-config-api:2011.85-ubi - name: collector - image: hub.juniper.net/contrail-nightly/contrail-analytics-collector:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-analytics-collector:2011.85-ubi - name: devicemanager - image: hub.juniper.net/contrail-nightly/contrail-controller-config-devicemgr:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-config-devicemgr:2011.85-ubi command: - "/bin/sh" - "-c" - "tail -f /dev/null" - name: dnsmasq - image: hub.juniper.net/contrail-nightly/contrail-controller-config-dnsmasq:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-config-dnsmasq:2011.85-ubi command: - "/bin/sh" - "-c" - "tail -f /dev/null" - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: init2 - image: busybox:1.31 + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/busybox:1.31 - name: redis - image: redis:4.0.2 + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/redis:4.0.2 - name: schematransformer - image: hub.juniper.net/contrail-nightly/contrail-controller-config-schema:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-config-schema:2011.85-ubi - name: servicemonitor - image: hub.juniper.net/contrail-nightly/contrail-controller-config-svcmonitor:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-config-svcmonitor:2011.85-ubi - name: queryengine - image: hub.juniper.net/contrail-nightly/contrail-analytics-query-engine:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-analytics-query-engine:2011.85-ubi - name: statusmonitor - image: hub.juniper.net/contrail-nightly/contrail-statusmonitor:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-statusmonitor:2011.85-ubi logLevel: SYS_DEBUG zookeeperInstance: zookeeper1 controls: @@ -95,15 +95,15 @@ spec: cassandraInstance: cassandra1 containers: - name: control - image: hub.juniper.net/contrail-nightly/contrail-controller-control-control:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-control-control:2011.85-ubi - name: dns - image: hub.juniper.net/contrail-nightly/contrail-controller-control-dns:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-control-dns:2011.85-ubi - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: named - image: hub.juniper.net/contrail-nightly/contrail-controller-control-named:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-control-named:2011.85-ubi - name: statusmonitor - image: hub.juniper.net/contrail-nightly/contrail-statusmonitor:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-statusmonitor:2011.85-ubi zookeeperInstance: zookeeper1 provisionManager: metadata: @@ -119,9 +119,9 @@ spec: serviceConfiguration: containers: - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: provisioner - image: hub.juniper.net/contrail-nightly/contrail-operator-provisioner:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-operator-provisioner:2011.85-ubi rabbitmq: metadata: labels: @@ -135,7 +135,7 @@ spec: serviceConfiguration: containers: - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: rabbitmq image: rabbitmq:3.7 webui: @@ -152,13 +152,13 @@ spec: cassandraInstance: cassandra1 containers: - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: redis - image: redis:4.0.2 + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/redis:4.0.2 - name: webuijob - image: hub.juniper.net/contrail-nightly/contrail-controller-webui-job:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-webui-job:2011.85-ubi - name: webuiweb - image: hub.juniper.net/contrail-nightly/contrail-controller-webui-web:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-controller-webui-web:2011.85-ubi zookeepers: - metadata: labels: @@ -172,9 +172,9 @@ spec: serviceConfiguration: containers: - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: zookeeper - image: docker.io/zookeeper:3.5.5 + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/zookeeper:3.5.5 kubemanagers: - metadata: labels: @@ -190,11 +190,11 @@ spec: zookeeperInstance: zookeeper1 containers: - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: kubemanager - image: hub.juniper.net/contrail-nightly/contrail-kubernetes-kube-manager:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-kubernetes-kube-manager:2011.85-ubi - name: statusmonitor - image: hub.juniper.net/contrail-nightly/contrail-statusmonitor:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-statusmonitor:2011.85-ubi ipFabricForwarding: false ipFabricSnat: true kubernetesTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token @@ -215,22 +215,22 @@ spec: serviceConfiguration: cassandraInstance: cassandra1 controlInstance: control1 - contrailStatusImage: hub.juniper.net/contrail-nightly/contrail-status:2011.31-ubi + contrailStatusImage: hub.juniper.net/contrail-nightly/contrail-status:2011.85-ubi containers: - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: nodeinit - image: hub.juniper.net/contrail-nightly/contrail-node-init:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-node-init:2011.85-ubi - name: vrouteragent - image: hub.juniper.net/contrail-nightly/contrail-vrouter-agent:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-vrouter-agent:2011.85-ubi - name: vroutercni - image: hub.juniper.net/contrail-nightly/contrail-kubernetes-cni-init:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-kubernetes-cni-init:2011.85-ubi - name: vrouterkernelbuildinit - image: hub.juniper.net/contrail-nightly/contrail-vrouter-kernel-build-init:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-vrouter-kernel-build-init:2011.85-ubi - name: vrouterkernelinit - image: hub.juniper.net/contrail-nightly/contrail-vrouter-kernel-init:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-vrouter-kernel-init:2011.85-ubi - name: multusconfig - image: busybox:1.31 + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/busybox:1.31 - metadata: labels: contrail_cluster: cluster1 @@ -245,19 +245,33 @@ spec: serviceConfiguration: cassandraInstance: cassandra1 controlInstance: control1 - contrailStatusImage: hub.juniper.net/contrail-nightly/contrail-status:2011.31-ubi + contrailStatusImage: hub.juniper.net/contrail-nightly/contrail-status:2011.85-ubi containers: - name: init - image: python:3.8.2-alpine + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/python:3.8.2-alpine - name: nodeinit - image: hub.juniper.net/contrail-nightly/contrail-node-init:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-node-init:2011.85-ubi - name: vrouteragent - image: hub.juniper.net/contrail-nightly/contrail-vrouter-agent:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-vrouter-agent:2011.85-ubi - name: vroutercni - image: hub.juniper.net/contrail-nightly/contrail-kubernetes-cni-init:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-kubernetes-cni-init:2011.85-ubi - name: vrouterkernelbuildinit - image: hub.juniper.net/contrail-nightly/contrail-vrouter-kernel-build-init:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-vrouter-kernel-build-init:2011.85-ubi - name: vrouterkernelinit - image: hub.juniper.net/contrail-nightly/contrail-vrouter-kernel-init:2011.31-ubi + image: hub.juniper.net/contrail-nightly/contrail-vrouter-kernel-init:2011.85-ubi - name: multusconfig - image: busybox:1.31 + image: hub.juniper.net/contrail-nightly/common-docker-third-party/contrail/busybox:1.31 + contrailmonitor: + metadata: + labels: + contrail_cluster: cluster1 + name: contrailmonitor + spec: + serviceConfiguration: + cassandraInstance: cassandra1 + rabbitmqInstance: rabbitmq1 + zookeeperInstance: zookeeper1 + webuiInstance: webui1 + configInstance: config1 + controlInstance: control1 + provisionmanagerInstance: provmanager1 diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 3f4db4b84..e156bd05a 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -23,12 +23,12 @@ spec: - name: init # Replace this with the built image name image: registry:5000/contrail-operator/engprod-269421/contrail-operator-crdsloader:master.latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent containers: - name: contrail-operator # Replace this with the built image name image: registry:5000/contrail-operator/engprod-269421/contrail-operator:master.latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent env: - name: WATCH_NAMESPACE valueFrom: diff --git a/deploy/role.yaml b/deploy/role.yaml index bc3e363da..50e3e3831 100644 --- a/deploy/role.yaml +++ b/deploy/role.yaml @@ -62,6 +62,7 @@ rules: - fernetkeymanagers - contrailmonitors - contrailstatusmonitors + - devicemanagers verbs: - '*' - apiGroups: diff --git a/deploy/svl_artifactory/1-create-operator.yaml b/deploy/svl_artifactory/1-create-operator.yaml index 883678f10..c62d6892a 100644 --- a/deploy/svl_artifactory/1-create-operator.yaml +++ b/deploy/svl_artifactory/1-create-operator.yaml @@ -5091,7 +5091,7 @@ spec: - name: contrail-operator # Replace this with the built image name image: svl-artifactory.juniper.net/contrail-operator/engprod-269421/contrail-operator:change_image_format.latest - imagePullPolicy: Always + imagePullPolicy: IfNotPresent env: - name: WATCH_NAMESPACE valueFrom: diff --git a/deployments/cassandra.yaml b/deployments/cassandra.yaml index 9be97f4f5..99f2e5252 100644 --- a/deployments/cassandra.yaml +++ b/deployments/cassandra.yaml @@ -19,7 +19,7 @@ spec: terminationGracePeriodSeconds: 1800 containers: - image: hub.juniper.net/contrail-nightly/contrail-external-cassandra:5.2.0-0.740 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent env: - name: POD_IP valueFrom: @@ -64,7 +64,7 @@ spec: - -c - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init resources: {} securityContext: diff --git a/deployments/config.yaml b/deployments/config.yaml index e3abab67f..8b5f5bf19 100644 --- a/deployments/config.yaml +++ b/deployments/config.yaml @@ -23,7 +23,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: api readinessProbe: httpGet: @@ -38,7 +38,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: devicemanager volumeMounts: - mountPath: /var/log/contrail @@ -49,7 +49,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: schematransformer volumeMounts: - mountPath: /var/log/contrail @@ -60,7 +60,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: servicemonitor volumeMounts: - mountPath: /var/log/contrail @@ -71,7 +71,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: analyticsapi volumeMounts: - mountPath: /var/log/contrail @@ -82,7 +82,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: collector volumeMounts: - mountPath: /var/log/contrail @@ -93,7 +93,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: redis volumeMounts: - mountPath: /var/log/contrail @@ -110,7 +110,7 @@ spec: fieldRef: fieldPath: status.podIP image: docker.io/michaelhenkel/contrail-nodemgr:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: nodemanagerconfig volumeMounts: - mountPath: /var/log/contrail @@ -127,7 +127,7 @@ spec: fieldRef: fieldPath: status.podIP image: docker.io/michaelhenkel/contrail-nodemgr:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: nodemanageranalytics volumeMounts: - mountPath: /var/log/contrail @@ -142,7 +142,7 @@ spec: - -c - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init volumeMounts: - mountPath: /tmp/podinfo diff --git a/deployments/control.yaml b/deployments/control.yaml index e6f5e6ddf..3d9fbf7f9 100644 --- a/deployments/control.yaml +++ b/deployments/control.yaml @@ -21,7 +21,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: control volumeMounts: - mountPath: /var/log/contrail @@ -32,7 +32,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: dns volumeMounts: - mountPath: /var/log/contrail @@ -47,7 +47,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: named securityContext: privileged: true @@ -69,7 +69,7 @@ spec: fieldRef: fieldPath: status.podIP image: docker.io/michaelhenkel/contrail-nodemgr:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: nodemanager lifecycle: preStop: @@ -94,7 +94,7 @@ spec: fieldRef: fieldPath: status.podIP image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init volumeMounts: - mountPath: /tmp/podinfo diff --git a/deployments/kubemanager.yaml b/deployments/kubemanager.yaml index dff48486c..f95440d78 100644 --- a/deployments/kubemanager.yaml +++ b/deployments/kubemanager.yaml @@ -20,7 +20,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: kubemanager volumeMounts: - mountPath: /var/log/contrail @@ -33,7 +33,7 @@ spec: - -c - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init volumeMounts: - mountPath: /tmp/podinfo diff --git a/deployments/rabbitmq.yaml b/deployments/rabbitmq.yaml index 2b9c544d9..6e5e4d2b9 100644 --- a/deployments/rabbitmq.yaml +++ b/deployments/rabbitmq.yaml @@ -31,7 +31,7 @@ spec: - -c - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init resources: {} securityContext: @@ -54,7 +54,7 @@ spec: fieldRef: fieldPath: metadata.name image: docker.io/michaelhenkel/contrail-external-rabbitmq:5.2.0-dev1 - imagePullPolicy: "" + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/lib/rabbitmq name: rabbitmq-data diff --git a/deployments/vrouter.yaml b/deployments/vrouter.yaml index 4f711e90d..4635c1ded 100644 --- a/deployments/vrouter.yaml +++ b/deployments/vrouter.yaml @@ -18,7 +18,7 @@ spec: - configMapRef: name: vrouter image: docker.io/michaelhenkel/contrail-vrouter-agent:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent lifecycle: preStop: exec: @@ -53,7 +53,7 @@ spec: - configMapRef: name: vrouter image: docker.io/michaelhenkel/contrail-nodemgr:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: nodemanager volumeMounts: - mountPath: /var/log/contrail @@ -67,7 +67,7 @@ spec: - configMapRef: name: vrouter image: docker.io/michaelhenkel/contrail-node-init:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: nodeinit securityContext: privileged: true @@ -81,7 +81,7 @@ spec: - configMapRef: name: vrouter image: docker.io/michaelhenkel/contrail-vrouter-kernel-init:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: vrouterkernelinit securityContext: privileged: true @@ -100,7 +100,7 @@ spec: - configMapRef: name: vrouter image: docker.io/michaelhenkel/contrail-kubernetes-cni-init:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: vroutercni securityContext: privileged: true diff --git a/deployments/webui.yaml b/deployments/webui.yaml index 9c11c2581..f662772b8 100644 --- a/deployments/webui.yaml +++ b/deployments/webui.yaml @@ -24,7 +24,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: webuiweb volumeMounts: - mountPath: /var/log/contrail @@ -39,7 +39,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: webuijob volumeMounts: - mountPath: /var/log/contrail @@ -50,7 +50,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: redis volumeMounts: - mountPath: /var/log/contrail diff --git a/deployments/zookeeper.yaml b/deployments/zookeeper.yaml index aa5fa3b32..1772a0b20 100644 --- a/deployments/zookeeper.yaml +++ b/deployments/zookeeper.yaml @@ -31,7 +31,7 @@ spec: - -c - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init resources: {} securityContext: @@ -50,7 +50,7 @@ spec: fieldRef: fieldPath: status.podIP image: docker.io/michaelhenkel/contrail-external-zookeeper:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent resources: requests: memory: "1Gi" diff --git a/pkg/apis/contrail/v1alpha1/BUILD.bazel b/pkg/apis/contrail/v1alpha1/BUILD.bazel index a2a3b53fa..d42be8604 100644 --- a/pkg/apis/contrail/v1alpha1/BUILD.bazel +++ b/pkg/apis/contrail/v1alpha1/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "contrailstatusmonitor_types.go", "control_types.go", "defaults.go", + "devicemanager_types.go", "doc.go", "exec_to_pod.go", "fernetkeymanager_types.go", @@ -23,6 +24,7 @@ go_library( "provisionmanager_types.go", "rabbitmq_types.go", "register.go", + "service.go", "status.go", "storage.go", "swift_types.go", @@ -37,8 +39,8 @@ go_library( importpath = "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1", visibility = ["//visibility:public"], deps = [ - "//pkg/apis/contrail/v1alpha1/templates:go_default_library", "//pkg/certificates:go_default_library", + "//pkg/configuration:go_default_library", "@com_github_go_openapi_spec//:go_default_library", "@in_gopkg_yaml.v2//:go_default_library", "@io_k8s_api//apps/v1:go_default_library", @@ -55,10 +57,10 @@ go_library( "@io_k8s_client_go//dynamic:go_default_library", "@io_k8s_client_go//kubernetes:go_default_library", "@io_k8s_client_go//rest:go_default_library", - "@io_k8s_client_go//tools/clientcmd:go_default_library", "@io_k8s_client_go//tools/remotecommand:go_default_library", "@io_k8s_kube_openapi//pkg/common:go_default_library", "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/client/config:go_default_library", "@io_k8s_sigs_controller_runtime//pkg/controller/controllerutil:go_default_library", "@io_k8s_sigs_controller_runtime//pkg/reconcile:go_default_library", "@io_k8s_sigs_controller_runtime//pkg/runtime/scheme:go_default_library", diff --git a/pkg/apis/contrail/v1alpha1/base_types.go b/pkg/apis/contrail/v1alpha1/base_types.go index 33cc3efa3..8cdc36dc0 100644 --- a/pkg/apis/contrail/v1alpha1/base_types.go +++ b/pkg/apis/contrail/v1alpha1/base_types.go @@ -978,6 +978,29 @@ func NewConfigClusterConfiguration(name string, namespace string, myclient clien return configCluster, nil } +// NewKeystoneClusterConfiguration gets a struct containing various representations of Keystone nodes string. +func NewKeystoneClusterConfiguration(name string, namespace string, client client.Client) (KeystoneClusterConfiguration, error) { + var keystoneCluster KeystoneClusterConfiguration + keystoneInstance := &Keystone{} + if name == "" { + return keystoneCluster, nil + } + err := client.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, keystoneInstance) + if err != nil { + return keystoneCluster, err + } + + keystoneConfig := keystoneInstance.ConfigurationParameters() + endpoint := keystoneInstance.Status.Endpoint + keystoneCluster = KeystoneClusterConfiguration{ + Endpoint: endpoint, + Port: keystoneConfig.ListenPort, + AuthProtocol: keystoneConfig.AuthProtocol, + UserDomainName: keystoneConfig.UserDomainName, + } + return keystoneCluster, nil +} + // WebUIClusterConfiguration defines all configuration knobs used to write the config file. type WebUIClusterConfiguration struct { AdminUsername string @@ -1117,3 +1140,25 @@ type VrouterClusterConfiguration struct { Gateway string MetaDataSecret string } + +// KeystoneClusterConfiguration defines all information about Keystone's endpoints. +type KeystoneClusterConfiguration struct { + Endpoint string `json:"endpoint,omitempty"` + Port int `json:"port,omitempty"` + AuthProtocol string `json:"authProtocol,omitempty"` + UserDomainName string `json:"userDomainName,omitempty"` +} + +// FillWithDefaultValues fills Keystone config with default values. +func (c *KeystoneClusterConfiguration) FillWithDefaultValues() { + if c.Port == 0 { + c.Port = KeystoneAuthPublicPort + } + if c.AuthProtocol != "http" && c.AuthProtocol != "https" { + c.AuthProtocol = KeystoneAuthProto + } + if c.UserDomainName == "" { + c.UserDomainName = KeystoneAuthUserDomainName + } + +} diff --git a/pkg/apis/contrail/v1alpha1/cassandra_types.go b/pkg/apis/contrail/v1alpha1/cassandra_types.go index 7b5a8c7d3..6b4438885 100644 --- a/pkg/apis/contrail/v1alpha1/cassandra_types.go +++ b/pkg/apis/contrail/v1alpha1/cassandra_types.go @@ -12,8 +12,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configtemplates "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates" "github.com/Juniper/contrail-operator/pkg/certificates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/apis/contrail/v1alpha1/command_types.go b/pkg/apis/contrail/v1alpha1/command_types.go index 0cc0e8c05..b844d675d 100644 --- a/pkg/apis/contrail/v1alpha1/command_types.go +++ b/pkg/apis/contrail/v1alpha1/command_types.go @@ -59,13 +59,14 @@ type CommandEndpoint struct { // CommandStatus defines the observed state of Command // +k8s:openapi-gen=true type CommandStatus struct { - Status `json:",inline"` - Endpoint string `json:"endpoint,omitempty"` - UpgradeState CommandUpgradeState `json:"upgradeState,omitempty"` - ContainerImage string `json:"containerImage,omitempty"` + Status `json:",inline"` + Endpoint string `json:"endpoint,omitempty"` + UpgradeState CommandUpgradeState `json:"upgradeState,omitempty"` + TargetContainerImage string `json:"targetContainerImage,omitempty"` + ContainerImage string `json:"containerImage,omitempty"` } -// +kubebuilder:validation:Enum={"","upgrading","not upgrading","shutting down before upgrade","starting upgraded deployment"} +// +kubebuilder:validation:Enum={"","upgrading","not upgrading","shutting down before upgrade","starting upgraded deployment", "upgrade failed"} type CommandUpgradeState string const ( @@ -73,6 +74,7 @@ const ( CommandShuttingDownBeforeUpgrade CommandUpgradeState = "shutting down before upgrade" CommandUpgrading CommandUpgradeState = "upgrading" CommandStartingUpgradedDeployment CommandUpgradeState = "starting upgraded deployment" + CommandUpgradeFailed CommandUpgradeState = "upgrade failed" ) // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -90,6 +92,15 @@ func (c *Command) PodsCertSubjects(podList *corev1.PodList, serviceIP string) [] return PodsCertSubjects(podList, c.Spec.CommonConfiguration.HostNetwork, altIPs) } +//Upgrading is used to check if command is in the upgrade process +func (c *Command) Upgrading() bool { + if c.Status.UpgradeState == CommandShuttingDownBeforeUpgrade || + c.Status.UpgradeState == CommandUpgrading { + return true + } + return false +} + func init() { SchemeBuilder.Register(&Command{}, &CommandList{}) } diff --git a/pkg/apis/contrail/v1alpha1/config_types.go b/pkg/apis/contrail/v1alpha1/config_types.go index e0c5692d4..688ab58f2 100644 --- a/pkg/apis/contrail/v1alpha1/config_types.go +++ b/pkg/apis/contrail/v1alpha1/config_types.go @@ -17,8 +17,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configtemplates "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates" "github.com/Juniper/contrail-operator/pkg/certificates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" ) // +kubebuilder:validation:Enum=noauth;keystone diff --git a/pkg/apis/contrail/v1alpha1/control_types.go b/pkg/apis/contrail/v1alpha1/control_types.go index 829d2feab..47812cf4d 100644 --- a/pkg/apis/contrail/v1alpha1/control_types.go +++ b/pkg/apis/contrail/v1alpha1/control_types.go @@ -11,8 +11,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configtemplates "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates" "github.com/Juniper/contrail-operator/pkg/certificates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/apis/contrail/v1alpha1/devicemanager_types.go b/pkg/apis/contrail/v1alpha1/devicemanager_types.go new file mode 100644 index 000000000..9406d91a3 --- /dev/null +++ b/pkg/apis/contrail/v1alpha1/devicemanager_types.go @@ -0,0 +1,585 @@ +package v1alpha1 + +import ( + "bytes" + "context" + "fmt" + "sort" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/labels" + + appsv1 "k8s.io/api/apps/v1" + + "k8s.io/apimachinery/pkg/runtime" + + "github.com/Juniper/contrail-operator/pkg/certificates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// DevicemanagerSpec defines the desired state of Devicemanager +type DevicemanagerSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file + // Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html + CommonConfiguration PodConfiguration `json:"commonConfiguration,omitempty"` + ServiceConfiguration DevicemanagerConfiguration `json:"serviceConfiguration"` +} + +// ConfigConfiguration is the Spec for the Config API. +// +k8s:openapi-gen=true +type DevicemanagerConfiguration struct { + Containers []*Container `json:"containers,omitempty"` + DeviceManagerIntrospectPort *int `json:"deviceManagerIntrospectPort,omitempty"` + CassandraInstance string `json:"cassandraInstance,omitempty"` + ZookeeperInstance string `json:"zookeeperInstance,omitempty"` + ConfigInstance string `json:"configInstance,omitempty"` + NodeManager *bool `json:"nodeManager,omitempty"` + RabbitmqUser string `json:"rabbitmqUser,omitempty"` + RabbitmqPassword string `json:"rabbitmqPassword,omitempty"` + RabbitmqVhost string `json:"rabbitmqVhost,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + KeystoneSecretName string `json:"keystoneSecretName,omitempty"` + KeystoneInstance string `json:"keystoneInstance,omitempty"` + AuthMode AuthenticationMode `json:"authMode,omitempty"` + AAAMode AAAMode `json:"aaaMode,omitempty"` + Storage Storage `json:"storage,omitempty"` + FabricMgmtIP string `json:"fabricMgmtIP,omitempty"` +} + +// DevicemanagerStatus defines the observed state of Devicemanager +type DevicemanagerStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file + // Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html + Active *bool `json:"active,omitempty"` + Nodes map[string]string `json:"nodes,omitempty"` + ConfigChanged *bool `json:"configChanged,omitempty"` + ServiceStatus map[string]DevicemanagerServiceStatus `json:"serviceStatus,omitempty"` + Endpoint string `json:"endpoint,omitempty"` +} + +type DevicemanagerServiceStatus struct { + NodeName string `json:"nodeName,omitempty"` + ModuleName string `json:"moduleName,omitempty"` + ModuleState string `json:"state"` + Description string `json:"description,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Devicemanager is the Schema for the devicemanagers API +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=devicemanagers,scope=Namespaced +type Devicemanager struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DevicemanagerSpec `json:"spec,omitempty"` + Status DevicemanagerStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// DevicemanagerList contains a list of Devicemanager +type DevicemanagerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Devicemanager `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Devicemanager{}, &DevicemanagerList{}) +} + +func (c *Devicemanager) InstanceConfiguration(request reconcile.Request, + podList *corev1.PodList, + client client.Client) error { + instanceDevicemanagerMapName := request.Name + "-" + "devicemanager" + "-configmap" + configMapInstanceDynamicConfig := &corev1.ConfigMap{} + err := client.Get(context.TODO(), types.NamespacedName{Name: instanceDevicemanagerMapName, + Namespace: request.Namespace}, configMapInstanceDynamicConfig) + if err != nil { + return err + } + + cassandraNodesInformation, err := NewCassandraClusterConfiguration(c.Spec.ServiceConfiguration.CassandraInstance, + request.Namespace, client) + if err != nil { + return err + } + + zookeeperNodesInformation, err := NewZookeeperClusterConfiguration(c.Spec.ServiceConfiguration.ZookeeperInstance, + request.Namespace, client) + if err != nil { + return err + } + + configNodesInformation, err := NewConfigClusterConfiguration(c.Spec.ServiceConfiguration.ConfigInstance, + request.Namespace, client) + if err != nil { + return err + } + + rabbitmqNodesInformation, err := NewRabbitmqClusterConfiguration(c.Labels["contrail_cluster"], + request.Namespace, client) + if err != nil { + return err + } + var rabbitmqSecretUser string + var rabbitmqSecretPassword string + var rabbitmqSecretVhost string + if rabbitmqNodesInformation.Secret != "" { + rabbitmqSecret := &corev1.Secret{} + err = client.Get(context.TODO(), types.NamespacedName{Name: rabbitmqNodesInformation.Secret, + Namespace: request.Namespace}, rabbitmqSecret) + if err != nil { + return err + } + rabbitmqSecretUser = string(rabbitmqSecret.Data["user"]) + rabbitmqSecretPassword = string(rabbitmqSecret.Data["password"]) + rabbitmqSecretVhost = string(rabbitmqSecret.Data["vhost"]) + } + + devicemanagerConfig := c.ConfigurationParameters() + if rabbitmqSecretUser == "" { + rabbitmqSecretUser = devicemanagerConfig.RabbitmqUser + } + if rabbitmqSecretPassword == "" { + rabbitmqSecretPassword = devicemanagerConfig.RabbitmqPassword + } + if rabbitmqSecretVhost == "" { + rabbitmqSecretVhost = devicemanagerConfig.RabbitmqVhost + } + var collectorServerList, analyticsServerList, apiServerList, analyticsServerSpaceSeparatedList, + apiServerSpaceSeparatedList string + var podIPList []string + for _, pod := range podList.Items { + podIPList = append(podIPList, pod.Status.PodIP) + } + sort.SliceStable(podList.Items, func(i, j int) bool { return podList.Items[i].Status.PodIP < podList.Items[j].Status.PodIP }) + sort.SliceStable(podIPList, func(i, j int) bool { return podIPList[i] < podIPList[j] }) + + collectorServerList = strings.Join(podIPList, ":"+strconv.Itoa(configNodesInformation.CollectorPort)+" ") + collectorServerList = collectorServerList + ":" + strconv.Itoa(configNodesInformation.CollectorPort) + analyticsServerList = strings.Join(podIPList, ",") + analyticsServerSpaceSeparatedList = strings.Join(podIPList, + ":"+strconv.Itoa(configNodesInformation.AnalyticsServerPort)+" ") + analyticsServerSpaceSeparatedList = analyticsServerSpaceSeparatedList + ":" + + strconv.Itoa(configNodesInformation.AnalyticsServerPort) + + apiServerList = strings.Join(podIPList, ",") + apiServerSpaceSeparatedList = strings.Join(podIPList, ":"+strconv.Itoa(configNodesInformation.APIServerPort)+" ") + apiServerSpaceSeparatedList = apiServerSpaceSeparatedList + ":" + strconv.Itoa(configNodesInformation.APIServerPort) + + cassandraEndpointList := configtemplates.EndpointList(cassandraNodesInformation.ServerIPList, cassandraNodesInformation.Port) + cassandraEndpointListSpaceSeparated := configtemplates.JoinListWithSeparator(cassandraEndpointList, " ") + + rabbitMqSSLEndpointList := configtemplates.EndpointList(rabbitmqNodesInformation.ServerIPList, rabbitmqNodesInformation.SSLPort) + rabbitmqSSLEndpointListCommaSeparated := configtemplates.JoinListWithSeparator(rabbitMqSSLEndpointList, ",") + + zookeeperEndpointList := configtemplates.EndpointList(zookeeperNodesInformation.ServerIPList, zookeeperNodesInformation.ClientPort) + zookeeperEndpointListCommaSeparated := configtemplates.JoinListWithSeparator(zookeeperEndpointList, ",") + + var data = make(map[string]string) + for idx, pod := range podList.Items { + devicemanagerAuth, err := c.AuthParameters(client) + if err != nil { + return err + } + configIntrospectNodes := make([]string, 0) + introspectPorts := map[string]int{ + "contrail-device-manager": *devicemanagerConfig.DeviceManagerIntrospectPort, + } + for service, port := range introspectPorts { + nodesPortStr := pod.Status.PodIP + ":" + strconv.Itoa(port) + "::" + service + configIntrospectNodes = append(configIntrospectNodes, nodesPortStr) + } + hostname := podList.Items[idx].Annotations["hostname"] + statusMonitorConfig, err := StatusMonitorConfig(hostname, configIntrospectNodes, + podList.Items[idx].Status.PodIP, "devicemanager", request.Name, request.Namespace, pod.Name) + if err != nil { + return err + } + data["monitorconfig."+podList.Items[idx].Status.PodIP+".yaml"] = statusMonitorConfig + + fabricMgmtIP := podList.Items[idx].Status.PodIP + if c.Spec.ServiceConfiguration.FabricMgmtIP != "" { + fabricMgmtIP = c.Spec.ServiceConfiguration.FabricMgmtIP + } + + var configDevicemanagerConfigBuffer bytes.Buffer + err = configtemplates.ConfigDeviceManagerConfig.Execute(&configDevicemanagerConfigBuffer, struct { + HostIP string + ApiServerList string + AnalyticsServerList string + CassandraServerList string + ZookeeperServerList string + RabbitmqServerList string + CollectorServerList string + RabbitmqUser string + RabbitmqPassword string + RabbitmqVhost string + LogLevel string + FabricMgmtIP string + CAFilePath string + DeviceManagerIntrospectPort string + }{ + HostIP: podList.Items[idx].Status.PodIP, + ApiServerList: apiServerList, + AnalyticsServerList: analyticsServerList, + CassandraServerList: cassandraEndpointListSpaceSeparated, + ZookeeperServerList: zookeeperEndpointListCommaSeparated, + RabbitmqServerList: rabbitmqSSLEndpointListCommaSeparated, + CollectorServerList: collectorServerList, + RabbitmqUser: rabbitmqSecretUser, + RabbitmqPassword: rabbitmqSecretPassword, + RabbitmqVhost: rabbitmqSecretVhost, + LogLevel: devicemanagerConfig.LogLevel, + FabricMgmtIP: fabricMgmtIP, + CAFilePath: certificates.SignerCAFilepath, + DeviceManagerIntrospectPort: strconv.Itoa(*devicemanagerConfig.DeviceManagerIntrospectPort), + }) + if err != nil { + return err + } + data["devicemanager."+podList.Items[idx].Status.PodIP] = configDevicemanagerConfigBuffer.String() + + var fabricAnsibleConfigBuffer bytes.Buffer + err = configtemplates.FabricAnsibleConf.Execute(&fabricAnsibleConfigBuffer, struct { + HostIP string + CollectorServerList string + LogLevel string + CAFilePath string + }{ + HostIP: podList.Items[idx].Status.PodIP, + CollectorServerList: collectorServerList, + LogLevel: devicemanagerConfig.LogLevel, + CAFilePath: certificates.SignerCAFilepath, + }) + if err != nil { + return err + } + data["contrail-fabric-ansible.conf."+podList.Items[idx].Status.PodIP] = fabricAnsibleConfigBuffer.String() + + var configKeystoneAuthConfBuffer bytes.Buffer + err = configtemplates.ConfigKeystoneAuthConf.Execute(&configKeystoneAuthConfBuffer, struct { + AdminUsername string + AdminPassword string + KeystoneAddress string + KeystonePort int + KeystoneAuthProtocol string + KeystoneUserDomainName string + KeystoneProjectDomainName string + KeystoneRegion string + CAFilePath string + }{ + AdminUsername: devicemanagerAuth.AdminUsername, + AdminPassword: devicemanagerAuth.AdminPassword, + KeystoneAddress: devicemanagerAuth.Address, + KeystonePort: devicemanagerAuth.Port, + KeystoneAuthProtocol: devicemanagerAuth.AuthProtocol, + KeystoneUserDomainName: devicemanagerAuth.UserDomainName, + KeystoneProjectDomainName: devicemanagerAuth.ProjectDomainName, + KeystoneRegion: devicemanagerAuth.Region, + CAFilePath: certificates.SignerCAFilepath, + }) + if err != nil { + return err + } + data["contrail-keystone-auth.conf"] = configKeystoneAuthConfBuffer.String() + + data["dnsmasq."+podList.Items[idx].Status.PodIP] = configtemplates.ConfigDNSMasqConfig + + } + configMapInstanceDynamicConfig.Data = data + err = client.Update(context.TODO(), configMapInstanceDynamicConfig) + if err != nil { + return err + } + + return nil +} + +func (c *Devicemanager) ConfigurationParameters() DevicemanagerConfiguration { + devicemanagerConfiguration := DevicemanagerConfiguration{} + var rabbitmqUser string + var rabbitmqPassword string + var rabbitmqVhost string + var logLevel string + if c.Spec.ServiceConfiguration.LogLevel != "" { + logLevel = c.Spec.ServiceConfiguration.LogLevel + } else { + logLevel = LogLevel + } + devicemanagerConfiguration.LogLevel = logLevel + + var deviceManagerIntrospectPort int + if c.Spec.ServiceConfiguration.DeviceManagerIntrospectPort != nil { + deviceManagerIntrospectPort = *c.Spec.ServiceConfiguration.DeviceManagerIntrospectPort + } else { + deviceManagerIntrospectPort = ConfigDeviceManagerIntrospectPort + } + devicemanagerConfiguration.DeviceManagerIntrospectPort = &deviceManagerIntrospectPort + + if c.Spec.ServiceConfiguration.NodeManager != nil { + devicemanagerConfiguration.NodeManager = c.Spec.ServiceConfiguration.NodeManager + } else { + nodeManager := true + devicemanagerConfiguration.NodeManager = &nodeManager + } + + if c.Spec.ServiceConfiguration.RabbitmqUser != "" { + rabbitmqUser = c.Spec.ServiceConfiguration.RabbitmqUser + } else { + rabbitmqUser = RabbitmqUser + } + devicemanagerConfiguration.RabbitmqUser = rabbitmqUser + + if c.Spec.ServiceConfiguration.RabbitmqPassword != "" { + rabbitmqPassword = c.Spec.ServiceConfiguration.RabbitmqPassword + } else { + rabbitmqPassword = RabbitmqPassword + } + devicemanagerConfiguration.RabbitmqPassword = rabbitmqPassword + + if c.Spec.ServiceConfiguration.RabbitmqVhost != "" { + rabbitmqVhost = c.Spec.ServiceConfiguration.RabbitmqVhost + } else { + rabbitmqVhost = RabbitmqVhost + } + devicemanagerConfiguration.RabbitmqVhost = rabbitmqVhost + + devicemanagerConfiguration.AuthMode = c.Spec.ServiceConfiguration.AuthMode + if devicemanagerConfiguration.AuthMode == "" { + devicemanagerConfiguration.AuthMode = AuthenticationModeNoAuth + } + + devicemanagerConfiguration.AAAMode = c.Spec.ServiceConfiguration.AAAMode + if devicemanagerConfiguration.AAAMode == "" { + devicemanagerConfiguration.AAAMode = AAAModeNoAuth + if devicemanagerConfiguration.AuthMode == AuthenticationModeKeystone { + devicemanagerConfiguration.AAAMode = AAAModeRBAC + } + } + + return devicemanagerConfiguration +} + +type DevicemanagerAuthParameters struct { + AdminUsername string + AdminPassword string + Address string + Port int + Region string + AuthProtocol string + UserDomainName string + ProjectDomainName string +} + +func (c *Devicemanager) AuthParameters(client client.Client) (*DevicemanagerAuthParameters, error) { + w := &DevicemanagerAuthParameters{ + AdminUsername: "admin", + } + adminPasswordSecretName := c.Spec.ServiceConfiguration.KeystoneSecretName + adminPasswordSecret := &corev1.Secret{} + if err := client.Get(context.TODO(), types.NamespacedName{Name: adminPasswordSecretName, Namespace: c.Namespace}, adminPasswordSecret); err != nil { + return nil, err + } + w.AdminPassword = string(adminPasswordSecret.Data["password"]) + + if c.Spec.ServiceConfiguration.AuthMode == AuthenticationModeKeystone { + keystoneInstanceName := c.Spec.ServiceConfiguration.KeystoneInstance + keystone := &Keystone{} + if err := client.Get(context.TODO(), types.NamespacedName{Namespace: c.Namespace, Name: keystoneInstanceName}, keystone); err != nil { + return nil, err + } + if keystone.Status.Endpoint == "" { + return nil, fmt.Errorf("%q Status.Endpoint empty", keystoneInstanceName) + } + w.Port = keystone.Spec.ServiceConfiguration.ListenPort + w.Region = keystone.Spec.ServiceConfiguration.Region + w.AuthProtocol = keystone.Spec.ServiceConfiguration.AuthProtocol + w.UserDomainName = keystone.Spec.ServiceConfiguration.UserDomainName + w.ProjectDomainName = keystone.Spec.ServiceConfiguration.ProjectDomainName + w.Address = keystone.Status.Endpoint + } + + return w, nil +} + +// CurrentConfigMapExists checks if a current configuration exists and returns it. +func (c *Devicemanager) CurrentConfigMapExists(configMapName string, + client client.Client, + scheme *runtime.Scheme, + request reconcile.Request) (corev1.ConfigMap, bool) { + return CurrentConfigMapExists(configMapName, + client, + scheme, + request) +} + +func (c *Devicemanager) CreateConfigMap(configMapName string, + client client.Client, + scheme *runtime.Scheme, + request reconcile.Request) (*corev1.ConfigMap, error) { + return CreateConfigMap(configMapName, + client, + scheme, + request, + "devicemanager", + c) +} + +// CreateSecret creates a secret. +func (c *Devicemanager) CreateSecret(secretName string, + client client.Client, + scheme *runtime.Scheme, + request reconcile.Request) (*corev1.Secret, error) { + return CreateSecret(secretName, + client, + scheme, + request, + "devicemanager", + c) +} + +// PrepareSTS prepares the intented statefulset for the devicemanager object +func (c *Devicemanager) PrepareSTS(sts *appsv1.StatefulSet, commonConfiguration *PodConfiguration, request reconcile.Request, scheme *runtime.Scheme, client client.Client) error { + return PrepareSTS(sts, commonConfiguration, "devicemanager", request, scheme, c, client, true) +} + +// AddVolumesToIntendedSTS adds volumes to the devicemanager statefulset +func (c *Devicemanager) AddVolumesToIntendedSTS(sts *appsv1.StatefulSet, volumeConfigMapMap map[string]string) { + AddVolumesToIntendedSTS(sts, volumeConfigMapMap) +} + +// AddSecretVolumesToIntendedSTS adds volumes to the Rabbitmq deployment. +func (c *Devicemanager) AddSecretVolumesToIntendedSTS(sts *appsv1.StatefulSet, volumeConfigMapMap map[string]string) { + AddSecretVolumesToIntendedSTS(sts, volumeConfigMapMap) +} + +//CreateSTS creates the STS +func (c *Devicemanager) CreateSTS(sts *appsv1.StatefulSet, instanceType string, request reconcile.Request, reconcileClient client.Client) error { + return CreateSTS(sts, instanceType, request, reconcileClient) +} + +//UpdateSTS updates the STS +func (c *Devicemanager) UpdateSTS(sts *appsv1.StatefulSet, instanceType string, request reconcile.Request, reconcileClient client.Client, strategy string) error { + return UpdateSTS(sts, instanceType, request, reconcileClient, strategy) +} + +// SetInstanceActive sets the Cassandra instance to active +func (c *Devicemanager) SetInstanceActive(client client.Client, activeStatus *bool, sts *appsv1.StatefulSet, request reconcile.Request) error { + if err := client.Get(context.TODO(), types.NamespacedName{Name: sts.Name, Namespace: request.Namespace}, + sts); err != nil { + return err + } + + *activeStatus = false + acceptableReadyReplicaCnt := int32(1) + if sts.Spec.Replicas != nil { + acceptableReadyReplicaCnt = *sts.Spec.Replicas/2 + 1 + } + + if sts.Status.ReadyReplicas >= acceptableReadyReplicaCnt { + *activeStatus = true + } + + if err := client.Status().Update(context.TODO(), c); err != nil { + return err + } + return nil +} + +// PodIPListAndIPMapFromInstance gets a list with POD IPs and a map of POD names and IPs. +func (c *Devicemanager) PodIPListAndIPMapFromInstance(request reconcile.Request, reconcileClient client.Client) (*corev1.PodList, map[string]string, error) { + return PodIPListAndIPMapFromInstance("devicemanager", &c.Spec.CommonConfiguration, request, reconcileClient, true, true, false, false, false, false) +} + +//PodsCertSubjects gets list of Devicemanager pods certificate subjets which can be passed to the certificate API +func (c *Devicemanager) PodsCertSubjects(podList *corev1.PodList) []certificates.CertificateSubject { + var altIPs PodAlternativeIPs + return PodsCertSubjects(podList, c.Spec.CommonConfiguration.HostNetwork, altIPs) +} + +func (c *Devicemanager) SetPodsToReady(podIPList *corev1.PodList, client client.Client) error { + return SetPodsToReady(podIPList, client) +} + +func (c *Devicemanager) WaitForPeerPods(request reconcile.Request, reconcileClient client.Client) error { + labelSelector := labels.SelectorFromSet(map[string]string{"devicemanager": request.Name}) + listOps := &client.ListOptions{Namespace: request.Namespace, LabelSelector: labelSelector} + list := &corev1.PodList{} + err := reconcileClient.List(context.TODO(), list, listOps) + if err != nil { + return err + } + sort.SliceStable(list.Items, func(i, j int) bool { return list.Items[i].Name < list.Items[j].Name }) + for idx, pod := range list.Items { + ready := true + for i := 0; i < idx; i++ { + for _, containerStatus := range list.Items[i].Status.ContainerStatuses { + if !containerStatus.Ready { + ready = false + } + } + } + if ready { + podTOUpdate := &corev1.Pod{} + err := reconcileClient.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, podTOUpdate) + if err != nil { + return err + } + podTOUpdate.ObjectMeta.Labels["peers_ready"] = "true" + err = reconcileClient.Update(context.TODO(), podTOUpdate) + if err != nil { + return err + } + } + } + return nil +} + +func (c *Devicemanager) ManageNodeStatus(podNameIPMap map[string]string, client client.Client) error { + c.Status.Nodes = podNameIPMap + err := client.Status().Update(context.TODO(), c) + if err != nil { + return err + } + return nil +} + +// IsActive returns true if instance is active +func (c *Devicemanager) IsActive(name string, namespace string, myclient client.Client) bool { + labelSelector := labels.SelectorFromSet(map[string]string{"contrail_cluster": name}) + listOps := &client.ListOptions{Namespace: namespace, LabelSelector: labelSelector} + list := &ConfigList{} + err := myclient.List(context.TODO(), list, listOps) + if err != nil { + return false + } + if len(list.Items) > 0 { + if list.Items[0].Status.Active != nil { + if *list.Items[0].Status.Active { + return true + } + } + } + return false +} + +func (c *Devicemanager) SetEndpointInStatus(client client.Client, clusterIP string) error { + c.Status.Endpoint = clusterIP + err := client.Status().Update(context.TODO(), c) + return err +} diff --git a/pkg/apis/contrail/v1alpha1/exec_to_pod.go b/pkg/apis/contrail/v1alpha1/exec_to_pod.go index b1b0c7f9e..dcd8e1867 100644 --- a/pkg/apis/contrail/v1alpha1/exec_to_pod.go +++ b/pkg/apis/contrail/v1alpha1/exec_to_pod.go @@ -4,107 +4,20 @@ import ( "bytes" "fmt" "io" - "os" - "os/user" - "path/filepath" core_v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client/config" // "k8s.io/client-go/kubernetes/scheme" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/remotecommand" ) const debug = false -// GetConfig creates a *rest.Config for talking to a Kubernetes apiserver. -// If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running -// in cluster and use the cluster provided kubeconfig. -// -// Config precedence -// -// * --kubeconfig flag pointing at a file -// -// * KUBECONFIG environment variable pointing at a file -// -// * In-cluster config if running in cluster -// -// * $HOME/.kube/config if exists -func GetConfig() (*rest.Config, error) { - var kubeconfig, masterURL string - // If a flag is specified with the config location, use that - if len(kubeconfig) > 0 { - return clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) - } - // If an env variable is specified with the config locaiton, use that - if len(os.Getenv("KUBECONFIG")) > 0 { - return clientcmd.BuildConfigFromFlags(masterURL, os.Getenv("KUBECONFIG")) - } - // If no explicit location, try the in-cluster config - if c, err := rest.InClusterConfig(); err == nil { - return c, nil - } - // If no in-cluster config, try the default location in the user's home directory - if usr, err := user.Current(); err == nil { - if c, err := clientcmd.BuildConfigFromFlags( - "", filepath.Join(usr.HomeDir, ".kube", "config")); err == nil { - return c, nil - } - } - - return nil, fmt.Errorf("could not locate a kubeconfig") -} - -// GetClientConfig first tries to get a config object which uses the service account kubernetes gives to pods, -// if it is called from a process running in a kubernetes environment. -// Otherwise, it tries to build config from a default kubeconfig filepath if it fails, it fallback to the default config. -// Once it get the config, it returns the same. -func GetClientConfig() (*rest.Config, error) { - config, err := rest.InClusterConfig() - if err != nil { - if debug { - fmt.Printf("Unable to create config. Error: %+v\n", err) - } - err1 := err - kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") - config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) - if err != nil { - err = fmt.Errorf("InClusterConfig as well as BuildConfigFromFlags Failed. Error in InClusterConfig: %+v\nError in BuildConfigFromFlags: %+v", err1, err) - return nil, err - } - } - - return config, nil -} - -// GetClientsetFromConfig takes REST config and Create a clientset based on that and return that clientset. -func GetClientsetFromConfig(config *rest.Config) (*kubernetes.Clientset, error) { - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - err = fmt.Errorf("failed creating clientset. Error: %+v", err) - return nil, err - } - - return clientset, nil -} - -// GetClientset first tries to get a config object which uses the service account kubernetes gives to pods, -// if it is called from a process running in a kubernetes environment. -// Otherwise, it tries to build config from a default kubeconfig filepath if it fails, it fallback to the default config. -// Once it get the config, it creates a new Clientset for the given config and returns the clientset. -func GetClientset() (*kubernetes.Clientset, error) { - config, err := GetClientConfig() - if err != nil { - return nil, err - } - - return GetClientsetFromConfig(config) -} - // GetDynamicClientFromConfig takes REST config and Create a dynamic client based on that return that client. func GetDynamicClientFromConfig(config *rest.Config) (dynamic.Interface, error) { dynamicClient, err := dynamic.NewForConfig(config) @@ -119,7 +32,7 @@ func GetDynamicClientFromConfig(config *rest.Config) (dynamic.Interface, error) // Otherwise, it tries to build config from a default kubeconfig filepath if it fails, it fallback to the default config. // Once it get the config, it creates a new Dynamic Client for the given config and returns the client. func GetDynamicClient() (dynamic.Interface, error) { - config, err := GetClientConfig() + config, err := config.GetConfig() if err != nil { return nil, err } @@ -132,7 +45,7 @@ func GetDynamicClient() (dynamic.Interface, error) { // Otherwise, it tries to build config from a default kubeconfig filepath if it fails, it fallback to the default config. // Once it get the config, it func GetRESTClient() (*rest.RESTClient, error) { - config, err := GetClientConfig() + config, err := config.GetConfig() if err != nil { return &rest.RESTClient{}, err } @@ -149,12 +62,12 @@ func GetRESTClient() (*rest.RESTClient, error) { // string: Errors. (STDERR) // error: If any error has occurred otherwise `nil` func ExecToPodThroughAPI(command []string, containerName, podName, namespace string, stdin io.Reader) (string, string, error) { - config, err := GetClientConfig() + config, err := config.GetConfig() if err != nil { return "", "", err } - clientset, err := GetClientsetFromConfig(config) + clientset, err := kubernetes.NewForConfig(config) if err != nil { return "", "", err } diff --git a/pkg/apis/contrail/v1alpha1/keystone_types.go b/pkg/apis/contrail/v1alpha1/keystone_types.go index f29a5ed03..81eff2378 100644 --- a/pkg/apis/contrail/v1alpha1/keystone_types.go +++ b/pkg/apis/contrail/v1alpha1/keystone_types.go @@ -1,8 +1,12 @@ package v1alpha1 import ( + "context" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/Juniper/contrail-operator/pkg/certificates" ) @@ -121,3 +125,59 @@ func (k *Keystone) SetDefaultValues() { func init() { SchemeBuilder.Register(&Keystone{}, &KeystoneList{}) } + +// IsActive returns true if instance is active. +func (k *Keystone) IsActive(name string, namespace string, client client.Client) bool { + instance := &Keystone{} + err := client.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, instance) + if err != nil { + return false + } + return instance.Status.Active +} + +// ConfigurationParameters sets the default for the configuration parameters. +func (k *Keystone) ConfigurationParameters() KeystoneConfiguration { + keystoneConfiguration := KeystoneConfiguration{} + if k.Spec.ServiceConfiguration.ListenPort == 0 { + keystoneConfiguration.ListenPort = KeystoneAuthPublicPort + } else { + keystoneConfiguration.ListenPort = k.Spec.ServiceConfiguration.ListenPort + } + if k.Spec.ServiceConfiguration.Region == "" { + keystoneConfiguration.Region = KeystoneAuthRegionName + } else { + keystoneConfiguration.Region = k.Spec.ServiceConfiguration.Region + } + if k.Spec.ServiceConfiguration.AuthProtocol == "" { + keystoneConfiguration.AuthProtocol = KeystoneAuthProto + } else { + keystoneConfiguration.AuthProtocol = k.Spec.ServiceConfiguration.AuthProtocol + } + if k.Spec.ServiceConfiguration.UserDomainName == "" { + keystoneConfiguration.UserDomainName = KeystoneAuthUserDomainName + } else { + keystoneConfiguration.UserDomainName = k.Spec.ServiceConfiguration.UserDomainName + } + if k.Spec.ServiceConfiguration.UserDomainID == "" { + keystoneConfiguration.UserDomainID = KeystoneAuthUserDomainID + } else { + keystoneConfiguration.UserDomainID = k.Spec.ServiceConfiguration.UserDomainID + } + if k.Spec.ServiceConfiguration.ProjectDomainName == "" { + keystoneConfiguration.ProjectDomainName = KeystoneAuthProjectDomainName + } else { + keystoneConfiguration.ProjectDomainName = k.Spec.ServiceConfiguration.ProjectDomainName + } + if k.Spec.ServiceConfiguration.ProjectDomainID == "" { + keystoneConfiguration.ProjectDomainID = KeystoneAuthProjectDomainID + } else { + keystoneConfiguration.ProjectDomainID = k.Spec.ServiceConfiguration.ProjectDomainID + } + if k.Spec.ServiceConfiguration.ExternalAddressRetrySec == 0 { + keystoneConfiguration.ExternalAddressRetrySec = KeystoneExtRetrySec + } else { + keystoneConfiguration.ExternalAddressRetrySec = k.Spec.ServiceConfiguration.ExternalAddressRetrySec + } + return keystoneConfiguration +} diff --git a/pkg/apis/contrail/v1alpha1/kubemanager_types.go b/pkg/apis/contrail/v1alpha1/kubemanager_types.go index e968f8863..724f52111 100644 --- a/pkg/apis/contrail/v1alpha1/kubemanager_types.go +++ b/pkg/apis/contrail/v1alpha1/kubemanager_types.go @@ -11,8 +11,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configtemplates "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates" "github.com/Juniper/contrail-operator/pkg/certificates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -62,26 +62,27 @@ type KubemanagerServiceConfiguration struct { // KubemanagerConfiguration is the configuration for the kubemanagers API. // +k8s:openapi-gen=true type KubemanagerConfiguration struct { - Containers []*Container `json:"containers,omitempty"` - UseKubeadmConfig *bool `json:"useKubeadmConfig,omitempty"` - ServiceAccount string `json:"serviceAccount,omitempty"` - ClusterRole string `json:"clusterRole,omitempty"` - ClusterRoleBinding string `json:"clusterRoleBinding,omitempty"` - CloudOrchestrator string `json:"cloudOrchestrator,omitempty"` - KubernetesAPIServer string `json:"kubernetesAPIServer,omitempty"` - KubernetesAPIPort *int `json:"kubernetesAPIPort,omitempty"` - KubernetesAPISSLPort *int `json:"kubernetesAPISSLPort,omitempty"` - PodSubnets string `json:"podSubnets,omitempty"` - ServiceSubnets string `json:"serviceSubnets,omitempty"` - KubernetesClusterName string `json:"kubernetesClusterName,omitempty"` - IPFabricSubnets string `json:"ipFabricSubnets,omitempty"` - IPFabricForwarding *bool `json:"ipFabricForwarding,omitempty"` - IPFabricSnat *bool `json:"ipFabricSnat,omitempty"` - KubernetesTokenFile string `json:"kubernetesTokenFile,omitempty"` - HostNetworkService *bool `json:"hostNetworkService,omitempty"` - RabbitmqUser string `json:"rabbitmqUser,omitempty"` - RabbitmqPassword string `json:"rabbitmqPassword,omitempty"` - RabbitmqVhost string `json:"rabbitmqVhost,omitempty"` + Containers []*Container `json:"containers,omitempty"` + UseKubeadmConfig *bool `json:"useKubeadmConfig,omitempty"` + ServiceAccount string `json:"serviceAccount,omitempty"` + ClusterRole string `json:"clusterRole,omitempty"` + ClusterRoleBinding string `json:"clusterRoleBinding,omitempty"` + CloudOrchestrator string `json:"cloudOrchestrator,omitempty"` + KubernetesAPIServer string `json:"kubernetesAPIServer,omitempty"` + KubernetesAPIPort *int `json:"kubernetesAPIPort,omitempty"` + KubernetesAPISSLPort *int `json:"kubernetesAPISSLPort,omitempty"` + PodSubnets string `json:"podSubnets,omitempty"` + ServiceSubnets string `json:"serviceSubnets,omitempty"` + KubernetesClusterName string `json:"kubernetesClusterName,omitempty"` + IPFabricSubnets string `json:"ipFabricSubnets,omitempty"` + IPFabricForwarding *bool `json:"ipFabricForwarding,omitempty"` + IPFabricSnat *bool `json:"ipFabricSnat,omitempty"` + KubernetesTokenFile string `json:"kubernetesTokenFile,omitempty"` + HostNetworkService *bool `json:"hostNetworkService,omitempty"` + RabbitmqUser string `json:"rabbitmqUser,omitempty"` + RabbitmqPassword string `json:"rabbitmqPassword,omitempty"` + RabbitmqVhost string `json:"rabbitmqVhost,omitempty"` + AuthMode AuthenticationMode `json:"authMode,omitempty"` } // KubemanagerNodesConfiguration is the configuration for third party dependencies @@ -91,6 +92,7 @@ type KubemanagerNodesConfiguration struct { RabbbitmqNodesConfiguration *RabbitmqClusterConfiguration `json:"rabbitmqNodesConfiguration,omitempty"` CassandraNodesConfiguration *CassandraClusterConfiguration `json:"cassandraNodesConfiguration,omitempty"` ZookeeperNodesConfiguration *ZookeeperClusterConfiguration `json:"zookeeperNodesConfiguration,omitempty"` + KeystoneNodesConfiguration *KeystoneClusterConfiguration `json:"keystoneNodesConfiguration,omitempty"` } // KubemanagerList contains a list of Kubemanager. @@ -126,6 +128,8 @@ func (c *Kubemanager) InstanceConfiguration(request reconcile.Request, rabbitmqNodesInformation.FillWithDefaultValues() zookeeperNodesInformation := c.Spec.ServiceConfiguration.ZookeeperNodesConfiguration zookeeperNodesInformation.FillWithDefaultValues() + keystoneNodesInformation := c.Spec.ServiceConfiguration.KeystoneNodesConfiguration + keystoneNodesInformation.FillWithDefaultValues() var rabbitmqSecretUser string var rabbitmqSecretPassword string @@ -184,7 +188,6 @@ func (c *Kubemanager) InstanceConfiguration(request reconcile.Request, } kubemanagerConfig.ServiceSubnets = serviceSubnets } - sort.SliceStable(podList.Items, func(i, j int) bool { return podList.Items[i].Status.PodIP < podList.Items[j].Status.PodIP }) var data = map[string]string{} for idx := range podList.Items { @@ -266,13 +269,23 @@ func (c *Kubemanager) InstanceConfiguration(request reconcile.Request, var vncApiConfigBuffer bytes.Buffer configtemplates.KubemanagerAPIVNC.Execute(&vncApiConfigBuffer, struct { - ListenAddress string - ListenPort string - CAFilePath string + ListenAddress string + ListenPort string + CAFilePath string + AuthMode AuthenticationMode + KeystoneAuthProtocol string + KeystoneAddress string + KeystonePort int + KeystoneUserDomainName string }{ - ListenAddress: podList.Items[idx].Status.PodIP, - ListenPort: strconv.Itoa(configNodesInformation.APIServerPort), - CAFilePath: certificates.SignerCAFilepath, + ListenAddress: podList.Items[idx].Status.PodIP, + ListenPort: strconv.Itoa(configNodesInformation.APIServerPort), + CAFilePath: certificates.SignerCAFilepath, + AuthMode: c.Spec.ServiceConfiguration.AuthMode, + KeystoneAuthProtocol: keystoneNodesInformation.AuthProtocol, + KeystoneAddress: keystoneNodesInformation.Endpoint, + KeystonePort: keystoneNodesInformation.Port, + KeystoneUserDomainName: keystoneNodesInformation.UserDomainName, }) data["vnc."+podList.Items[idx].Status.PodIP] = vncApiConfigBuffer.String() } @@ -454,6 +467,10 @@ func (c *Kubemanager) ConfigurationParameters() KubemanagerConfiguration { ipFabricSnat = KubernetesIPFabricSnat } + if c.Spec.ServiceConfiguration.AuthMode == "" { + c.Spec.ServiceConfiguration.AuthMode = AuthenticationModeNoAuth + } + kubemanagerConfiguration.CloudOrchestrator = cloudOrchestrator kubemanagerConfiguration.KubernetesAPIServer = kubernetesApiServer kubemanagerConfiguration.KubernetesAPIPort = &kubernetesApiPort diff --git a/pkg/apis/contrail/v1alpha1/kubemanager_types_test.go b/pkg/apis/contrail/v1alpha1/kubemanager_types_test.go index 8dc904942..c8599c3dd 100644 --- a/pkg/apis/contrail/v1alpha1/kubemanager_types_test.go +++ b/pkg/apis/contrail/v1alpha1/kubemanager_types_test.go @@ -164,6 +164,7 @@ func TestInstanceConfigurationWithStaticConfiguration(t *testing.T) { ServerIPList: []string{"7.7.7.7", "8.8.8.8"}, ClientPort: 4444, }, + KeystoneNodesConfiguration: &KeystoneClusterConfiguration{}, }, }, }, diff --git a/pkg/apis/contrail/v1alpha1/manager_types.go b/pkg/apis/contrail/v1alpha1/manager_types.go index c4b230f8d..24af62df5 100644 --- a/pkg/apis/contrail/v1alpha1/manager_types.go +++ b/pkg/apis/contrail/v1alpha1/manager_types.go @@ -23,32 +23,32 @@ type ManagerSpec struct { // Services defines the desired state of Services. // +k8s:openapi-gen=true type Services struct { - Config *Config `json:"config,omitempty"` - Controls []*Control `json:"controls,omitempty"` - Kubemanagers []*KubemanagerService `json:"kubemanagers,omitempty"` - Webui *Webui `json:"webui,omitempty"` - Vrouters []*VrouterService `json:"vrouters,omitempty"` - Cassandras []*Cassandra `json:"cassandras,omitempty"` - Zookeepers []*Zookeeper `json:"zookeepers,omitempty"` - Rabbitmq *Rabbitmq `json:"rabbitmq,omitempty"` - ProvisionManager *ProvisionManager `json:"provisionManager,omitempty"` - Command *Command `json:"command,omitempty"` - Postgres *Postgres `json:"postgres,omitempty"` - Keystone *Keystone `json:"keystone,omitempty"` - Swift *Swift `json:"swift,omitempty"` - Memcached *Memcached `json:"memcached,omitempty"` - Contrailmonitor *Contrailmonitor `json:"contrailmonitor,omitempty"` - ContrailCNIs []*ContrailCNI `json:"contrailCNIs,omitempty"` + Config *Config `json:"config,omitempty"` + Controls []*Control `json:"controls,omitempty"` + Kubemanagers []*KubemanagerService `json:"kubemanagers,omitempty"` + Webui *Webui `json:"webui,omitempty"` + Vrouters []*VrouterService `json:"vrouters,omitempty"` + Cassandras []*Cassandra `json:"cassandras,omitempty"` + Zookeepers []*Zookeeper `json:"zookeepers,omitempty"` + Rabbitmq *Rabbitmq `json:"rabbitmq,omitempty"` + ProvisionManager *ProvisionManagerService `json:"provisionManager,omitempty"` + Command *Command `json:"command,omitempty"` + Postgres *Postgres `json:"postgres,omitempty"` + Keystone *Keystone `json:"keystone,omitempty"` + Swift *Swift `json:"swift,omitempty"` + Memcached *Memcached `json:"memcached,omitempty"` + Contrailmonitor *Contrailmonitor `json:"contrailmonitor,omitempty"` + ContrailCNIs []*ContrailCNI `json:"contrailCNIs,omitempty"` } -// VrouterService defines desired confgiuration of vRouter +// VrouterService defines desired configuration of vRouter // +k8s:openapi-gen=true type VrouterService struct { metav1.ObjectMeta `json:"metadata,omitempty"` Spec VrouterServiceSpec `json:"spec,omitempty"` } -// VrouterServiceSpec defines desired spec confgiuration of vRouter +// VrouterServiceSpec defines desired spec configuration of vRouter // +k8s:openapi-gen=true type VrouterServiceSpec struct { CommonConfiguration PodConfiguration `json:"commonConfiguration,omitempty"` @@ -62,14 +62,28 @@ type VrouterManagerServiceConfiguration struct { VrouterConfiguration `json:",inline"` } -// KubemanagerService defines desired configuration of vRouter +// ProvisionManagerService defines desired configuration of ProvisionManager +// +k8s:openapi-gen=true +type ProvisionManagerService struct { + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec ProvisionManagerServiceSpec `json:"spec,omitempty"` +} + +// ProvisionManagerServiceSpec defines desired spec configuration of ProvisionManager +// +k8s:openapi-gen=true +type ProvisionManagerServiceSpec struct { + CommonConfiguration PodConfiguration `json:"commonConfiguration,omitempty"` + ServiceConfiguration ProvisionManagerConfiguration `json:"serviceConfiguration"` +} + +// KubemanagerService defines desired configuration of Kubemanager // +k8s:openapi-gen=true type KubemanagerService struct { metav1.ObjectMeta `json:"metadata,omitempty"` Spec KubemanagerServiceSpec `json:"spec,omitempty"` } -// KubemanagerServiceSpec defines desired spec configuration of vRouter +// KubemanagerServiceSpec defines desired spec configuration of Kubemanager // +k8s:openapi-gen=true type KubemanagerServiceSpec struct { CommonConfiguration PodConfiguration `json:"commonConfiguration,omitempty"` @@ -81,6 +95,7 @@ type KubemanagerServiceSpec struct { type KubemanagerManagerServiceConfiguration struct { CassandraInstance string `json:"cassandraInstance,omitempty"` ZookeeperInstance string `json:"zookeeperInstance,omitempty"` + KeystoneInstance string `json:"keystoneInstance,omitempty"` KubemanagerConfiguration `json:",inline"` } diff --git a/pkg/apis/contrail/v1alpha1/manager_types_test.go b/pkg/apis/contrail/v1alpha1/manager_types_test.go index a73e730a1..ebf385329 100644 --- a/pkg/apis/contrail/v1alpha1/manager_types_test.go +++ b/pkg/apis/contrail/v1alpha1/manager_types_test.go @@ -96,7 +96,7 @@ func TestManagerTypeTwo(t *testing.T) { }, Spec: contrail.ManagerSpec{ Services: contrail.Services{ - ProvisionManager: provisionmanager, + ProvisionManager: provisionmanagerService, }, KeystoneSecretName: "keystone-adminpass-secret", }, @@ -127,7 +127,7 @@ var managerCR = &contrail.Manager{ Zookeepers: []*contrail.Zookeeper{zookeeper}, Controls: []*contrail.Control{control}, Kubemanagers: []*contrail.KubemanagerService{kubemanagerService}, - ProvisionManager: provisionmanager, + ProvisionManager: provisionmanagerService, Webui: webui, Config: config, Command: command, @@ -244,13 +244,13 @@ var rabbitmq = &contrail.Rabbitmq{ Status: contrail.RabbitmqStatus{Active: &falseVal}, } -var provisionmanager = &contrail.ProvisionManager{ +var provisionmanagerService = &contrail.ProvisionManagerService{ ObjectMeta: meta.ObjectMeta{ Name: "provisionmanager", Namespace: "default", Labels: map[string]string{"contrail_cluster": "cluster1"}, }, - Spec: contrail.ProvisionManagerSpec{ + Spec: contrail.ProvisionManagerServiceSpec{ CommonConfiguration: contrail.PodConfiguration{ Replicas: &replicas, }, diff --git a/pkg/apis/contrail/v1alpha1/provisionmanager_types.go b/pkg/apis/contrail/v1alpha1/provisionmanager_types.go index 1db759ed4..224f000a1 100644 --- a/pkg/apis/contrail/v1alpha1/provisionmanager_types.go +++ b/pkg/apis/contrail/v1alpha1/provisionmanager_types.go @@ -17,8 +17,8 @@ import ( runtimeClient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configtemplates "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates" "github.com/Juniper/contrail-operator/pkg/certificates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! @@ -27,8 +27,15 @@ import ( // ProvisionManagerSpec defines the desired state of ProvisionManager // +k8s:openapi-gen=true type ProvisionManagerSpec struct { - CommonConfiguration PodConfiguration `json:"commonConfiguration,omitempty"` - ServiceConfiguration ProvisionManagerConfiguration `json:"serviceConfiguration"` + CommonConfiguration PodConfiguration `json:"commonConfiguration,omitempty"` + ServiceConfiguration ProvisionManagerServiceConfiguration `json:"serviceConfiguration"` +} + +// ProvisionManagerServiceConfiguration is the Spec for the provisionmanagers API. +// +k8s:openapi-gen=true +type ProvisionManagerServiceConfiguration struct { + ProvisionManagerConfiguration `json:",inline"` + ProvisionManagerNodesConfiguration `json:",inline"` } // ProvisionManagerConfiguration defines the provision manager configuration @@ -55,6 +62,12 @@ type GlobalVrouterConfiguration struct { VxlanNetworkIdentifierMode string `json:"vxlanNetworkIdentifierMode,omitempty"` } +// ProvisionManagerNodesConfiguration is the configuration for third party dependencies +// +k8s:openapi-gen=true +type ProvisionManagerNodesConfiguration struct { + ConfigNodesConfiguration *ConfigClusterConfiguration `json:"configNodesConfiguration,omitempty"` +} + // ProvisionManagerStatus defines the observed state of ProvisionManager // +k8s:openapi-gen=true type ProvisionManagerStatus struct { @@ -312,11 +325,8 @@ func (c *ProvisionManager) InstanceConfiguration(request reconcile.Request, return err } - configNodesInformation, err := NewConfigClusterConfiguration(c.Labels["contrail_cluster"], - request.Namespace, client) - if err != nil { - return err - } + configNodesInformation := c.Spec.ServiceConfiguration.ConfigNodesConfiguration + configNodesInformation.FillWithDefaultValues() listOps := &runtimeClient.ListOptions{Namespace: request.Namespace} configList := &ConfigList{} diff --git a/pkg/apis/contrail/v1alpha1/rabbitmq_types.go b/pkg/apis/contrail/v1alpha1/rabbitmq_types.go index 39ed06368..93b9a1488 100644 --- a/pkg/apis/contrail/v1alpha1/rabbitmq_types.go +++ b/pkg/apis/contrail/v1alpha1/rabbitmq_types.go @@ -16,8 +16,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configtemplates "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates" "github.com/Juniper/contrail-operator/pkg/certificates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/apis/contrail/v1alpha1/service.go b/pkg/apis/contrail/v1alpha1/service.go new file mode 100644 index 000000000..41ecac81e --- /dev/null +++ b/pkg/apis/contrail/v1alpha1/service.go @@ -0,0 +1,19 @@ +package v1alpha1 + +// Service is the configuration of the service that exposes a workload +// +k8s:openapi-gen=true +type Service struct { + ServiceType serviceType `json:"serviceType,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// +kubebuilder:validation:Enum={"","ClusterIP","NodePort","LoadBalancer","ExternalName"} +type serviceType string + +func (s *Service) GetServiceType() serviceType { + return s.ServiceType +} + +func (s *Service) GetAnnotations() map[string]string { + return s.Annotations +} diff --git a/pkg/apis/contrail/v1alpha1/swiftproxy_types.go b/pkg/apis/contrail/v1alpha1/swiftproxy_types.go index 46bf4315b..a19f42cd4 100644 --- a/pkg/apis/contrail/v1alpha1/swiftproxy_types.go +++ b/pkg/apis/contrail/v1alpha1/swiftproxy_types.go @@ -25,6 +25,7 @@ type SwiftProxyConfiguration struct { SwiftConfSecretName string `json:"swiftConfSecretName,omitempty"` RingConfigMapName string `json:"ringConfigMapName,omitempty"` Containers []*Container `json:"containers,omitempty"` + Service Service `json:"service,omitempty"` // Service name registered in Keystone, default "swift" SwiftServiceName string `json:"swiftServiceName,omitempty"` } @@ -67,12 +68,31 @@ type SwiftProxyList struct { // SwiftProxyInstanceType is type unique name used for labels const SwiftProxyInstanceType = "SwiftProxy" -//PodsCertSubjects gets list of SwiftProxy pods certificate subjets which can be passed to the certificate API +// PodsCertSubjects gets list of SwiftProxy pods certificate subjets which can be passed to the certificate API func (s *SwiftProxy) PodsCertSubjects(podList *corev1.PodList, serviceIP string) []certificates.CertificateSubject { altIPs := PodAlternativeIPs{ServiceIP: serviceIP} return PodsCertSubjects(podList, s.Spec.CommonConfiguration.HostNetwork, altIPs) } +// GetServiceType returns chosen Service type for Swift Proxy, default is LoadBalancer +func (s *SwiftProxy) GetServiceType() corev1.ServiceType { + st := s.Spec.ServiceConfiguration.Service.GetServiceType() + if st == "" { + return corev1.ServiceTypeLoadBalancer + } + return corev1.ServiceType(st) +} + +// GetServiceAnnotations returns annotations for Swift Proxy Service +func (s *SwiftProxy) GetServiceAnnotations() map[string]string { + annotations := s.Spec.ServiceConfiguration.Service.GetAnnotations() + if len(annotations) == 0 { + // TODO remove after adding Annotations in manager structure in CaVA + return map[string]string{"metallb.universe.tf/address-pool": "mgmt"} + } + return annotations +} + func init() { SchemeBuilder.Register(&SwiftProxy{}, &SwiftProxyList{}) } diff --git a/pkg/apis/contrail/v1alpha1/vrouter_types.go b/pkg/apis/contrail/v1alpha1/vrouter_types.go index bb5956968..94d2852d5 100644 --- a/pkg/apis/contrail/v1alpha1/vrouter_types.go +++ b/pkg/apis/contrail/v1alpha1/vrouter_types.go @@ -6,8 +6,8 @@ import ( "sort" "strconv" - configtemplates "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates" "github.com/Juniper/contrail-operator/pkg/certificates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/apis/contrail/v1alpha1/webui_types.go b/pkg/apis/contrail/v1alpha1/webui_types.go index ccd625b9b..4e721b8b8 100644 --- a/pkg/apis/contrail/v1alpha1/webui_types.go +++ b/pkg/apis/contrail/v1alpha1/webui_types.go @@ -12,8 +12,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configtemplates "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates" "github.com/Juniper/contrail-operator/pkg/certificates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/apis/contrail/v1alpha1/zookeeper_types.go b/pkg/apis/contrail/v1alpha1/zookeeper_types.go index a6a23c192..6210505bc 100644 --- a/pkg/apis/contrail/v1alpha1/zookeeper_types.go +++ b/pkg/apis/contrail/v1alpha1/zookeeper_types.go @@ -12,7 +12,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configtemplates "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates" + configtemplates "github.com/Juniper/contrail-operator/pkg/configuration" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/apis/contrail/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/contrail/v1alpha1/zz_generated.deepcopy.go index 4cbe510ba..ac72c227f 100644 --- a/pkg/apis/contrail/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/contrail/v1alpha1/zz_generated.deepcopy.go @@ -18,6 +18,13 @@ func (in *APIServer) DeepCopyInto(out *APIServer) { copy(*out, *in) } out.Encryption = in.Encryption + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } @@ -55,6 +62,7 @@ func (in *ActiveStatus) DeepCopy() *ActiveStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AnalyticsNode) DeepCopyInto(out *AnalyticsNode) { *out = *in + in.Node.DeepCopyInto(&out.Node) return } @@ -722,6 +730,7 @@ func (in *ConfigList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConfigNode) DeepCopyInto(out *ConfigNode) { *out = *in + in.Node.DeepCopyInto(&out.Node) return } @@ -1354,6 +1363,7 @@ func (in *ControlList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ControlNode) DeepCopyInto(out *ControlNode) { *out = *in + in.Node.DeepCopyInto(&out.Node) return } @@ -1486,6 +1496,7 @@ func (in *CrdStatus) DeepCopy() *CrdStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DatabaseNode) DeepCopyInto(out *DatabaseNode) { *out = *in + in.Node.DeepCopyInto(&out.Node) return } @@ -1499,6 +1510,195 @@ func (in *DatabaseNode) DeepCopy() *DatabaseNode { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Devicemanager) DeepCopyInto(out *Devicemanager) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Devicemanager. +func (in *Devicemanager) DeepCopy() *Devicemanager { + if in == nil { + return nil + } + out := new(Devicemanager) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Devicemanager) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevicemanagerAuthParameters) DeepCopyInto(out *DevicemanagerAuthParameters) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicemanagerAuthParameters. +func (in *DevicemanagerAuthParameters) DeepCopy() *DevicemanagerAuthParameters { + if in == nil { + return nil + } + out := new(DevicemanagerAuthParameters) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevicemanagerConfiguration) DeepCopyInto(out *DevicemanagerConfiguration) { + *out = *in + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]*Container, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Container) + (*in).DeepCopyInto(*out) + } + } + } + if in.DeviceManagerIntrospectPort != nil { + in, out := &in.DeviceManagerIntrospectPort, &out.DeviceManagerIntrospectPort + *out = new(int) + **out = **in + } + if in.NodeManager != nil { + in, out := &in.NodeManager, &out.NodeManager + *out = new(bool) + **out = **in + } + out.Storage = in.Storage + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicemanagerConfiguration. +func (in *DevicemanagerConfiguration) DeepCopy() *DevicemanagerConfiguration { + if in == nil { + return nil + } + out := new(DevicemanagerConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevicemanagerList) DeepCopyInto(out *DevicemanagerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Devicemanager, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicemanagerList. +func (in *DevicemanagerList) DeepCopy() *DevicemanagerList { + if in == nil { + return nil + } + out := new(DevicemanagerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DevicemanagerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevicemanagerServiceStatus) DeepCopyInto(out *DevicemanagerServiceStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicemanagerServiceStatus. +func (in *DevicemanagerServiceStatus) DeepCopy() *DevicemanagerServiceStatus { + if in == nil { + return nil + } + out := new(DevicemanagerServiceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevicemanagerSpec) DeepCopyInto(out *DevicemanagerSpec) { + *out = *in + in.CommonConfiguration.DeepCopyInto(&out.CommonConfiguration) + in.ServiceConfiguration.DeepCopyInto(&out.ServiceConfiguration) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicemanagerSpec. +func (in *DevicemanagerSpec) DeepCopy() *DevicemanagerSpec { + if in == nil { + return nil + } + out := new(DevicemanagerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevicemanagerStatus) DeepCopyInto(out *DevicemanagerStatus) { + *out = *in + if in.Active != nil { + in, out := &in.Active, &out.Active + *out = new(bool) + **out = **in + } + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ConfigChanged != nil { + in, out := &in.ConfigChanged, &out.ConfigChanged + *out = new(bool) + **out = **in + } + if in.ServiceStatus != nil { + in, out := &in.ServiceStatus, &out.ServiceStatus + *out = make(map[string]DevicemanagerServiceStatus, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicemanagerStatus. +func (in *DevicemanagerStatus) DeepCopy() *DevicemanagerStatus { + if in == nil { + return nil + } + out := new(DevicemanagerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EcmpHashingIncludeFields) DeepCopyInto(out *EcmpHashingIncludeFields) { *out = *in @@ -1686,6 +1886,22 @@ func (in *KeystoneAuthParameters) DeepCopy() *KeystoneAuthParameters { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KeystoneClusterConfiguration) DeepCopyInto(out *KeystoneClusterConfiguration) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeystoneClusterConfiguration. +func (in *KeystoneClusterConfiguration) DeepCopy() *KeystoneClusterConfiguration { + if in == nil { + return nil + } + out := new(KeystoneClusterConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KeystoneConfiguration) DeepCopyInto(out *KeystoneConfiguration) { *out = *in @@ -1938,6 +2154,11 @@ func (in *KubemanagerNodesConfiguration) DeepCopyInto(out *KubemanagerNodesConfi *out = new(ZookeeperClusterConfiguration) (*in).DeepCopyInto(*out) } + if in.KeystoneNodesConfiguration != nil { + in, out := &in.KeystoneNodesConfiguration, &out.KeystoneNodesConfiguration + *out = new(KeystoneClusterConfiguration) + **out = **in + } return } @@ -2516,6 +2737,29 @@ func (in *MonitorEncryption) DeepCopy() *MonitorEncryption { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Node) DeepCopyInto(out *Node) { + *out = *in + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Node. +func (in *Node) DeepCopy() *Node { + if in == nil { + return nil + } + out := new(Node) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodConfiguration) DeepCopyInto(out *PodConfiguration) { *out = *in @@ -2781,6 +3025,81 @@ func (in *ProvisionManagerList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProvisionManagerNodesConfiguration) DeepCopyInto(out *ProvisionManagerNodesConfiguration) { + *out = *in + if in.ConfigNodesConfiguration != nil { + in, out := &in.ConfigNodesConfiguration, &out.ConfigNodesConfiguration + *out = new(ConfigClusterConfiguration) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionManagerNodesConfiguration. +func (in *ProvisionManagerNodesConfiguration) DeepCopy() *ProvisionManagerNodesConfiguration { + if in == nil { + return nil + } + out := new(ProvisionManagerNodesConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProvisionManagerService) DeepCopyInto(out *ProvisionManagerService) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionManagerService. +func (in *ProvisionManagerService) DeepCopy() *ProvisionManagerService { + if in == nil { + return nil + } + out := new(ProvisionManagerService) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProvisionManagerServiceConfiguration) DeepCopyInto(out *ProvisionManagerServiceConfiguration) { + *out = *in + in.ProvisionManagerConfiguration.DeepCopyInto(&out.ProvisionManagerConfiguration) + in.ProvisionManagerNodesConfiguration.DeepCopyInto(&out.ProvisionManagerNodesConfiguration) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionManagerServiceConfiguration. +func (in *ProvisionManagerServiceConfiguration) DeepCopy() *ProvisionManagerServiceConfiguration { + if in == nil { + return nil + } + out := new(ProvisionManagerServiceConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProvisionManagerServiceSpec) DeepCopyInto(out *ProvisionManagerServiceSpec) { + *out = *in + in.CommonConfiguration.DeepCopyInto(&out.CommonConfiguration) + in.ServiceConfiguration.DeepCopyInto(&out.ServiceConfiguration) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionManagerServiceSpec. +func (in *ProvisionManagerServiceSpec) DeepCopy() *ProvisionManagerServiceSpec { + if in == nil { + return nil + } + out := new(ProvisionManagerServiceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProvisionManagerSpec) DeepCopyInto(out *ProvisionManagerSpec) { *out = *in @@ -3016,6 +3335,29 @@ func (in *RabbitmqStatusPorts) DeepCopy() *RabbitmqStatusPorts { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Service) DeepCopyInto(out *Service) { + *out = *in + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Service. +func (in *Service) DeepCopy() *Service { + if in == nil { + return nil + } + out := new(Service) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceStatus) DeepCopyInto(out *ServiceStatus) { *out = *in @@ -3122,7 +3464,7 @@ func (in *Services) DeepCopyInto(out *Services) { } if in.ProvisionManager != nil { in, out := &in.ProvisionManager, &out.ProvisionManager - *out = new(ProvisionManager) + *out = new(ProvisionManagerService) (*in).DeepCopyInto(*out) } if in.Command != nil { @@ -3166,6 +3508,7 @@ func (in *Services) DeepCopyInto(out *Services) { } } } + in.Service.DeepCopyInto(&out.Service) return } @@ -3744,6 +4087,7 @@ func (in *VrouterManagerServiceConfiguration) DeepCopy() *VrouterManagerServiceC // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VrouterNode) DeepCopyInto(out *VrouterNode) { *out = *in + in.Node.DeepCopyInto(&out.Node) return } diff --git a/pkg/apis/contrail/v1alpha1/templates/BUILD.bazel b/pkg/configuration/BUILD.bazel similarity index 88% rename from pkg/apis/contrail/v1alpha1/templates/BUILD.bazel rename to pkg/configuration/BUILD.bazel index 99f654088..af7b16f85 100644 --- a/pkg/apis/contrail/v1alpha1/templates/BUILD.bazel +++ b/pkg/configuration/BUILD.bazel @@ -13,7 +13,7 @@ go_library( "webui_config.go", "zookeeper_config.go", ], - importpath = "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1/templates", + importpath = "github.com/Juniper/contrail-operator/pkg/configuration", visibility = ["//visibility:public"], deps = ["@io_k8s_api//core/v1:go_default_library"], ) diff --git a/pkg/apis/contrail/v1alpha1/templates/cassandra_config.go b/pkg/configuration/cassandra_config.go similarity index 99% rename from pkg/apis/contrail/v1alpha1/templates/cassandra_config.go rename to pkg/configuration/cassandra_config.go index 7bb462cca..ba296744d 100644 --- a/pkg/apis/contrail/v1alpha1/templates/cassandra_config.go +++ b/pkg/configuration/cassandra_config.go @@ -1,4 +1,4 @@ -package templates +package configuration import "text/template" diff --git a/pkg/apis/contrail/v1alpha1/templates/config_config.go b/pkg/configuration/config_config.go similarity index 99% rename from pkg/apis/contrail/v1alpha1/templates/config_config.go rename to pkg/configuration/config_config.go index a55acddec..c9028cac6 100644 --- a/pkg/apis/contrail/v1alpha1/templates/config_config.go +++ b/pkg/configuration/config_config.go @@ -1,4 +1,4 @@ -package templates +package configuration import "text/template" diff --git a/pkg/apis/contrail/v1alpha1/templates/control_config.go b/pkg/configuration/control_config.go similarity index 99% rename from pkg/apis/contrail/v1alpha1/templates/control_config.go rename to pkg/configuration/control_config.go index 63dbfdac8..a66ef592a 100644 --- a/pkg/apis/contrail/v1alpha1/templates/control_config.go +++ b/pkg/configuration/control_config.go @@ -1,4 +1,4 @@ -package templates +package configuration import "text/template" diff --git a/pkg/apis/contrail/v1alpha1/templates/kubemanager_config.go b/pkg/configuration/kubemanager_config.go similarity index 88% rename from pkg/apis/contrail/v1alpha1/templates/kubemanager_config.go rename to pkg/configuration/kubemanager_config.go index e05fcaff5..a720ecc31 100644 --- a/pkg/apis/contrail/v1alpha1/templates/kubemanager_config.go +++ b/pkg/configuration/kubemanager_config.go @@ -1,4 +1,4 @@ -package templates +package configuration import "text/template" @@ -61,11 +61,11 @@ use_ssl = True cafile = {{ .CAFilePath }} ; Authentication settings (optional) [auth] -AUTHN_TYPE = noauth -;AUTHN_TYPE = keystone -;AUTHN_PROTOCOL = http -;AUTHN_SERVER = 127.0.0.1 -;AUTHN_PORT = 35357 -;AUTHN_URL = /v2.0/tokens -;AUTHN_TOKEN_URL = http://127.0.0.1:35357/v2.0/tokens +AUTHN_TYPE = {{ .AuthMode }} +AUTHN_PROTOCOL = {{ .KeystoneAuthProtocol }} +AUTHN_SERVER = {{ .KeystoneAddress }} +AUTHN_PORT = {{ .KeystonePort }} +AUTHN_DOMAIN = {{ .KeystoneUserDomainName }} +cafile = {{ .CAFilePath }} +AUTHN_URL = /v3/auth/tokens `)) diff --git a/pkg/apis/contrail/v1alpha1/templates/rabbitmq_config.go b/pkg/configuration/rabbitmq_config.go similarity index 98% rename from pkg/apis/contrail/v1alpha1/templates/rabbitmq_config.go rename to pkg/configuration/rabbitmq_config.go index deab634d8..f73a64ef2 100644 --- a/pkg/apis/contrail/v1alpha1/templates/rabbitmq_config.go +++ b/pkg/configuration/rabbitmq_config.go @@ -1,4 +1,4 @@ -package templates +package configuration import "text/template" diff --git a/pkg/apis/contrail/v1alpha1/templates/templates_helper_functions.go b/pkg/configuration/templates_helper_functions.go similarity index 97% rename from pkg/apis/contrail/v1alpha1/templates/templates_helper_functions.go rename to pkg/configuration/templates_helper_functions.go index aba65f92b..c29c06b45 100644 --- a/pkg/apis/contrail/v1alpha1/templates/templates_helper_functions.go +++ b/pkg/configuration/templates_helper_functions.go @@ -1,4 +1,4 @@ -package templates +package configuration import ( "strconv" diff --git a/pkg/apis/contrail/v1alpha1/templates/templates_helper_functions_test.go b/pkg/configuration/templates_helper_functions_test.go similarity index 99% rename from pkg/apis/contrail/v1alpha1/templates/templates_helper_functions_test.go rename to pkg/configuration/templates_helper_functions_test.go index ddf263eb1..3ae0cc8c4 100644 --- a/pkg/apis/contrail/v1alpha1/templates/templates_helper_functions_test.go +++ b/pkg/configuration/templates_helper_functions_test.go @@ -1,4 +1,4 @@ -package templates +package configuration import ( "testing" diff --git a/pkg/apis/contrail/v1alpha1/tests/BUILD.bazel b/pkg/configuration/tests/BUILD.bazel similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/BUILD.bazel rename to pkg/configuration/tests/BUILD.bazel diff --git a/pkg/apis/contrail/v1alpha1/tests/config_cassandra_test.go b/pkg/configuration/tests/config_cassandra_test.go similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/config_cassandra_test.go rename to pkg/configuration/tests/config_cassandra_test.go diff --git a/pkg/apis/contrail/v1alpha1/tests/config_config_test.go b/pkg/configuration/tests/config_config_test.go similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/config_config_test.go rename to pkg/configuration/tests/config_config_test.go diff --git a/pkg/apis/contrail/v1alpha1/tests/config_control_test.go b/pkg/configuration/tests/config_control_test.go similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/config_control_test.go rename to pkg/configuration/tests/config_control_test.go diff --git a/pkg/apis/contrail/v1alpha1/tests/config_customized_test.go b/pkg/configuration/tests/config_customized_test.go similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/config_customized_test.go rename to pkg/configuration/tests/config_customized_test.go diff --git a/pkg/apis/contrail/v1alpha1/tests/config_kubemanager_test.go b/pkg/configuration/tests/config_kubemanager_test.go similarity index 72% rename from pkg/apis/contrail/v1alpha1/tests/config_kubemanager_test.go rename to pkg/configuration/tests/config_kubemanager_test.go index 2dc9b068c..347d77503 100644 --- a/pkg/apis/contrail/v1alpha1/tests/config_kubemanager_test.go +++ b/pkg/configuration/tests/config_kubemanager_test.go @@ -5,7 +5,9 @@ import ( "testing" "github.com/kylelemons/godebug/diff" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/ini.v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -45,6 +47,19 @@ func TestKubemanagerConfig(t *testing.T) { diff := diff.Diff(environment.kubemanagerConfigMap.Data["kubemanager.1.1.6.1"], kubemanagerConfig) t.Fatalf("get kubemanager config: \n%v\n", diff) } + t.Run("Assert correct Kubemanager Keystone config", func(t *testing.T) { + kubemanagerApiIni, err := ini.Load([]byte(environment.kubemanagerConfigMap.Data["vnc.1.1.6.1"])) + require.NoError(t, err, "Error while reading config") + assert.Equal(t, "1.1.6.1", kubemanagerApiIni.Section("global").Key("WEB_SERVER").String()) + assert.Equal(t, "8082", kubemanagerApiIni.Section("global").Key("WEB_PORT").String()) + assert.Equal(t, "/etc/ssl/certs/kubernetes/ca-bundle.crt", kubemanagerApiIni.Section("global").Key("cafile").String()) + assert.Equal(t, "keystone", kubemanagerApiIni.Section("auth").Key("AUTHN_TYPE").String()) + assert.Equal(t, "https", kubemanagerApiIni.Section("auth").Key("AUTHN_PROTOCOL").String()) + assert.Equal(t, "10.11.12.13", kubemanagerApiIni.Section("auth").Key("AUTHN_SERVER").String()) + assert.Equal(t, "5555", kubemanagerApiIni.Section("auth").Key("AUTHN_PORT").String()) + assert.Equal(t, "Default", kubemanagerApiIni.Section("auth").Key("AUTHN_DOMAIN").String()) + assert.Equal(t, "/etc/ssl/certs/kubernetes/ca-bundle.crt", kubemanagerApiIni.Section("auth").Key("cafile").String()) + }) } var kubemanagerConfig = `[DEFAULTS] diff --git a/pkg/apis/contrail/v1alpha1/tests/config_rabbitmq_test.go b/pkg/configuration/tests/config_rabbitmq_test.go similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/config_rabbitmq_test.go rename to pkg/configuration/tests/config_rabbitmq_test.go diff --git a/pkg/apis/contrail/v1alpha1/tests/config_test.go b/pkg/configuration/tests/config_test.go similarity index 98% rename from pkg/apis/contrail/v1alpha1/tests/config_test.go rename to pkg/configuration/tests/config_test.go index 760560f06..8eab61060 100644 --- a/pkg/apis/contrail/v1alpha1/tests/config_test.go +++ b/pkg/configuration/tests/config_test.go @@ -73,6 +73,9 @@ var kubemanager = &v1alpha1.Kubemanager{ }, Spec: v1alpha1.KubemanagerSpec{ ServiceConfiguration: v1alpha1.KubemanagerServiceConfiguration{ + KubemanagerConfiguration: v1alpha1.KubemanagerConfiguration{ + AuthMode: "keystone", + }, KubemanagerNodesConfiguration: v1alpha1.KubemanagerNodesConfiguration{ CassandraNodesConfiguration: &v1alpha1.CassandraClusterConfiguration{ Port: 9160, @@ -92,6 +95,11 @@ var kubemanager = &v1alpha1.Kubemanager{ CollectorPort: 8086, APIServerIPList: []string{"1.1.1.1", "1.1.1.2", "1.1.1.3"}, CollectorServerIPList: []string{"1.1.1.1", "1.1.1.2", "1.1.1.3"}, + AuthMode: v1alpha1.AuthenticationModeKeystone, + }, + KeystoneNodesConfiguration: &v1alpha1.KeystoneClusterConfiguration{ + Port: 5555, + Endpoint: "10.11.12.13", }, }, }, diff --git a/pkg/apis/contrail/v1alpha1/tests/config_vrouter_test.go b/pkg/configuration/tests/config_vrouter_test.go similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/config_vrouter_test.go rename to pkg/configuration/tests/config_vrouter_test.go diff --git a/pkg/apis/contrail/v1alpha1/tests/config_webui_test.go b/pkg/configuration/tests/config_webui_test.go similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/config_webui_test.go rename to pkg/configuration/tests/config_webui_test.go diff --git a/pkg/apis/contrail/v1alpha1/tests/config_zookeeper_test.go b/pkg/configuration/tests/config_zookeeper_test.go similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/config_zookeeper_test.go rename to pkg/configuration/tests/config_zookeeper_test.go diff --git a/pkg/apis/contrail/v1alpha1/tests/pod_cert_subj_test.go b/pkg/configuration/tests/pod_cert_subj_test.go similarity index 100% rename from pkg/apis/contrail/v1alpha1/tests/pod_cert_subj_test.go rename to pkg/configuration/tests/pod_cert_subj_test.go diff --git a/pkg/apis/contrail/v1alpha1/templates/vrouter_config.go b/pkg/configuration/vrouter_config.go similarity index 99% rename from pkg/apis/contrail/v1alpha1/templates/vrouter_config.go rename to pkg/configuration/vrouter_config.go index 8cdcf375d..c497c0d6f 100644 --- a/pkg/apis/contrail/v1alpha1/templates/vrouter_config.go +++ b/pkg/configuration/vrouter_config.go @@ -1,4 +1,4 @@ -package templates +package configuration import "text/template" diff --git a/pkg/apis/contrail/v1alpha1/templates/webui_config.go b/pkg/configuration/webui_config.go similarity index 99% rename from pkg/apis/contrail/v1alpha1/templates/webui_config.go rename to pkg/configuration/webui_config.go index 24a39738d..131289cc7 100644 --- a/pkg/apis/contrail/v1alpha1/templates/webui_config.go +++ b/pkg/configuration/webui_config.go @@ -1,4 +1,4 @@ -package templates +package configuration import "text/template" diff --git a/pkg/apis/contrail/v1alpha1/templates/zookeeper_config.go b/pkg/configuration/zookeeper_config.go similarity index 99% rename from pkg/apis/contrail/v1alpha1/templates/zookeeper_config.go rename to pkg/configuration/zookeeper_config.go index 6b4fb4513..f3ba1e3af 100644 --- a/pkg/apis/contrail/v1alpha1/templates/zookeeper_config.go +++ b/pkg/configuration/zookeeper_config.go @@ -1,4 +1,4 @@ -package templates +package configuration import ( "fmt" diff --git a/pkg/controller/BUILD.bazel b/pkg/controller/BUILD.bazel index 25f82dad9..cfad7db13 100644 --- a/pkg/controller/BUILD.bazel +++ b/pkg/controller/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "add_config.go", "add_contrailmonitor.go", "add_control.go", + "add_devicemanager.go", "add_fernetkeymanager.go", "add_keystone.go", "add_manager.go", @@ -31,6 +32,7 @@ go_library( "//pkg/controller/config:go_default_library", "//pkg/controller/contrailmonitor:go_default_library", "//pkg/controller/control:go_default_library", + "//pkg/controller/devicemanager:go_default_library", "//pkg/controller/fernetkeymanager:go_default_library", "//pkg/controller/keystone:go_default_library", "//pkg/controller/manager:go_default_library", diff --git a/pkg/controller/add_devicemanager.go b/pkg/controller/add_devicemanager.go new file mode 100644 index 000000000..6c7122e45 --- /dev/null +++ b/pkg/controller/add_devicemanager.go @@ -0,0 +1,10 @@ +package controller + +import ( + "github.com/Juniper/contrail-operator/pkg/controller/devicemanager" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, devicemanager.Add) +} diff --git a/pkg/controller/cassandra/sts.go b/pkg/controller/cassandra/sts.go index 080e5fe69..9ca01c282 100644 --- a/pkg/controller/cassandra/sts.go +++ b/pkg/controller/cassandra/sts.go @@ -26,7 +26,7 @@ spec: terminationGracePeriodSeconds: 1800 containers: - image: hub.juniper.net/contrail-nightly/contrail-external-cassandra:5.2.0-0.740 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent env: - name: POD_IP valueFrom: @@ -75,7 +75,7 @@ spec: fieldRef: fieldPath: status.podIP image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init resources: {} securityContext: @@ -96,7 +96,7 @@ spec: fieldRef: fieldPath: status.podIP image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init2 resources: {} securityContext: diff --git a/pkg/controller/command/command_config_bootstrap.go b/pkg/controller/command/command_config_bootstrap.go index ad7b2348e..48ac80069 100644 --- a/pkg/controller/command/command_config_bootstrap.go +++ b/pkg/controller/command/command_config_bootstrap.go @@ -42,6 +42,8 @@ func (c *commandBootstrapConf) FillConfigMap(cm *core.ConfigMap) { cm.Data["bootstrap.sh"] = c.executeTemplate(commandInitBootstrapScript) cm.Data["init_cluster.yml"] = c.executeTemplate(commandInitCluster) cm.Data["command-app-server.yml"] = c.executeTemplate(commandConfig) + cm.Data["migration.yml"] = c.executeTemplate(migrationConfig) + cm.Data["migration.sh"] = c.executeTemplate(migrationScriptConfig) } func (c *commandBootstrapConf) executeTemplate(t *template.Template) string { @@ -231,3 +233,48 @@ resources: kind: endpoint {{end -}} `)) + +var migrationConfig = template.Must(template.New("").Parse(` +database: + host: {{ .PostgresAddress }} + user: {{ .PostgresUser }} + password: {{ .PGPassword }} + name: {{ .PostgresDBName }}_migrated + max_open_conn: 100 + connection_retries: 10 + retry_period: 3s + replication_status_timeout: 10s + debug: false + +log_level: debug`)) + +var migrationScriptConfig = template.Must(template.New("").Parse(` +#!/usr/bin/env bash + +set -e +export PGPASSWORD={{ .PGPassword }} +export PGHOST={{ .PostgresAddress }} +export PGUSER={{ .PostgresUser }} + +# Try to drop old databases the may or may not exists. +dropdb -w --if-exists {{ .PostgresDBName }}_migrated +dropdb -w --if-exists {{ .PostgresDBName }}_backup + +# Migrate old database dump to new one. +commandutil migrate --in /backups/db.yml --out /backups/db_migrated.yml + +# Create a database for migrated data, initialize it with a new schema. +createdb -w {{ .PostgresDBName }}_migrated +psql -v ON_ERROR_STOP=ON -w -d {{ .PostgresDBName }}_migrated -f /usr/share/contrail/gen_init_psql.sql +psql -v ON_ERROR_STOP=ON -w -d {{ .PostgresDBName }}_migrated -f /usr/share/contrail/init_psql.sql + +# Upload migrated data to the new database. +commandutil convert --intype yaml --in /backups/db_migrated.yml --outtype rdbms -c /etc/contrail/migration.yml + +# Replace original database with the migrated one and store original one as backup. +psql -v ON_ERROR_STOP=ON -w -d postgres < /dev/null 2>&1; do sleep 1; done"}, VolumeMounts: []core.VolumeMount{{ @@ -1200,6 +1330,7 @@ func assertBootstrapConfigMap(t *testing.T, actual *core.ConfigMap) { assert.Equal(t, expectedBootstrapScript, actual.Data["bootstrap.sh"]) assert.Equal(t, expectedCommandInitCluster, actual.Data["init_cluster.yml"]) + assert.Equal(t, expectedMigrationScript, actual.Data["migration.sh"]) } func newAdminSecret() *core.Secret { @@ -1303,7 +1434,7 @@ func newMigrationJob(oldTag, newTag string) *batch.Job { InitContainers: []core.Container{ { Name: "db-dump", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "registry:5000/contrail-command" + oldTag, Command: []string{"bash", "-c", "commandutil convert --intype rdbms --outtype yaml --out /backups/db.yml -c /etc/contrail/command-app-server.yml"}, @@ -1311,7 +1442,7 @@ func newMigrationJob(oldTag, newTag string) *batch.Job { }, { Name: "migrate-db-dump", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "registry:5000/contrail-command" + newTag, Command: []string{"bash", "-c", "commandutil migrate --in /backups/db.yml --out /backups/db_migrated.yml"}, @@ -1321,7 +1452,7 @@ func newMigrationJob(oldTag, newTag string) *batch.Job { Containers: []core.Container{ { Name: "restore-migrated-db-dump", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "registry:5000/contrail-command" + newTag, Command: []string{"bash", "-c", "commandutil convert --intype yaml --in /backups/db_migrated.yml --outtype rdbms -c /etc/contrail/command-app-server.yml"}, @@ -1347,6 +1478,19 @@ func newMigrationJob(oldTag, newTag string) *batch.Job { } } +func newMigrationJobFailed(oldTag, newTag string) (job *batch.Job) { + job = newMigrationJob(oldTag, newTag) + job.Status = batch.JobStatus{ + Conditions: []batch.JobCondition{ + { + Type: batch.JobFailed, + Status: core.ConditionTrue, + }, + }, + } + return +} + func newBootstrapJob() *batch.Job { executableMode := int32(0744) trueVal := true @@ -1386,7 +1530,7 @@ func newBootstrapJob() *batch.Job { Containers: []core.Container{ { Name: "command-init", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "registry:5000/contrail-command", Command: []string{"bash", "-c", "/etc/contrail/bootstrap.sh"}, VolumeMounts: []core.VolumeMount{{ @@ -1722,3 +1866,33 @@ resources: public_url: https://1.1.1.1:7000 kind: endpoint ` +const expectedMigrationScript = ` +#!/usr/bin/env bash + +set -e +export PGPASSWORD=test123 +export PGHOST=10.219.10.10 +export PGUSER=root + +# Try to drop old databases the may or may not exists. +dropdb -w --if-exists contrail_test_migrated +dropdb -w --if-exists contrail_test_backup + +# Migrate old database dump to new one. +commandutil migrate --in /backups/db.yml --out /backups/db_migrated.yml + +# Create a database for migrated data, initialize it with a new schema. +createdb -w contrail_test_migrated +psql -v ON_ERROR_STOP=ON -w -d contrail_test_migrated -f /usr/share/contrail/gen_init_psql.sql +psql -v ON_ERROR_STOP=ON -w -d contrail_test_migrated -f /usr/share/contrail/init_psql.sql + +# Upload migrated data to the new database. +commandutil convert --intype yaml --in /backups/db_migrated.yml --outtype rdbms -c /etc/contrail/migration.yml + +# Replace original database with the migrated one and store original one as backup. +psql -v ON_ERROR_STOP=ON -w -d postgres < /dev/null 2>&1; do sleep 1; done - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /tmp/podinfo name: status @@ -39,7 +39,7 @@ spec: - sh - -c - until grep true /tmp/podinfo/peers_ready > /dev/null 2>&1; do sleep 1; done - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /tmp/podinfo name: status @@ -51,7 +51,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent readinessProbe: httpGet: scheme: HTTPS @@ -67,7 +67,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail/config-device-manager name: config-device-manager-logs @@ -82,7 +82,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs @@ -93,7 +93,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs @@ -104,7 +104,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs @@ -119,7 +119,7 @@ spec: value: "true" - name: ANALYTICS_ALARM_ENABLE value: "true" - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs @@ -130,7 +130,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs @@ -141,7 +141,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs @@ -152,7 +152,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs @@ -169,7 +169,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs @@ -188,7 +188,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs @@ -203,7 +203,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: config-logs diff --git a/pkg/controller/contrailcni/job.go b/pkg/controller/contrailcni/job.go index 482b7808b..560cac5a3 100644 --- a/pkg/controller/contrailcni/job.go +++ b/pkg/controller/contrailcni/job.go @@ -61,7 +61,7 @@ func GetJob(cniDir CniDirs, requestName, instanceType string, replicas *int32) * MountPath: "/var/run/multus", }, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", } if cniDir.DeploymentType == "openshift" { diff --git a/pkg/controller/contrailmonitor/contrailmonitor_controller.go b/pkg/controller/contrailmonitor/contrailmonitor_controller.go index 1292d4dec..e897793f8 100644 --- a/pkg/controller/contrailmonitor/contrailmonitor_controller.go +++ b/pkg/controller/contrailmonitor/contrailmonitor_controller.go @@ -2,6 +2,7 @@ package contrailmonitor import ( "context" + "os" "strings" contrailv1alpha1 "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1" @@ -54,17 +55,28 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return err } - err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Postgres{}}, &handler.EnqueueRequestForOwner{ - OwnerType: &contrailv1alpha1.Contrailmonitor{}, - }) - if err != nil { - return err - } - err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Memcached{}}, &handler.EnqueueRequestForOwner{ - OwnerType: &contrailv1alpha1.Contrailmonitor{}, - }) - if err != nil { - return err + _, ok := os.LookupEnv("CLUSTER_TYPE") + if !ok { + + err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Postgres{}}, &handler.EnqueueRequestForOwner{ + OwnerType: &contrailv1alpha1.Contrailmonitor{}, + }) + if err != nil { + return err + } + err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Memcached{}}, &handler.EnqueueRequestForOwner{ + OwnerType: &contrailv1alpha1.Contrailmonitor{}, + }) + if err != nil { + return err + } + + err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Keystone{}}, &handler.EnqueueRequestForOwner{ + OwnerType: &contrailv1alpha1.Contrailmonitor{}, + }) + if err != nil { + return err + } } err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Cassandra{}}, &handler.EnqueueRequestForOwner{ @@ -85,25 +97,26 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { if err != nil { return err } - err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Keystone{}}, &handler.EnqueueRequestForOwner{ + + err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Zookeeper{}}, &handler.EnqueueRequestForOwner{ OwnerType: &contrailv1alpha1.Contrailmonitor{}, }) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Zookeeper{}}, &handler.EnqueueRequestForOwner{ + err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Rabbitmq{}}, &handler.EnqueueRequestForOwner{ OwnerType: &contrailv1alpha1.Contrailmonitor{}, }) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Rabbitmq{}}, &handler.EnqueueRequestForOwner{ + err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Webui{}}, &handler.EnqueueRequestForOwner{ OwnerType: &contrailv1alpha1.Contrailmonitor{}, }) if err != nil { return err } - err = c.Watch(&source.Kind{Type: &contrailv1alpha1.Webui{}}, &handler.EnqueueRequestForOwner{ + err = c.Watch(&source.Kind{Type: &contrailv1alpha1.ProvisionManager{}}, &handler.EnqueueRequestForOwner{ OwnerType: &contrailv1alpha1.Contrailmonitor{}, }) if err != nil { @@ -153,71 +166,82 @@ func (r *ReconcileContrailmonitor) Reconcile(request reconcile.Request) (reconci return reconcile.Result{}, nil } - psql, err := r.getPostgres(instance) - if err != nil { - return reconcile.Result{}, err - } - if err = r.kubernetes.Owner(instance).EnsureOwns(psql); err != nil { - return reconcile.Result{}, err - } - if err != nil { - return reconcile.Result{}, err - } - serIns := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: psql.Name, Namespace: "contrail"}} - var datasql string - if psql.Status.Active { - datasql = "Active" - } else { - datasql = "NotActive" - } - _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, serIns, func() error { - serIns.Status = datasql - return controllerutil.SetControllerReference(instance, serIns, r.scheme) - }) - if err != nil { - return reconcile.Result{}, nil - } + _, ok := os.LookupEnv("CLUSTER_TYPE") - memcached, err := r.getMemcached(instance) - if err != nil { - return reconcile.Result{}, err - } + if !ok { - serInsmemcached := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: memcached.Name, Namespace: "contrail"}} + psql, err := r.getPostgres(instance) + if err != nil { + return reconcile.Result{}, err + } + if err = r.kubernetes.Owner(instance).EnsureOwns(psql); err != nil { + return reconcile.Result{}, err + } - var datamemcached string - if memcached.Status.Active { - datamemcached = "Active" - } else { - datamemcached = "NotActive" - } - _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, serInsmemcached, func() error { - serInsmemcached.Status = datamemcached - return controllerutil.SetControllerReference(instance, serInsmemcached, r.scheme) - }) - if err != nil { - return reconcile.Result{}, nil - } + psqllist, _ := r.getPsqllist() + serInslist := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: psqllist.Items[0].Name, Namespace: "contrail"}} + var sqlListStatus string + if psqllist.Items[0].Status.Active { + sqlListStatus = "Functional" + } else { + sqlListStatus = "Non-Functional" + } + _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, serInslist, func() error { + serInslist.Status = sqlListStatus + return controllerutil.SetControllerReference(instance, serInslist, r.scheme) + }) + if err != nil { + return reconcile.Result{}, nil + } - var dataValue string - keystone, err := r.getKeystone(instance) - if err != nil { - return reconcile.Result{}, err - } + memcached, err := r.getMemcached(instance) + if err != nil { + return reconcile.Result{}, err + } + if err = r.kubernetes.Owner(instance).EnsureOwns(memcached); err != nil { + return reconcile.Result{}, err + } - serInskey := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: keystone.Name, Namespace: "contrail"}} + memlist, _ := r.getMemlist() + sermemlist := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: memlist.Items[0].Name, Namespace: "contrail"}} + var memListStatus string + if memlist.Items[0].Status.Active { + memListStatus = "Functional" + } else { + memListStatus = "Non-Functional" + } + _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, sermemlist, func() error { + sermemlist.Status = memListStatus + return controllerutil.SetControllerReference(instance, sermemlist, r.scheme) + }) + if err != nil { + return reconcile.Result{}, nil + } + + keystone, err := r.getKeystone(instance) + if err != nil { + return reconcile.Result{}, err + } + if err = r.kubernetes.Owner(instance).EnsureOwns(keystone); err != nil { + return reconcile.Result{}, err + } + + keylist, _ := r.getKeylist() + serKeylist := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: keylist.Items[0].Name, Namespace: "contrail"}} + var keyListStatus string + if keylist.Items[0].Status.Active { + keyListStatus = "Functional" + } else { + keyListStatus = "Non-Functional" + } + _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, serKeylist, func() error { + serKeylist.Status = keyListStatus + return controllerutil.SetControllerReference(instance, serKeylist, r.scheme) + }) + if err != nil { + return reconcile.Result{}, nil + } - if keystone.Status.Active { - dataValue = "Active" - } else { - dataValue = "NotActive" - } - _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, serInskey, func() error { - serInskey.Status = dataValue - return controllerutil.SetControllerReference(instance, serInskey, r.scheme) - }) - if err != nil { - return reconcile.Result{}, nil } rabbitmq, err := r.getRabbitmq(instance) @@ -228,16 +252,14 @@ func (r *ReconcileContrailmonitor) Reconcile(request reconcile.Request) (reconci if err = r.kubernetes.Owner(instance).EnsureOwns(rabbitmq); err != nil { return reconcile.Result{}, err } - if err != nil { - return reconcile.Result{}, err - } - serInsrabbitmq := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: rabbitmq.Name, Namespace: "contrail"}} + rablist, _ := r.getRabbitlist() + serInsrabbitmq := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: rablist.Items[0].Name, Namespace: "contrail"}} var datarabbitmq string - if rabbitmq.Status.Active == nil { - datarabbitmq = "NotActive" + if rablist.Items[0].Status.Active == nil { + datarabbitmq = "Non-Functional" } else { - datarabbitmq = "Active" + datarabbitmq = "Functional" } _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, serInsrabbitmq, func() error { @@ -255,18 +277,16 @@ func (r *ReconcileContrailmonitor) Reconcile(request reconcile.Request) (reconci if err = r.kubernetes.Owner(instance).EnsureOwns(zookeeper); err != nil { return reconcile.Result{}, err } - if err != nil { - return reconcile.Result{}, err - } + zoolist, _ := r.getZoolist() zookeeperActive := zookeeper.IsActive(instance.Spec.ServiceConfiguration.ZookeeperInstance, request.Namespace, r.client) - serInszookeeper := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: zookeeper.Name, Namespace: "contrail"}} + serInszookeeper := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: zoolist.Items[0].Name, Namespace: "contrail"}} var datazookeeper string - if zookeeper.Status.Active == nil || !zookeeperActive { - datazookeeper = "NotActive" + if zoolist.Items[0].Status.Active == nil || !zookeeperActive { + datazookeeper = "Non-Functional" } else { - datazookeeper = "Active" + datazookeeper = "Functional" } _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, serInszookeeper, func() error { @@ -284,18 +304,16 @@ func (r *ReconcileContrailmonitor) Reconcile(request reconcile.Request) (reconci if err = r.kubernetes.Owner(instance).EnsureOwns(cassandra); err != nil { return reconcile.Result{}, err } - if err != nil { - return reconcile.Result{}, err - } + casslist, _ := r.getCasslist() cassandraActive := cassandra.IsActive(instance.Spec.ServiceConfiguration.CassandraInstance, request.Namespace, r.client) - serInscass := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: cassandra.Name, Namespace: "contrail"}} + serInscass := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: casslist.Items[0].Name, Namespace: "contrail"}} var datacassandra string - if cassandra.Status.Active == nil || !cassandraActive { - datacassandra = "NotActive" + if casslist.Items[0].Status.Active == nil || !cassandraActive { + datacassandra = "Non-Functional" } else { - datacassandra = "Active" + datacassandra = "Functional" } _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, serInscass, func() error { @@ -316,85 +334,129 @@ func (r *ReconcileContrailmonitor) Reconcile(request reconcile.Request) (reconci if err != nil { return reconcile.Result{}, err } + + wklist, _ := r.getWeblist() var dataWeb string - if webui.Status.ServiceStatus != nil { - for _, valueOne := range webui.Status.ServiceStatus { - for _, valuetwo := range valueOne { - serInsWeb := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: valuetwo.ModuleName, Namespace: "contrail"}} - dataWeb = valuetwo.ModuleState - _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInsWeb, func() error { - serInsWeb.Status = dataWeb - return controllerutil.SetControllerReference(instance, serInsWeb, r.scheme) - }) - if err != nil { - return reconcile.Result{}, err + var webName string + wcount := len(wklist.Items) + if wcount > 0 { + for i := 0; i < wcount; i++ { + if data, ok := wklist.Items[i].Status.ServiceStatus["kind-control-plane"]; ok { + for _, j := range data { + webName = j.ModuleName + "-" + "one" + serInsWeb := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: webName, Namespace: "contrail"}} + dataWeb = j.ModuleState + _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInsWeb, func() error { + serInsWeb.Status = dataWeb + return controllerutil.SetControllerReference(instance, serInsWeb, r.scheme) + }) + if err != nil { + return reconcile.Result{}, err + } + } + } + if data1, ok := wklist.Items[i].Status.ServiceStatus["kind-worker"]; ok { + for _, j := range data1 { + webName = j.ModuleName + "-" + "two" + serInsWeb := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: webName, Namespace: "contrail"}} + dataWeb = j.ModuleState + _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInsWeb, func() error { + serInsWeb.Status = dataWeb + return controllerutil.SetControllerReference(instance, serInsWeb, r.scheme) + }) + if err != nil { + return reconcile.Result{}, err + } + } + } + if data2, ok := wklist.Items[i].Status.ServiceStatus["kind-worker"]; ok { + for _, j := range data2 { + webName = j.ModuleName + "-" + "three" + serInsWeb := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: webName, Namespace: "contrail"}} + dataWeb = j.ModuleState + _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInsWeb, func() error { + serInsWeb.Status = dataWeb + return controllerutil.SetControllerReference(instance, serInsWeb, r.scheme) + }) + if err != nil { + return reconcile.Result{}, err + } } } } - serInsWeb := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: webui.Name, Namespace: "contrail"}} - dataWeb = "Functional" - _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInsWeb, func() error { - serInsWeb.Status = dataWeb - return controllerutil.SetControllerReference(instance, serInsWeb, r.scheme) - }) - if err != nil { - return reconcile.Result{}, err - } - } else { - serInsWeb := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: webui.Name, Namespace: "contrail"}} - dataWeb = "NotActive" - _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInsWeb, func() error { - serInsWeb.Status = dataWeb - return controllerutil.SetControllerReference(instance, serInsWeb, r.scheme) - }) - if err != nil { - return reconcile.Result{}, err - } } - config, _ := r.getConfig(instance) - var dataConfig string - if config.Status.ServiceStatus != nil { - for _, valueOne := range config.Status.ServiceStatus { - for _, valuetwo := range valueOne { - serInsConfig := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: valuetwo.ModuleName, Namespace: "contrail"}} - dataConfig = valuetwo.ModuleState - _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInsConfig, func() error { - serInsConfig.Status = dataConfig - return controllerutil.SetControllerReference(instance, serInsConfig, r.scheme) - }) - if err != nil { - return reconcile.Result{}, err + config, err := r.getConfig(instance) + if err != nil { + return reconcile.Result{}, err + } + if err = r.kubernetes.Owner(instance).EnsureOwns(config); err != nil { + return reconcile.Result{}, err + } + + clist, _ := r.getConfiglist() + var dataSeven string + var configName string + ccount := len(clist.Items) + if ccount > 0 { + for k := 0; k < ccount; k++ { + if clist.Items[k].Status.Active != nil { + mapvalues := clist.Items[k].Status.ServiceStatus + if data, ok := mapvalues["kind-control-plane"]; ok { + for _, j := range data { + configName = j.ModuleName + "-" + "one" + serIns7 := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: configName, Namespace: "contrail"}} + dataSeven = j.ModuleState + _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serIns7, func() error { + serIns7.Status = dataSeven + return controllerutil.SetControllerReference(instance, serIns7, r.scheme) + }) + if err != nil { + return reconcile.Result{}, err + } + + } + } + if data, ok := mapvalues["kind-worker"]; ok { + for _, j := range data { + configName = j.ModuleName + "-" + "two" + serIns7 := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: configName, Namespace: "contrail"}} + dataSeven = j.ModuleState + _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serIns7, func() error { + serIns7.Status = dataSeven + return controllerutil.SetControllerReference(instance, serIns7, r.scheme) + }) + if err != nil { + return reconcile.Result{}, err + } + + } + } + if data, ok := mapvalues["kind-worker2"]; ok { + for _, j := range data { + configName = j.ModuleName + "-" + "three" + serIns7 := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: configName, Namespace: "contrail"}} + dataSeven = j.ModuleState + _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serIns7, func() error { + serIns7.Status = dataSeven + return controllerutil.SetControllerReference(instance, serIns7, r.scheme) + }) + if err != nil { + return reconcile.Result{}, err + } + } } } } - serInsConfig := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: "config", Namespace: "contrail"}} - dataConfig = "Functional" - _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInsConfig, func() error { - serInsConfig.Status = dataConfig - return controllerutil.SetControllerReference(instance, serInsConfig, r.scheme) - }) - if err != nil { - return reconcile.Result{}, err - } - } else { - serInsConfig := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: "config", Namespace: "contrail"}} - dataConfig = "NotActive" - _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInsConfig, func() error { - serInsConfig.Status = dataConfig - return controllerutil.SetControllerReference(instance, serInsConfig, r.scheme) - }) - if err != nil { - return reconcile.Result{}, err - } } var dataProvision string provisionmanager, _ := r.getProvisionmanager(instance) + if provisionmanager.Status.Active != nil { - dataProvision = "Active" + dataProvision = "Functional" } else { - dataProvision = "NotActive" + dataProvision = "Non-Functional" } serInsprovision := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: provisionmanager.Name, Namespace: "contrail"}} _, err = controllerutil.CreateOrUpdate(context.Background(), r.client, serInsprovision, func() error { @@ -412,27 +474,62 @@ func (r *ReconcileContrailmonitor) Reconcile(request reconcile.Request) (reconci if err = r.kubernetes.Owner(instance).EnsureOwns(control); err != nil { return reconcile.Result{}, err } - if err != nil { - return reconcile.Result{}, err - } conlist, _ := r.getControllist() + var datacontrol string - var namevalue string + var tempNameVal string conCount := len(conlist.Items) + if conCount > 0 { for j := 0; j < conCount; j++ { if conlist.Items[j].Status.Active != nil { - for _, x := range conlist.Items[j].Status.ServiceStatus { - tempdata := x.Connections - for k := 0; k < len(tempdata); k++ { - if tempdata[k].Name == "" { - namevalue = strings.ToLower("control" + "-" + tempdata[k].Type) + kindvalues := conlist.Items[j].Status.ServiceStatus + if data, ok := kindvalues["kind-control-plane"]; ok { + for m := 0; m < len(data.Connections); m++ { + if data.Connections[m].Name == "" { + tempNameVal = strings.ToLower("control" + "-" + data.Connections[m].Type + "-" + "one") + } else { + tempNameVal = strings.ToLower("control" + "-" + data.Connections[m].Type + "-" + data.Connections[m].Name + "-" + "one") + } + serInscontrol := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: tempNameVal, Namespace: "contrail"}} + datacontrol = data.Connections[m].Status + _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInscontrol, func() error { + serInscontrol.Status = datacontrol + return controllerutil.SetControllerReference(instance, serInscontrol, r.scheme) + }) + if err != nil { + return reconcile.Result{}, err + } + } + } + if data, ok := kindvalues["kind-worker"]; ok { + for m := 0; m < len(data.Connections); m++ { + if data.Connections[m].Name == "" { + tempNameVal = strings.ToLower("control" + "-" + data.Connections[m].Type + "-" + "one") } else { - namevalue = strings.ToLower("control" + "-" + tempdata[k].Type + "-" + tempdata[k].Name) + tempNameVal = strings.ToLower("control" + "-" + data.Connections[m].Type + "-" + data.Connections[m].Name + "-" + "one") } - serInscontrol := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: namevalue, Namespace: "contrail"}} - datacontrol = tempdata[k].Status + serInscontrol := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: tempNameVal, Namespace: "contrail"}} + datacontrol = data.Connections[m].Status + _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInscontrol, func() error { + serInscontrol.Status = datacontrol + return controllerutil.SetControllerReference(instance, serInscontrol, r.scheme) + }) + if err != nil { + return reconcile.Result{}, err + } + } + } + if data, ok := kindvalues["kind-worker2"]; ok { + for m := 0; m < len(data.Connections); m++ { + if data.Connections[m].Name == "" { + tempNameVal = strings.ToLower("control" + "-" + data.Connections[m].Type + "-" + "one") + } else { + tempNameVal = strings.ToLower("control" + "-" + data.Connections[m].Type + "-" + data.Connections[m].Name + "-" + "one") + } + serInscontrol := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: tempNameVal, Namespace: "contrail"}} + datacontrol = data.Connections[m].Status _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInscontrol, func() error { serInscontrol.Status = datacontrol return controllerutil.SetControllerReference(instance, serInscontrol, r.scheme) @@ -444,27 +541,6 @@ func (r *ReconcileContrailmonitor) Reconcile(request reconcile.Request) (reconci } } } - serInscontrol := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: "control", Namespace: "contrail"}} - datacontrol = "Functional" - _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInscontrol, func() error { - serInscontrol.Status = datacontrol - return controllerutil.SetControllerReference(instance, serInscontrol, r.scheme) - }) - if err != nil { - return reconcile.Result{}, err - } - } else { - serInscontrol := &contrailv1alpha1.Contrailstatusmonitor{ObjectMeta: metav1.ObjectMeta{Name: "control", Namespace: "contrail"}} - - datacontrol = "NotActive" - _, err := controllerutil.CreateOrUpdate(context.Background(), r.client, serInscontrol, func() error { - serInscontrol.Status = datacontrol - return controllerutil.SetControllerReference(instance, serInscontrol, r.scheme) - }) - if err != nil { - return reconcile.Result{}, err - } - } instance.Status.Name = "contrailmonitor" @@ -557,3 +633,66 @@ func (r *ReconcileContrailmonitor) getControllist() (*contrailv1alpha1.ControlLi err := r.client.List(context.TODO(), conlist, listOps) return conlist, err } + +func (r *ReconcileContrailmonitor) getPsqllist() (*contrailv1alpha1.PostgresList, error) { + listOps := &client.ListOptions{Namespace: "contrail"} + psqllist := &contrailv1alpha1.PostgresList{} + err := r.client.List(context.TODO(), psqllist, listOps) + return psqllist, err +} + +func (r *ReconcileContrailmonitor) getMemlist() (*contrailv1alpha1.MemcachedList, error) { + listOps := &client.ListOptions{Namespace: "contrail"} + memlist := &contrailv1alpha1.MemcachedList{} + err := r.client.List(context.TODO(), memlist, listOps) + return memlist, err +} + +func (r *ReconcileContrailmonitor) getRabbitlist() (*contrailv1alpha1.RabbitmqList, error) { + listOps := &client.ListOptions{Namespace: "contrail"} + rablist := &contrailv1alpha1.RabbitmqList{} + err := r.client.List(context.TODO(), rablist, listOps) + return rablist, err +} + +func (r *ReconcileContrailmonitor) getZoolist() (*contrailv1alpha1.ZookeeperList, error) { + listOps := &client.ListOptions{Namespace: "contrail"} + zoolist := &contrailv1alpha1.ZookeeperList{} + err := r.client.List(context.TODO(), zoolist, listOps) + return zoolist, err +} + +func (r *ReconcileContrailmonitor) getKeylist() (*contrailv1alpha1.KeystoneList, error) { + listOps := &client.ListOptions{Namespace: "contrail"} + keylist := &contrailv1alpha1.KeystoneList{} + err := r.client.List(context.TODO(), keylist, listOps) + return keylist, err +} + +func (r *ReconcileContrailmonitor) getConfiglist() (*contrailv1alpha1.ConfigList, error) { + listOps := &client.ListOptions{Namespace: "contrail"} + configlist := &contrailv1alpha1.ConfigList{} + err := r.client.List(context.TODO(), configlist, listOps) + return configlist, err +} + +func (r *ReconcileContrailmonitor) getCasslist() (*contrailv1alpha1.CassandraList, error) { + listOps := &client.ListOptions{Namespace: "contrail"} + casslist := &contrailv1alpha1.CassandraList{} + err := r.client.List(context.TODO(), casslist, listOps) + return casslist, err +} + +func (r *ReconcileContrailmonitor) getWeblist() (*contrailv1alpha1.WebuiList, error) { + listOps := &client.ListOptions{Namespace: "contrail"} + weblist := &contrailv1alpha1.WebuiList{} + err := r.client.List(context.TODO(), weblist, listOps) + return weblist, err +} + +func (r *ReconcileContrailmonitor) getProlist() (*contrailv1alpha1.ProvisionManagerList, error) { + listOps := &client.ListOptions{Namespace: "contrail"} + prolist := &contrailv1alpha1.ProvisionManagerList{} + err := r.client.List(context.TODO(), prolist, listOps) + return prolist, err +} diff --git a/pkg/controller/control/sts.go b/pkg/controller/control/sts.go index 2ca0ba863..9e375d458 100644 --- a/pkg/controller/control/sts.go +++ b/pkg/controller/control/sts.go @@ -36,7 +36,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /tmp/podinfo name: status @@ -48,7 +48,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: control-logs @@ -59,7 +59,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: control-logs @@ -70,7 +70,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: control-logs @@ -85,7 +85,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent securityContext: privileged: true runAsGroup: 1999 @@ -107,7 +107,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent lifecycle: preStop: exec: diff --git a/pkg/controller/devicemanager/BUILD.bazel b/pkg/controller/devicemanager/BUILD.bazel new file mode 100644 index 000000000..7eadef076 --- /dev/null +++ b/pkg/controller/devicemanager/BUILD.bazel @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "devicemanager_controller.go", + "sts.go", + ], + importpath = "github.com/Juniper/contrail-operator/pkg/controller/devicemanager", + visibility = ["//visibility:public"], + deps = [ + "//pkg/apis/contrail/v1alpha1:go_default_library", + "//pkg/certificates:go_default_library", + "//pkg/controller/utils:go_default_library", + "//pkg/k8s:go_default_library", + "@com_github_ghodss//:go_default_library", + "@io_k8s_api//apps/v1:go_default_library", + "@io_k8s_api//core/v1:go_default_library", + "@io_k8s_apimachinery//pkg/api/errors:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library", + "@io_k8s_apimachinery//pkg/runtime:go_default_library", + "@io_k8s_apimachinery//pkg/types:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/controller:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/handler:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/log:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/manager:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/reconcile:go_default_library", + "@io_k8s_sigs_controller_runtime//pkg/source:go_default_library", + ], +) diff --git a/pkg/controller/devicemanager/devicemanager_controller.go b/pkg/controller/devicemanager/devicemanager_controller.go new file mode 100644 index 000000000..7bd2ce1a0 --- /dev/null +++ b/pkg/controller/devicemanager/devicemanager_controller.go @@ -0,0 +1,456 @@ +package devicemanager + +import ( + "context" + "reflect" + + "github.com/Juniper/contrail-operator/pkg/certificates" + + appsv1 "k8s.io/api/apps/v1" + + "github.com/Juniper/contrail-operator/pkg/controller/utils" + + "github.com/Juniper/contrail-operator/pkg/k8s" + + "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1" + contrailv1alpha1 "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var log = logf.Log.WithName("controller_devicemanager") + +// Add creates a new Devicemanager Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileDevicemanager{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Manager: mgr, + Kubernetes: k8s.New(mgr.GetClient(), mgr.GetScheme()), + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("devicemanager-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to primary resource Devicemanager. + if err = c.Watch(&source.Kind{Type: &v1alpha1.Devicemanager{}}, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + + // Watch for changes to secondary resource Pods and requeue the owner Devicemanager + if err = c.Watch(&source.Kind{Type: &corev1.Service{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &v1alpha1.Devicemanager{}, + }); err != nil { + return err + } + + srcSTS := &source.Kind{Type: &appsv1.StatefulSet{}} + stsHandler := &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &v1alpha1.Devicemanager{}, + } + if err = c.Watch(srcSTS, stsHandler); err != nil { + return err + } + + return nil +} + +// blank assignment to verify that ReconcileDevicemanager implements reconcile.Reconciler +var _ reconcile.Reconciler = &ReconcileDevicemanager{} + +// ReconcileDevicemanager reconciles a Devicemanager object +type ReconcileDevicemanager struct { + // This client, initialized using mgr.Client() above, is a split client + // that reads objects from the cache and writes to the apiserver + Client client.Client + Scheme *runtime.Scheme + Manager manager.Manager + Kubernetes *k8s.Kubernetes +} + +// Reconcile reads that state of the cluster for a Devicemanager object and makes changes based on the state read +// and what is in the Devicemanager.Spec +// TODO(user): Modify this Reconcile function to implement your Controller logic. This example creates +// a Pod as an example +// Note: +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (r *ReconcileDevicemanager) Reconcile(request reconcile.Request) (reconcile.Result, error) { + reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + reqLogger.Info("Reconciling Devicemanager") + instanceType := "devicemanager" + + // Fetch the Devicemanager instance + devicemanager := &contrailv1alpha1.Devicemanager{} + err := r.Client.Get(context.TODO(), request.NamespacedName, devicemanager) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + if !devicemanager.GetDeletionTimestamp().IsZero() { + return reconcile.Result{}, nil + } + + reqLogger.Info("Checking dependencies are up") + + cassandraInstance := &v1alpha1.Cassandra{} + zookeeperInstance := &v1alpha1.Zookeeper{} + rabbitmqInstance := &v1alpha1.Rabbitmq{} + configInstance := &v1alpha1.Config{} + + cassandraActive := cassandraInstance.IsActive(devicemanager.Spec.ServiceConfiguration.CassandraInstance, + request.Namespace, r.Client) + zookeeperActive := zookeeperInstance.IsActive(devicemanager.Spec.ServiceConfiguration.ZookeeperInstance, + request.Namespace, r.Client) + configActive := configInstance.IsActive(devicemanager.Labels["contrail_cluster"], + request.Namespace, r.Client) + rabbitmqActive := rabbitmqInstance.IsActive(devicemanager.Labels["contrail_cluster"], + request.Namespace, r.Client) + + if !cassandraActive || !rabbitmqActive || !zookeeperActive || !configActive { + reqLogger.Info("Skipping: Waiting for dependencies") + return reconcile.Result{}, nil + } + // Define a new Pod object + //pod := newPodForCR(instance) + + // TODO figure out if we need it. We don't use SetControllerReference in config + // Set Devicemanager instance as the owner and controller + //if err := controllerutil.SetControllerReference(devicemanager, pod, r.Scheme); err != nil { + // return reconcile.Result{}, err + //} + + // TODO check if need this + // Check if this Pod already exists + //found := &corev1.Pod{} + //err = r.Client.Get(context.TODO(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, found) + //if err != nil && errors.IsNotFound(err) { + // reqLogger.Info("Creating a new Pod", "Pod.Namespace", pod.Namespace, "Pod.Name", pod.Name) + // err = r.Client.Create(context.TODO(), pod) + // if err != nil { + // return reconcile.Result{}, err + // } + // + // // Pod created successfully - don't requeue + // return reconcile.Result{}, nil + //} else if err != nil { + // return reconcile.Result{}, err + //} + // + //// Pod already exists - don't requeue + //reqLogger.Info("Skip reconcile: Pod already exists", "Pod.Namespace", found.Namespace, "Pod.Name", found.Name) + //return reconcile.Result{}, nil + + servicePortsMap := map[int32]string{ + int32(v1alpha1.ConfigDeviceManagerIntrospectPort): "introspect", + } + devicemanagerService := r.Kubernetes.Service(request.Name+"-"+instanceType, corev1.ServiceTypeClusterIP, + servicePortsMap, instanceType, devicemanager) + + if err := devicemanagerService.EnsureExists(); err != nil { + return reconcile.Result{}, err + } + + currentConfigMap, currentConfigExists := devicemanager.CurrentConfigMapExists( + request.Name+"-"+instanceType+"-configmap", r.Client, r.Scheme, request) + + reqLogger.Info("Creating config map") + + configMap, err := devicemanager.CreateConfigMap(request.Name+"-"+instanceType+"-configmap", + r.Client, r.Scheme, request) + if err != nil { + return reconcile.Result{}, err + } + + reqLogger.Info("Creating secrets") + + secretCertificates, err := devicemanager.CreateSecret(request.Name+"-secret-certificates", r.Client, r.Scheme, request) + if err != nil { + return reconcile.Result{}, err + } + + statefulSet := GetSTS() + trueVal := true + statefulSet.Spec.Template.Spec.ShareProcessNamespace = &trueVal + if err = devicemanager.PrepareSTS(statefulSet, &devicemanager.Spec.CommonConfiguration, request, r.Scheme, + r.Client); err != nil { + return reconcile.Result{}, err + } + + csrSignerCaVolumeName := request.Name + "-csr-signer-ca" + devicemanager.AddVolumesToIntendedSTS(statefulSet, map[string]string{ + configMap.Name: request.Name + "-" + instanceType + "-volume", + certificates.SignerCAConfigMapName: csrSignerCaVolumeName, + }) + devicemanager.AddSecretVolumesToIntendedSTS(statefulSet, map[string]string{secretCertificates.Name: request.Name + "-secret-certificates"}) + + reqLogger.Info("Creating account statusmonitor-devicemanager") + if err = v1alpha1.CreateAccount("statusmonitor-devicemanager", request.Namespace, r.Client, + r.Scheme, devicemanager); err != nil { + return reconcile.Result{}, err + } + + statefulSet.Spec.Template.Spec.ServiceAccountName = "serviceaccount-statusmonitor-devicemanager" + statefulSet.Spec.Template.Spec.Affinity = &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: instanceType, + Operator: "In", + Values: []string{request.Name}, + }}, + }, + TopologyKey: "kubernetes.io/hostname", + }}, + }, + } + + reqLogger.Info("Adding containers") + for idx, container := range statefulSet.Spec.Template.Spec.Containers { + reqLogger.Info("Adding container", "name", container.Name) + switch container.Name { + case "devicemanager": + deviceManagerCommand := `/usr/bin/rm -f /etc/contrail/vnc_api_lib.ini; ln -s /etc/contrailconfigmaps/vnc.${POD_IP} /etc/contrail/vnc_api_lib.ini; +/usr/bin/rm -f /etc/contrail/contrail-keystone-auth.conf; ln -s /etc/contrailconfigmaps/contrail-keystone-auth.conf /etc/contrail/contrail-keystone-auth.conf; +/usr/bin/rm -f /etc/contrail/contrail-fabric-ansible.conf; ln -s /etc/contrailconfigmaps/contrail-fabric-ansible.conf.${POD_IP} /etc/contrail/contrail-fabric-ansible.conf; +/usr/bin/python /usr/bin/contrail-device-manager --conf_file /etc/contrailconfigmaps/devicemanager.${POD_IP} --conf_file /etc/contrail/contrail-keystone-auth.conf +` + command := []string{"bash", "-c", deviceManagerCommand} + instanceContainer := utils.GetContainerFromList(container.Name, devicemanager.Spec.ServiceConfiguration.Containers) + if instanceContainer.Command == nil { + (&statefulSet.Spec.Template.Spec.Containers[idx]).Command = command + } else { + (&statefulSet.Spec.Template.Spec.Containers[idx]).Command = instanceContainer.Command + } + (&statefulSet.Spec.Template.Spec.Containers[idx]).SecurityContext = &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_PTRACE"}, + }, + } + volumeMountList := statefulSet.Spec.Template.Spec.Containers[idx].VolumeMounts + volumeMountList = append(volumeMountList, + corev1.VolumeMount{ + Name: request.Name + "-" + instanceType + "-volume", + MountPath: "/etc/contrailconfigmaps", + }, + corev1.VolumeMount{ + Name: request.Name + "-secret-certificates", + MountPath: "/etc/certificates", + }, + corev1.VolumeMount{ + Name: csrSignerCaVolumeName, + MountPath: certificates.SignerCAMountPath, + }, + corev1.VolumeMount{ + Name: "tftp", + MountPath: "/var/lib/tftp", + }, + corev1.VolumeMount{ + Name: "dnsmasq", + MountPath: "/var/lib/dnsmasq", + }, + ) + (&statefulSet.Spec.Template.Spec.Containers[idx]).VolumeMounts = volumeMountList + (&statefulSet.Spec.Template.Spec.Containers[idx]).Image = instanceContainer.Image + + case "dnsmasq": + container := &statefulSet.Spec.Template.Spec.Containers[idx] + container.Command = []string{"bash", "-c", + "/usr/bin/rm -f /etc/contrail/vnc_api_lib.ini;ln -s /etc/contrailconfigmaps/vnc.${POD_IP} /etc/contrail/vnc_api_lib.ini;" + + "dnsmasq -k -p0 --conf-file=/etc/contrailconfigmaps/dnsmasq.${POD_IP}"} + instanceContainer := utils.GetContainerFromList(container.Name, devicemanager.Spec.ServiceConfiguration.Containers) + if instanceContainer.Command != nil { + container.Command = instanceContainer.Command + } + container.SecurityContext = &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_RAW"}, + }, + } + volumeMountList := statefulSet.Spec.Template.Spec.Containers[idx].VolumeMounts + volumeMountList = append(volumeMountList, + corev1.VolumeMount{ + Name: request.Name + "-" + instanceType + "-volume", + MountPath: "/etc/contrailconfigmaps", + }, + corev1.VolumeMount{ + Name: request.Name + "-secret-certificates", + MountPath: "/etc/certificates", + }, + corev1.VolumeMount{ + Name: csrSignerCaVolumeName, + MountPath: certificates.SignerCAMountPath, + }, + corev1.VolumeMount{ + Name: "tftp", + MountPath: "/etc/tftp", + }, + corev1.VolumeMount{ + Name: "dnsmasq", + MountPath: "/var/lib/dnsmasq", + }, + ) + // DNSMasq container requires those variables to be set + // TODO: Pass keystone credentials + container.Env = append(container.Env, []corev1.EnvVar{ + {Name: "KEYSTONE_AUTH_ADMIN_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: devicemanager.Spec.ServiceConfiguration.KeystoneSecretName, + }, + Key: "password", + }, + }, + }, + {Name: "KEYSTONE_AUTH_ADMIN_USER", Value: "admin"}, + {Name: "KEYSTONE_AUTH_ADMIN_TENANT", Value: "admin"}, + }...) + container.VolumeMounts = volumeMountList + container.Image = instanceContainer.Image + + case "statusmonitor": + instanceContainer := utils.GetContainerFromList(container.Name, devicemanager.Spec.ServiceConfiguration.Containers) + command := []string{"sh", "-c", + "/app/statusmonitor/contrail-statusmonitor-image.binary -config /etc/contrailconfigmaps/monitorconfig.${POD_IP}.yaml"} + if instanceContainer.Command == nil { + (&statefulSet.Spec.Template.Spec.Containers[idx]).Command = command + } else { + (&statefulSet.Spec.Template.Spec.Containers[idx]).Command = instanceContainer.Command + } + + volumeMountList := []corev1.VolumeMount{} + if len((&statefulSet.Spec.Template.Spec.Containers[idx]).VolumeMounts) > 0 { + volumeMountList = (&statefulSet.Spec.Template.Spec.Containers[idx]).VolumeMounts + } + volumeMount := corev1.VolumeMount{ + Name: request.Name + "-" + instanceType + "-volume", + MountPath: "/etc/contrailconfigmaps", + } + volumeMountList = append(volumeMountList, volumeMount) + volumeMount = corev1.VolumeMount{ + Name: request.Name + "-secret-certificates", + MountPath: "/etc/certificates", + } + volumeMountList = append(volumeMountList, volumeMount) + volumeMount = corev1.VolumeMount{ + Name: csrSignerCaVolumeName, + MountPath: certificates.SignerCAMountPath, + } + volumeMountList = append(volumeMountList, volumeMount) + (&statefulSet.Spec.Template.Spec.Containers[idx]).VolumeMounts = volumeMountList + (&statefulSet.Spec.Template.Spec.Containers[idx]).Image = instanceContainer.Image + } + } + + reqLogger.Info("Adding init containers") + // Configure InitContainers + for idx, container := range statefulSet.Spec.Template.Spec.InitContainers { + instanceContainer := utils.GetContainerFromList(container.Name, devicemanager.Spec.ServiceConfiguration.Containers) + (&statefulSet.Spec.Template.Spec.InitContainers[idx]).Image = instanceContainer.Image + if instanceContainer.Command != nil { + (&statefulSet.Spec.Template.Spec.InitContainers[idx]).Command = instanceContainer.Command + } + } + + configChanged := false + if devicemanager.Status.ConfigChanged != nil { + configChanged = *devicemanager.Status.ConfigChanged + } + + if err = devicemanager.CreateSTS(statefulSet, instanceType, request, r.Client); err != nil { + return reconcile.Result{}, err + } + podIPList, podIPMap, err := devicemanager.PodIPListAndIPMapFromInstance(request, r.Client) + if err != nil { + return reconcile.Result{}, err + } + if len(podIPMap) > 0 { + if err = devicemanager.InstanceConfiguration(request, podIPList, r.Client); err != nil { + return reconcile.Result{}, err + } + + if err := r.ensureCertificatesExist(devicemanager, podIPList, instanceType); err != nil { + return reconcile.Result{}, err + } + + if err = devicemanager.SetPodsToReady(podIPList, r.Client); err != nil { + return reconcile.Result{}, err + } + + if err = devicemanager.WaitForPeerPods(request, r.Client); err != nil { + return reconcile.Result{}, err + } + + if err = devicemanager.ManageNodeStatus(podIPMap, r.Client); err != nil { + return reconcile.Result{}, err + } + } + + if err = devicemanager.SetEndpointInStatus(r.Client, devicemanagerService.ClusterIP()); err != nil { + return reconcile.Result{}, err + } + if currentConfigExists { + newConfigMap := &corev1.ConfigMap{} + _ = r.Client.Get(context.TODO(), types.NamespacedName{Name: request.Name + "-" + instanceType + "-configmap", Namespace: request.Namespace}, newConfigMap) + if !reflect.DeepEqual(currentConfigMap.Data, newConfigMap.Data) { + configChanged = true + } else { + configChanged = false + } + devicemanager.Status.ConfigChanged = &configChanged + } + + if devicemanager.Status.Active == nil { + active := false + devicemanager.Status.Active = &active + } + if err = devicemanager.SetInstanceActive(r.Client, devicemanager.Status.Active, statefulSet, request); err != nil { + return reconcile.Result{}, err + } + if devicemanager.Status.ConfigChanged != nil { + if *devicemanager.Status.ConfigChanged { + return reconcile.Result{Requeue: true}, nil + } + } + return reconcile.Result{}, nil + +} + +func (r *ReconcileDevicemanager) ensureCertificatesExist(devicemaanger *v1alpha1.Devicemanager, pods *corev1.PodList, instanceType string) error { + subjects := devicemaanger.PodsCertSubjects(pods) + crt := certificates.NewCertificate(r.Client, r.Scheme, devicemaanger, subjects, instanceType) + return crt.EnsureExistsAndIsSigned() +} diff --git a/pkg/controller/devicemanager/sts.go b/pkg/controller/devicemanager/sts.go new file mode 100644 index 000000000..6e21a3984 --- /dev/null +++ b/pkg/controller/devicemanager/sts.go @@ -0,0 +1,150 @@ +package devicemanager + +import ( + "github.com/ghodss/yaml" + appsv1 "k8s.io/api/apps/v1" +) + +var yamlDataconfig_sts = ` +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: devicemanager +spec: + selector: + matchLabels: + app: config + serviceName: "devicemanager" + replicas: 1 + template: + metadata: + labels: + app: devicemanager + contrail_manager: devicemanager + spec: + initContainers: + - name: init + image: busybox + command: + - sh + - -c + - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done + imagePullPolicy: Always + volumeMounts: + - mountPath: /tmp/podinfo + name: status + - name: init2 + image: busybox + command: + - sh + - -c + - until grep true /tmp/podinfo/peers_ready > /dev/null 2>&1; do sleep 1; done + imagePullPolicy: Always + volumeMounts: + - mountPath: /tmp/podinfo + name: status + containers: + - name: devicemanager + image: docker.io/michaelhenkel/contrail-controller-config-devicemgr:5.2.0-dev1 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: Always + volumeMounts: + - mountPath: /var/log/contrail/device-manager + name: config-device-manager-logs + - name: dnsmasq + image: docker.io/michaelhenkel/contrail-external-dnsmasq:5.2.0-dev1 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: CONTROLLER_NODES + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: Always + volumeMounts: + - mountPath: /var/log/contrail/device-manager + name: config-device-manager-logs + - name: statusmonitor + image: docker.io/kaweue/contrail-statusmonitor:debug + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + imagePullPolicy: Always + volumeMounts: + - mountPath: /var/log/contrail/device-manager + name: config-device-manager-logs + dnsPolicy: ClusterFirst + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - emptyDir: {} + name: tftp + - emptyDir: {} + name: dnsmasq + - hostPath: + path: /var/log/contrail/config-device-manager + type: "" + name: config-device-manager-logs + - hostPath: + path: /var/contrail/crashes + type: "" + name: crashes + - hostPath: + path: /var/lib/contrail/config + type: "" + name: config-data + - hostPath: + path: /var/run + type: "" + name: docker-unix-socket + - hostPath: + path: /usr/local/bin + type: "" + name: host-usr-local-bin + - downwardAPI: + defaultMode: 420 + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.labels + path: pod_labels + - fieldRef: + apiVersion: v1 + fieldPath: metadata.labels + path: peers_ready + - fieldRef: + apiVersion: v1 + fieldPath: metadata.labels + path: pod_labelsx + name: status` + +func GetSTS() *appsv1.StatefulSet { + sts := appsv1.StatefulSet{} + err := yaml.Unmarshal([]byte(yamlDataconfig_sts), &sts) + if err != nil { + panic(err) + } + jsonData, err := yaml.YAMLToJSON([]byte(yamlDataconfig_sts)) + if err != nil { + panic(err) + } + err = yaml.Unmarshal([]byte(jsonData), &sts) + if err != nil { + panic(err) + } + return &sts +} diff --git a/pkg/controller/keystone/keystone_controller.go b/pkg/controller/keystone/keystone_controller.go index ee85d70e7..29c46fd45 100644 --- a/pkg/controller/keystone/keystone_controller.go +++ b/pkg/controller/keystone/keystone_controller.go @@ -380,7 +380,7 @@ func newKeystoneSTS(cr *contrail.Keystone) *apps.StatefulSet { InitContainers: []core.Container{ { Name: "wait-for-ready-conf", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: getImage(cr, "wait-for-ready-conf"), Command: getCommand(cr, "wait-for-ready-conf"), VolumeMounts: []core.VolumeMount{{ @@ -393,7 +393,7 @@ func newKeystoneSTS(cr *contrail.Keystone) *apps.StatefulSet { { Name: "keystone", Image: getImage(cr, "keystone"), - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Env: newKollaEnvs("keystone"), Command: getCommand(cr, "keystone"), VolumeMounts: []core.VolumeMount{ @@ -570,7 +570,7 @@ func newBootStrapJob(cr *contrail.Keystone, name types.NamespacedName, kcbName, { Name: "keystone-db-init", Image: getImage(cr, "keystoneDbInit"), - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Command: getCommand(cr, "keystoneDbInit"), Args: []string{"-c", initDBScript}, Env: []core.EnvVar{ @@ -596,7 +596,7 @@ func newBootStrapJob(cr *contrail.Keystone, name types.NamespacedName, kcbName, { Name: "keystone-bootstrap", Image: getImage(cr, "keystoneInit"), - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Env: []core.EnvVar{{ Name: "KOLLA_SERVICE_NAME", Value: "keystone", diff --git a/pkg/controller/keystone/keystone_controller_test.go b/pkg/controller/keystone/keystone_controller_test.go index 19c1970b6..1b251be1d 100644 --- a/pkg/controller/keystone/keystone_controller_test.go +++ b/pkg/controller/keystone/keystone_controller_test.go @@ -578,7 +578,7 @@ func newExpectedSTS() *apps.StatefulSet { InitContainers: []core.Container{ { Name: "wait-for-ready-conf", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "localhost:5000/busybox", Command: []string{"sh", "-c", expectedCommandWaitForReadyContainer}, VolumeMounts: []core.VolumeMount{{ @@ -591,7 +591,7 @@ func newExpectedSTS() *apps.StatefulSet { { Image: "localhost:5000/centos-binary-keystone:train", Name: "keystone", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Env: []core.EnvVar{ { Name: "KOLLA_SERVICE_NAME", @@ -856,7 +856,7 @@ func newExpectedSTSWithCustomImages() *apps.StatefulSet { sts.Spec.Template.Spec.InitContainers = []core.Container{ { Name: "wait-for-ready-conf", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "localhost:5000/busybox", Command: []string{"sh", "-c", expectedCommandWaitForReadyContainer}, VolumeMounts: []core.VolumeMount{{ @@ -870,7 +870,7 @@ func newExpectedSTSWithCustomImages() *apps.StatefulSet { { Image: "image3", Name: "keystone", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Env: []core.EnvVar{ { Name: "KOLLA_SERVICE_NAME", @@ -976,7 +976,7 @@ func newExpectedBootstrapJob() *batch.Job { { Name: "keystone-db-init", Image: "localhost:5000/postgresql-client", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Command: []string{"/bin/sh"}, Args: []string{"-c", expectedCommandImage}, Env: []core.EnvVar{ @@ -1002,7 +1002,7 @@ func newExpectedBootstrapJob() *batch.Job { { Name: "keystone-bootstrap", Image: "localhost:5000/centos-binary-keystone:train", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Env: []core.EnvVar{{ Name: "KOLLA_SERVICE_NAME", Value: "keystone", diff --git a/pkg/controller/kubemanager/sts.go b/pkg/controller/kubemanager/sts.go index 1990ae0ff..c3566eae4 100644 --- a/pkg/controller/kubemanager/sts.go +++ b/pkg/controller/kubemanager/sts.go @@ -43,7 +43,7 @@ func GetSTS() *apps.StatefulSet { MountPath: "/tmp/podinfo", }, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", }, } @@ -57,7 +57,7 @@ func GetSTS() *apps.StatefulSet { Env: []core.EnvVar{ podIPEnv, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", }, { Name: "statusmonitor", @@ -68,7 +68,7 @@ func GetSTS() *apps.StatefulSet { Env: []core.EnvVar{ podIPEnv, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", }, } diff --git a/pkg/controller/manager/manager_controller.go b/pkg/controller/manager/manager_controller.go index 2f0afe464..affd50e37 100644 --- a/pkg/controller/manager/manager_controller.go +++ b/pkg/controller/manager/manager_controller.go @@ -469,23 +469,34 @@ func (r *ReconcileManager) processProvisionManager(manager *v1alpha1.Manager, re return nil } - pm := &v1alpha1.ProvisionManager{} - pm.ObjectMeta = manager.Spec.Services.ProvisionManager.ObjectMeta - pm.ObjectMeta.Namespace = manager.Namespace - manager.Spec.Services.ProvisionManager.Spec.ServiceConfiguration.KeystoneSecretName = manager.Spec.KeystoneSecretName - _, err := controllerutil.CreateOrUpdate(context.TODO(), r.client, pm, func() error { - pm.Spec = manager.Spec.Services.ProvisionManager.Spec - pm.Spec.CommonConfiguration = utils.MergeCommonConfiguration(manager.Spec.CommonConfiguration, pm.Spec.CommonConfiguration) - if pm.Spec.CommonConfiguration.Replicas == nil { - pm.Spec.CommonConfiguration.Replicas = &replicas + var pmServiceStatus *v1alpha1.ServiceStatus + if provisionManagerDependenciesReady(manager.ObjectMeta, r.client) { + pm := &v1alpha1.ProvisionManager{} + pm.ObjectMeta = manager.Spec.Services.ProvisionManager.ObjectMeta + pm.ObjectMeta.Namespace = manager.Namespace + manager.Spec.Services.ProvisionManager.Spec.ServiceConfiguration.KeystoneSecretName = manager.Spec.KeystoneSecretName + _, err := controllerutil.CreateOrUpdate(context.TODO(), r.client, pm, func() error { + pm.Spec.ServiceConfiguration.ProvisionManagerConfiguration = manager.Spec.Services.ProvisionManager.Spec.ServiceConfiguration + pm.Spec.CommonConfiguration = utils.MergeCommonConfiguration(manager.Spec.CommonConfiguration, + manager.Spec.Services.ProvisionManager.Spec.CommonConfiguration) + if err := fillProvisionManagerConfiguration(pm, manager.ObjectMeta, r.client); err != nil { + return err + } + if pm.Spec.CommonConfiguration.Replicas == nil { + pm.Spec.CommonConfiguration.Replicas = &replicas + } + return controllerutil.SetControllerReference(manager, pm, r.scheme) + }) + if err != nil { + return err } - return controllerutil.SetControllerReference(manager, pm, r.scheme) - }) - status := &v1alpha1.ServiceStatus{} - status.Name = &pm.Name - status.Active = pm.Status.Active - manager.Status.ProvisionManager = status - return err + status := &v1alpha1.ServiceStatus{} + status.Name = &pm.Name + status.Active = pm.Status.Active + pmServiceStatus = status + } + manager.Status.ProvisionManager = pmServiceStatus + return nil } func (r *ReconcileManager) processConfig(manager *v1alpha1.Manager, replicas int32, hostAliases []corev1.HostAlias) error { @@ -557,7 +568,7 @@ func (r *ReconcileManager) processKubemanagers(manager *v1alpha1.Manager, replic var kubemanagerServiceStatus []*v1alpha1.ServiceStatus for _, kubemanagerService := range manager.Spec.Services.Kubemanagers { - if !kubemanagerDependenciesReady(kubemanagerService.Spec.ServiceConfiguration.CassandraInstance, kubemanagerService.Spec.ServiceConfiguration.ZookeeperInstance, manager.ObjectMeta, r.client) { + if !kubemanagerDependenciesReady(kubemanagerService.Spec.ServiceConfiguration.CassandraInstance, kubemanagerService.Spec.ServiceConfiguration.ZookeeperInstance, kubemanagerService.Spec.ServiceConfiguration.KeystoneInstance, manager.ObjectMeta, r.client) { continue } kubemanager := &v1alpha1.Kubemanager{} @@ -565,7 +576,12 @@ func (r *ReconcileManager) processKubemanagers(manager *v1alpha1.Manager, replic kubemanager.ObjectMeta.Namespace = manager.Namespace _, err := controllerutil.CreateOrUpdate(context.TODO(), r.client, kubemanager, func() error { kubemanager.Spec.ServiceConfiguration.KubemanagerConfiguration = kubemanagerService.Spec.ServiceConfiguration.KubemanagerConfiguration - if err := fillKubemanagerConfiguration(kubemanager, kubemanagerService.Spec.ServiceConfiguration.CassandraInstance, kubemanagerService.Spec.ServiceConfiguration.ZookeeperInstance, manager.ObjectMeta, r.client); err != nil { + if kubemanagerService.Spec.ServiceConfiguration.KeystoneInstance != "" { + kubemanager.Spec.ServiceConfiguration.AuthMode = v1alpha1.AuthenticationModeKeystone + } else { + kubemanager.Spec.ServiceConfiguration.AuthMode = v1alpha1.AuthenticationModeNoAuth + } + if err := fillKubemanagerConfiguration(kubemanager, kubemanagerService.Spec.ServiceConfiguration.CassandraInstance, kubemanagerService.Spec.ServiceConfiguration.ZookeeperInstance, kubemanagerService.Spec.ServiceConfiguration.KeystoneInstance, manager.ObjectMeta, r.client); err != nil { return err } kubemanager.Spec.CommonConfiguration = utils.MergeCommonConfiguration(manager.Spec.CommonConfiguration, kubemanagerService.Spec.CommonConfiguration) @@ -1036,11 +1052,12 @@ func modifyContrailCNI(ContrailCNI, ContrailCNIService *v1alpha1.ContrailCNI, ma } } -func kubemanagerDependenciesReady(cassandraName, zookeeperName string, managerMeta v1.ObjectMeta, client client.Client) bool { +func kubemanagerDependenciesReady(cassandraName, zookeeperName, keystoneName string, managerMeta v1.ObjectMeta, client client.Client) bool { cassandraInstance := v1alpha1.Cassandra{} zookeeperInstance := v1alpha1.Zookeeper{} rabbitmqInstance := v1alpha1.Rabbitmq{} configInstance := v1alpha1.Config{} + keystoneInstance := v1alpha1.Keystone{} cassandraActive := cassandraInstance.IsActive(cassandraName, managerMeta.Namespace, client) zookeeperActive := zookeeperInstance.IsActive(zookeeperName, managerMeta.Namespace, client) @@ -1049,10 +1066,12 @@ func kubemanagerDependenciesReady(cassandraName, zookeeperName string, managerMe configActive := configInstance.IsActive(managerMeta.Name, managerMeta.Namespace, client) - return cassandraActive && zookeeperActive && rabbitmqActive && configActive + keystoneActive := keystoneName == "" || keystoneInstance.IsActive(keystoneName, managerMeta.Namespace, client) + + return cassandraActive && zookeeperActive && rabbitmqActive && configActive && keystoneActive } -func fillKubemanagerConfiguration(kubemanager *v1alpha1.Kubemanager, cassandraName, zookeeperName string, managerMeta v1.ObjectMeta, client client.Client) error { +func fillKubemanagerConfiguration(kubemanager *v1alpha1.Kubemanager, cassandraName, zookeeperName, keystoneName string, managerMeta v1.ObjectMeta, client client.Client) error { cassandraConfig, err := v1alpha1.NewCassandraClusterConfiguration(cassandraName, managerMeta.Namespace, client) if err != nil { return err @@ -1073,6 +1092,11 @@ func fillKubemanagerConfiguration(kubemanager *v1alpha1.Kubemanager, cassandraNa return err } (&kubemanager.Spec.ServiceConfiguration).ConfigNodesConfiguration = &configConfig + keystoneConfig, err := v1alpha1.NewKeystoneClusterConfiguration(keystoneName, managerMeta.Namespace, client) + if err != nil { + return err + } + (&kubemanager.Spec.ServiceConfiguration).KeystoneNodesConfiguration = &keystoneConfig return nil } @@ -1098,3 +1122,19 @@ func fillVrouterConfiguration(vrouter *v1alpha1.Vrouter, controlName string, man (&vrouter.Spec.ServiceConfiguration).ConfigNodesConfiguration = &configConfig return nil } + +func provisionManagerDependenciesReady(managerMeta v1.ObjectMeta, client client.Client) bool { + configInstance := v1alpha1.Config{} + configActive := configInstance.IsActive(managerMeta.Name, managerMeta.Namespace, client) + + return configActive +} + +func fillProvisionManagerConfiguration(pm *v1alpha1.ProvisionManager, managerMeta v1.ObjectMeta, client client.Client) error { + configConfig, err := v1alpha1.NewConfigClusterConfiguration(managerMeta.Name, managerMeta.Namespace, client) + if err != nil { + return err + } + (&pm.Spec.ServiceConfiguration).ConfigNodesConfiguration = &configConfig + return nil +} diff --git a/pkg/controller/manager/manager_controller_test.go b/pkg/controller/manager/manager_controller_test.go index 5e2943788..b406eabbf 100644 --- a/pkg/controller/manager/manager_controller_test.go +++ b/pkg/controller/manager/manager_controller_test.go @@ -82,6 +82,14 @@ func TestManagerController(t *testing.T) { }, Spec: contrail.ProvisionManagerSpec{}, } + provisionmanagerService := &contrail.ProvisionManagerService{ + ObjectMeta: meta.ObjectMeta{ + Name: "provisionmanager", + Namespace: "default", + Labels: map[string]string{"contrail_cluster": "cluster1"}, + }, + Spec: contrail.ProvisionManagerServiceSpec{}, + } kubemanager := &contrail.Kubemanager{ ObjectMeta: meta.ObjectMeta{ Name: "kubemanager", @@ -262,7 +270,7 @@ func TestManagerController(t *testing.T) { Zookeepers: []*contrail.Zookeeper{zookeeper}, Kubemanagers: []*contrail.KubemanagerService{kubemanagerService}, Rabbitmq: rabbitmq, - ProvisionManager: provisionmanager, + ProvisionManager: provisionmanagerService, Webui: webui, Contrailmonitor: contrailmonitorCR, Controls: []*contrail.Control{control}, @@ -389,6 +397,24 @@ func TestManagerController(t *testing.T) { Labels: map[string]string{"contrail_cluster": "cluster1"}, }, Spec: contrail.ProvisionManagerSpec{ + ServiceConfiguration: contrail.ProvisionManagerServiceConfiguration{ + ProvisionManagerConfiguration: contrail.ProvisionManagerConfiguration{ + Containers: []*contrail.Container{ + {Name: "provisionmanager", Image: "provisionmanager:3.5"}, + {Name: "init", Image: "busybox"}, + {Name: "init2", Image: "provisionmanager:3.5"}, + }, + }, + }, + }, + } + provisionmanagerService := &contrail.ProvisionManagerService{ + ObjectMeta: meta.ObjectMeta{ + Name: "provisionmanager", + Namespace: "default", + Labels: map[string]string{"contrail_cluster": "cluster1"}, + }, + Spec: contrail.ProvisionManagerServiceSpec{ ServiceConfiguration: contrail.ProvisionManagerConfiguration{ Containers: []*contrail.Container{ {Name: "provisionmanager", Image: "provisionmanager:3.5"}, @@ -583,7 +609,7 @@ func TestManagerController(t *testing.T) { Zookeepers: []*contrail.Zookeeper{zookeeper}, Kubemanagers: []*contrail.KubemanagerService{kubemanagerService}, Rabbitmq: rabbitmq, - ProvisionManager: provisionmanager, + ProvisionManager: provisionmanagerService, Webui: webui, Controls: []*contrail.Control{control}, Vrouters: []*contrail.VrouterService{vrouterService}, @@ -719,7 +745,33 @@ func TestManagerController(t *testing.T) { Namespace: "default", Labels: map[string]string{"contrail_cluster": "cluster1"}, }, - Spec: contrail.ProvisionManagerSpec{}, + Spec: contrail.ProvisionManagerSpec{ + ServiceConfiguration: contrail.ProvisionManagerServiceConfiguration{ + ProvisionManagerConfiguration: contrail.ProvisionManagerConfiguration{ + Containers: []*contrail.Container{ + {Name: "provisionmanager", Image: "provisionmanager:3.5"}, + {Name: "init", Image: "busybox"}, + {Name: "init2", Image: "provisionmanager:3.5"}, + }, + }, + }, + }, + } + provisionmanagerService := &contrail.ProvisionManagerService{ + ObjectMeta: meta.ObjectMeta{ + Name: "provisionmanager", + Namespace: "default", + Labels: map[string]string{"contrail_cluster": "cluster1"}, + }, + Spec: contrail.ProvisionManagerServiceSpec{ + ServiceConfiguration: contrail.ProvisionManagerConfiguration{ + Containers: []*contrail.Container{ + {Name: "provisionmanager", Image: "provisionmanager:3.5"}, + {Name: "init", Image: "busybox"}, + {Name: "init2", Image: "provisionmanager:3.5"}, + }, + }, + }, } kubemanager := &contrail.Kubemanager{ ObjectMeta: meta.ObjectMeta{ @@ -901,7 +953,7 @@ func TestManagerController(t *testing.T) { Zookeepers: []*contrail.Zookeeper{zookeeper}, Kubemanagers: []*contrail.KubemanagerService{kubemanagerService}, Rabbitmq: rabbitmq, - ProvisionManager: provisionmanager, + ProvisionManager: provisionmanagerService, Webui: webui, Controls: []*contrail.Control{control}, Vrouters: []*contrail.VrouterService{vrouterService}, @@ -2553,6 +2605,18 @@ func cassandraWithActiveState(state bool) *contrail.Cassandra { } } +func keystoneWithActiveState(state bool) *contrail.Keystone { + return &contrail.Keystone{ + ObjectMeta: meta.ObjectMeta{ + Name: "keystone1", + Namespace: "test-ns", + }, + Status: contrail.KeystoneStatus{ + Active: state, + }, + } +} + func zookeeperWithActiveState(state bool) *contrail.Zookeeper { return &contrail.Zookeeper{ ObjectMeta: meta.ObjectMeta{ @@ -2635,7 +2699,45 @@ func TestKubemanagerDependenciesReady(t *testing.T) { configWithActiveState(tc.configActive), rabbitmqWithActiveState(tc.rabbitmqActive), zookeeperWithActiveState(tc.zookeeperActive)) - got := kubemanagerDependenciesReady("cassandra1", "zookeeper1", meta.ObjectMeta{Name: "cluster1", Namespace: "test-ns"}, cl) + got := kubemanagerDependenciesReady("cassandra1", "zookeeper1", "", meta.ObjectMeta{Name: "cluster1", Namespace: "test-ns"}, cl) + assert.Equal(t, tc.expected, got) + } +} + +func TestKubemanagerDependenciesReadyWithKeystone(t *testing.T) { + + tests := []struct { + cassandraActive bool + zookeeperActive bool + rabbitmqActive bool + configActive bool + keystoneActive bool + expected bool + }{ + {cassandraActive: true, zookeeperActive: true, rabbitmqActive: true, configActive: true, keystoneActive: true, expected: true}, + {cassandraActive: false, zookeeperActive: false, rabbitmqActive: false, configActive: false, keystoneActive: false, expected: false}, + {cassandraActive: true, zookeeperActive: true, rabbitmqActive: false, configActive: false, keystoneActive: false, expected: false}, + {cassandraActive: false, zookeeperActive: false, rabbitmqActive: true, configActive: true, keystoneActive: false, expected: false}, + {cassandraActive: true, zookeeperActive: false, rabbitmqActive: true, configActive: false, keystoneActive: false, expected: false}, + {cassandraActive: false, zookeeperActive: true, rabbitmqActive: false, configActive: true, keystoneActive: false, expected: false}, + {cassandraActive: false, zookeeperActive: true, rabbitmqActive: true, configActive: true, keystoneActive: false, expected: false}, + {cassandraActive: true, zookeeperActive: false, rabbitmqActive: true, configActive: true, keystoneActive: false, expected: false}, + {cassandraActive: true, zookeeperActive: true, rabbitmqActive: false, configActive: true, keystoneActive: false, expected: false}, + {cassandraActive: true, zookeeperActive: true, rabbitmqActive: true, configActive: false, keystoneActive: false, expected: false}, + {cassandraActive: true, zookeeperActive: true, rabbitmqActive: true, configActive: true, keystoneActive: false, expected: false}, + } + + scheme, err := contrail.SchemeBuilder.Build() + require.NoError(t, err, "Failed to build scheme") + + for _, tc := range tests { + cl := fake.NewFakeClientWithScheme(scheme, + cassandraWithActiveState(tc.cassandraActive), + configWithActiveState(tc.configActive), + rabbitmqWithActiveState(tc.rabbitmqActive), + zookeeperWithActiveState(tc.zookeeperActive), + keystoneWithActiveState(tc.keystoneActive)) + got := kubemanagerDependenciesReady("cassandra1", "zookeeper1", "keystone1", meta.ObjectMeta{Name: "cluster1", Namespace: "test-ns"}, cl) assert.Equal(t, tc.expected, got) } } @@ -2665,6 +2767,26 @@ func TestVrouterDependenciesReady(t *testing.T) { } } +func TestProvisionManagerDependenciesReady(t *testing.T) { + tests := []struct { + configActive bool + expected bool + }{ + {configActive: true, expected: true}, + {configActive: false, expected: false}, + } + + scheme, err := contrail.SchemeBuilder.Build() + require.NoError(t, err, "Failed to build scheme") + + for _, tc := range tests { + cl := fake.NewFakeClientWithScheme(scheme, + configWithActiveState(tc.configActive)) + got := provisionManagerDependenciesReady(meta.ObjectMeta{Name: "cluster1", Namespace: "test-ns"}, cl) + assert.Equal(t, tc.expected, got) + } +} + func TestFillKubemanagerConfiguration(t *testing.T) { scheme, err := contrail.SchemeBuilder.Build() require.NoError(t, err, "Failed to build scheme") @@ -2673,14 +2795,16 @@ func TestFillKubemanagerConfiguration(t *testing.T) { cassandraWithActiveState(true), configWithActiveState(true), rabbitmqWithActiveState(true), - zookeeperWithActiveState(true)) + zookeeperWithActiveState(true), + keystoneWithActiveState(true)) newKubemanager := &contrail.Kubemanager{} - require.NoError(t, fillKubemanagerConfiguration(newKubemanager, "cassandra1", "zookeeper1", meta.ObjectMeta{Name: "cluster1", Namespace: "test-ns"}, cl)) + require.NoError(t, fillKubemanagerConfiguration(newKubemanager, "cassandra1", "zookeeper1", "keystone1", meta.ObjectMeta{Name: "cluster1", Namespace: "test-ns"}, cl)) assert.NotNil(t, newKubemanager.Spec.ServiceConfiguration.CassandraNodesConfiguration) assert.NotNil(t, newKubemanager.Spec.ServiceConfiguration.ConfigNodesConfiguration) assert.NotNil(t, newKubemanager.Spec.ServiceConfiguration.ZookeeperNodesConfiguration) assert.NotNil(t, newKubemanager.Spec.ServiceConfiguration.RabbbitmqNodesConfiguration) + assert.NotNil(t, newKubemanager.Spec.ServiceConfiguration.KeystoneNodesConfiguration) } func TestFillVrouterConfiguration(t *testing.T) { @@ -2697,6 +2821,18 @@ func TestFillVrouterConfiguration(t *testing.T) { assert.NotNil(t, newVrouter.Spec.ServiceConfiguration.ControlNodesConfiguration) } +func TestFillProvisionManagerConfiguration(t *testing.T) { + scheme, err := contrail.SchemeBuilder.Build() + require.NoError(t, err, "Failed to build scheme") + + cl := fake.NewFakeClientWithScheme(scheme, + configWithActiveState(true)) + + newProvisionManager := &contrail.ProvisionManager{} + require.NoError(t, fillProvisionManagerConfiguration(newProvisionManager, meta.ObjectMeta{Name: "cluster1", Namespace: "test-ns"}, cl)) + assert.NotNil(t, newProvisionManager.Spec.ServiceConfiguration.ConfigNodesConfiguration) +} + func TestProcessVrouters(t *testing.T) { scheme, err := contrail.SchemeBuilder.Build() require.NoError(t, err) @@ -2750,7 +2886,8 @@ func TestProcessKubemanagers(t *testing.T) { cassandraWithActiveState(true), configWithActiveState(true), rabbitmqWithActiveState(true), - zookeeperWithActiveState(true)) + zookeeperWithActiveState(true), + keystoneWithActiveState(true)) reconciler := ReconcileManager{ client: cl, scheme: scheme, @@ -2772,11 +2909,13 @@ func TestProcessKubemanagers(t *testing.T) { ServiceConfiguration: contrail.KubemanagerManagerServiceConfiguration{ CassandraInstance: "cassandra1", ZookeeperInstance: "zookeeper1", + KeystoneInstance: "keystone1", }, }, }, }, }, + KeystoneSecretName: "SecretName", }, } @@ -2790,4 +2929,251 @@ func TestProcessKubemanagers(t *testing.T) { assert.NotNil(t, createdKubemanager.Spec.ServiceConfiguration.ZookeeperNodesConfiguration) assert.NotNil(t, createdKubemanager.Spec.ServiceConfiguration.CassandraNodesConfiguration) assert.NotNil(t, createdKubemanager.Spec.ServiceConfiguration.RabbbitmqNodesConfiguration) + assert.NotNil(t, createdKubemanager.Spec.ServiceConfiguration.KeystoneNodesConfiguration) +} + +func TestProcessProvisionManager(t *testing.T) { + scheme, err := contrail.SchemeBuilder.Build() + require.NoError(t, err) + + cl := fake.NewFakeClientWithScheme(scheme, + configWithActiveState(true)) + reconciler := ReconcileManager{ + client: cl, + scheme: scheme, + kubernetes: k8s.New(cl, scheme), + } + var replicas int32 + replicas = 1 + managerCR := &contrail.Manager{ + ObjectMeta: meta.ObjectMeta{ + Name: "cluster1", + Namespace: "test-ns", + }, + Spec: contrail.ManagerSpec{ + Services: contrail.Services{ + ProvisionManager: &contrail.ProvisionManagerService{ + ObjectMeta: meta.ObjectMeta{ + Name: "test-provisionmanager", + Namespace: "default", + }, + Spec: contrail.ProvisionManagerServiceSpec{ + CommonConfiguration: contrail.PodConfiguration{ + Replicas: &replicas, + }, + ServiceConfiguration: contrail.ProvisionManagerConfiguration{}, + }, + }, + }, + }, + } + require.NoError(t, reconciler.processProvisionManager(managerCR, 3)) + createdProvisionManager := &contrail.ProvisionManager{} + require.NoError(t, cl.Get(context.TODO(), types.NamespacedName{ + Name: "test-provisionmanager", + Namespace: "test-ns", + }, createdProvisionManager)) + assert.NotNil(t, createdProvisionManager.Spec.ServiceConfiguration.ConfigNodesConfiguration) + + t.Run("Check if number of replicas is equal to declared in manager", func(t *testing.T) { + createdProvisionManager.SetResourceVersion("") + assert.Equal(t, &replicas, createdProvisionManager.Spec.CommonConfiguration.Replicas) + }) +} + +func TestKubemanagerWithAuth(t *testing.T) { + scheme, err := contrail.SchemeBuilder.Build() + require.NoError(t, err) + require.NoError(t, apps.SchemeBuilder.AddToScheme(scheme)) + require.NoError(t, core.SchemeBuilder.AddToScheme(scheme)) + trueVal := true + config := &contrail.Config{ + ObjectMeta: meta.ObjectMeta{ + Name: "config", + Namespace: "test-ns", + Labels: map[string]string{"contrail_cluster": "cluster1"}, + }, + Spec: contrail.ConfigSpec{ + ServiceConfiguration: contrail.ConfigConfiguration{ + KeystoneSecretName: "keystone-adminpass-secret", + AuthMode: contrail.AuthenticationModeKeystone, + }, + }, + Status: contrail.ConfigStatus{Active: &trueVal}, + } + keystone := &contrail.Keystone{ + ObjectMeta: meta.ObjectMeta{ + Name: "keystone", + Namespace: "test-ns", + }, + Spec: contrail.KeystoneSpec{ + ServiceConfiguration: contrail.KeystoneConfiguration{ + PostgresInstance: "psql", + ListenPort: 5555, + KeystoneSecretName: "keystone-adminpass-secre", + AuthProtocol: "https", + }, + }, + Status: contrail.KeystoneStatus{ + Active: trueVal, + Endpoint: "10.11.12.13", + }, + } + kubemanager := &contrail.Kubemanager{ + ObjectMeta: meta.ObjectMeta{ + Name: "kubemanager", + Namespace: "test-ns", + Labels: map[string]string{"contrail_cluster": "cluster1"}, + }, + Spec: contrail.KubemanagerSpec{ + ServiceConfiguration: contrail.KubemanagerServiceConfiguration{ + KubemanagerConfiguration: contrail.KubemanagerConfiguration{ + Containers: []*contrail.Container{ + {Name: "kubemanager", Image: "kubemanager"}, + {Name: "init", Image: "busybox"}, + {Name: "init2", Image: "kubemanager"}, + }, + }, + }, + }, + } + kubemanagerService := &contrail.KubemanagerService{ + ObjectMeta: meta.ObjectMeta{ + Name: "kubemanager", + Namespace: "test-ns", + Labels: map[string]string{"contrail_cluster": "cluster1"}, + }, + Spec: contrail.KubemanagerServiceSpec{ + ServiceConfiguration: contrail.KubemanagerManagerServiceConfiguration{ + CassandraInstance: "cassandra1", + ZookeeperInstance: "zookeeper1", + KeystoneInstance: "keystone", + KubemanagerConfiguration: contrail.KubemanagerConfiguration{ + Containers: []*contrail.Container{ + {Name: "kubemanager", Image: "kubemanager"}, + {Name: "init", Image: "busybox"}, + {Name: "init2", Image: "kubemanager"}, + }, + }, + }, + }, + } + managerCR := &contrail.Manager{ + ObjectMeta: meta.ObjectMeta{ + Name: "cluster1", + Namespace: "test-ns", + UID: "manager-uid-1", + }, + Spec: contrail.ManagerSpec{ + Services: contrail.Services{ + Kubemanagers: []*contrail.KubemanagerService{kubemanagerService}, + Keystone: keystone, + Config: config, + }, + KeystoneSecretName: "keystone-adminpass-secret", + }, + Status: contrail.ManagerStatus{}, + } + initObjs := []runtime.Object{ + managerCR, + newAdminSecret(), + cassandraWithActiveState(true), + rabbitmqWithActiveState(true), + zookeeperWithActiveState(true), + kubemanager, + config, + keystone, + newNode(1), + } + fakeClient := fake.NewFakeClientWithScheme(scheme, initObjs...) + reconciler := ReconcileManager{ + client: fakeClient, + scheme: scheme, + kubernetes: k8s.New(fakeClient, scheme), + } + // when + result, err := reconciler.Reconcile(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "cluster1", + Namespace: "test-ns", + }, + }) + assert.NoError(t, err) + assert.False(t, result.Requeue) + var replicas int32 + replicas = 1 + expectedKube := contrail.Kubemanager{ + ObjectMeta: meta.ObjectMeta{ + Name: "kubemanager", + Namespace: "test-ns", + Labels: map[string]string{"contrail_cluster": "cluster1"}, + OwnerReferences: []meta.OwnerReference{ + { + APIVersion: "contrail.juniper.net/v1alpha1", + Kind: "Manager", + Name: "cluster1", + UID: "manager-uid-1", + Controller: &trueVal, + BlockOwnerDeletion: &trueVal, + }, + }, + }, + TypeMeta: meta.TypeMeta{Kind: "Kubemanager", APIVersion: "contrail.juniper.net/v1alpha1"}, + + Spec: contrail.KubemanagerSpec{ + CommonConfiguration: contrail.PodConfiguration{ + Replicas: &replicas, + }, + ServiceConfiguration: contrail.KubemanagerServiceConfiguration{ + KubemanagerConfiguration: contrail.KubemanagerConfiguration{ + Containers: []*contrail.Container{ + {Name: "kubemanager", Image: "kubemanager"}, + {Name: "init", Image: "busybox"}, + {Name: "init2", Image: "kubemanager"}, + }, + AuthMode: "keystone", + }, + KubemanagerNodesConfiguration: contrail.KubemanagerNodesConfiguration{ + ConfigNodesConfiguration: &contrail.ConfigClusterConfiguration{ + APIServerPort: 8082, + AnalyticsServerPort: 8081, + CollectorPort: 8086, + RedisPort: 6379, + AuthMode: contrail.AuthenticationModeKeystone, + }, + RabbbitmqNodesConfiguration: &contrail.RabbitmqClusterConfiguration{ + Port: 5673, + SSLPort: 15673, + }, + CassandraNodesConfiguration: &contrail.CassandraClusterConfiguration{ + Port: 9160, + CQLPort: 9042, + Endpoint: ":9160", + JMXPort: 7200, + }, + ZookeeperNodesConfiguration: &contrail.ZookeeperClusterConfiguration{ + ClientPort: 2181, + }, + KeystoneNodesConfiguration: &contrail.KeystoneClusterConfiguration{ + Port: 5555, + Endpoint: "10.11.12.13", + AuthProtocol: "https", + UserDomainName: "Default", + }, + }, + }, + }, + } + assertKubemanager(t, expectedKube, fakeClient) +} + +func assertKubemanager(t *testing.T, expected contrail.Kubemanager, fakeClient client.Client) { + kube := contrail.Kubemanager{} + err := fakeClient.Get(context.Background(), types.NamespacedName{ + Name: expected.Name, + Namespace: expected.Namespace, + }, &kube) + assert.NoError(t, err) + kube.SetResourceVersion("") + assert.Equal(t, expected, kube) } diff --git a/pkg/controller/memcached/memcached_controller.go b/pkg/controller/memcached/memcached_controller.go index 515a5a381..20466b0f6 100644 --- a/pkg/controller/memcached/memcached_controller.go +++ b/pkg/controller/memcached/memcached_controller.go @@ -188,7 +188,7 @@ func memcachedContainer(memcachedCR *contrail.Memcached) core.Container { return core.Container{ Name: "memcached", Image: instanceContainer.Image, - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, ReadinessProbe: &core.Probe{ InitialDelaySeconds: 5, TimeoutSeconds: 1, diff --git a/pkg/controller/memcached/memcached_controller_test.go b/pkg/controller/memcached/memcached_controller_test.go index 11cd068c7..d34a83480 100644 --- a/pkg/controller/memcached/memcached_controller_test.go +++ b/pkg/controller/memcached/memcached_controller_test.go @@ -286,7 +286,7 @@ func newExpectedDeployment() *apps.Deployment { Containers: []core.Container{{ Name: "memcached", Image: "localhost:5000/centos-binary-memcached:train", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, ReadinessProbe: &core.Probe{ InitialDelaySeconds: 5, TimeoutSeconds: 1, diff --git a/pkg/controller/postgres/postgres_controller.go b/pkg/controller/postgres/postgres_controller.go index 5bdf25b36..5d1689763 100644 --- a/pkg/controller/postgres/postgres_controller.go +++ b/pkg/controller/postgres/postgres_controller.go @@ -399,7 +399,7 @@ func (r *ReconcilePostgres) initContainers(postgres *contrail.Postgres) []core.C Name: "wait-for-ready-conf", Image: getImage(postgres.Spec.ServiceConfiguration.Containers, "wait-for-ready-conf"), Command: getCommand(postgres.Spec.ServiceConfiguration.Containers, "wait-for-ready-conf"), - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, VolumeMounts: []core.VolumeMount{ { Name: "status", @@ -411,7 +411,7 @@ func (r *ReconcilePostgres) initContainers(postgres *contrail.Postgres) []core.C Name: "init", Image: getImage(postgres.Spec.ServiceConfiguration.Containers, "init"), Command: []string{"/bin/sh", "-c", "if [[ -d /mnt/postgres/postgres ]]; then chmod 0750 /mnt/postgres/postgres; fi"}, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", VolumeMounts: []core.VolumeMount{ { Name: "postgres-storage-init", @@ -440,7 +440,7 @@ func (r *ReconcilePostgres) containers(postgres *contrail.Postgres, rootPassSecr TimeoutSeconds: 5, }, Env: r.containerEnv(postgres.Name, rootPassSecretName, replicationPassSecretName), - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", VolumeMounts: []core.VolumeMount{ { Name: "pgdata", diff --git a/pkg/controller/postgres/postgres_controller_test.go b/pkg/controller/postgres/postgres_controller_test.go index 8a21f9fb6..e2e746fee 100644 --- a/pkg/controller/postgres/postgres_controller_test.go +++ b/pkg/controller/postgres/postgres_controller_test.go @@ -508,7 +508,7 @@ func newSTS(name string) apps.StatefulSet { InitContainers: []core.Container{ { Name: "wait-for-ready-conf", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "localhost:5000/busybox:1.31", Command: []string{"sh", "-c", "until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done"}, VolumeMounts: []core.VolumeMount{{ @@ -520,7 +520,7 @@ func newSTS(name string) apps.StatefulSet { Name: "init", Image: "localhost:5000/busybox:1.31", Command: []string{"/bin/sh", "-c", "if [[ -d /mnt/postgres/postgres ]]; then chmod 0750 /mnt/postgres/postgres; fi"}, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", VolumeMounts: []core.VolumeMount{ { Name: "postgres-storage-init", @@ -534,7 +534,7 @@ func newSTS(name string) apps.StatefulSet { { Image: "localhost:5000/patroni:2.0.0.logical", Name: "patroni", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Env: []core.EnvVar{ nameEnv, scopeEnv, diff --git a/pkg/controller/provisionmanager/provisionmanager_controller.go b/pkg/controller/provisionmanager/provisionmanager_controller.go index 1327e7051..a87eab8ce 100644 --- a/pkg/controller/provisionmanager/provisionmanager_controller.go +++ b/pkg/controller/provisionmanager/provisionmanager_controller.go @@ -186,12 +186,6 @@ func (r *ReconcileProvisionManager) Reconcile(request reconcile.Request) (reconc return reconcile.Result{}, nil } - configInstance := &v1alpha1.Config{} - configActive := configInstance.IsActive(instance.Labels["contrail_cluster"], request.Namespace, r.Client) - if !configActive { - return reconcile.Result{}, nil - } - configMapConfigNodes, err := instance.CreateConfigMap(request.Name+"-"+instanceType+"-configmap-confignodes", r.Client, r.Scheme, request) if err != nil { return reconcile.Result{}, err diff --git a/pkg/controller/provisionmanager/provisionmanager_controller_test.go b/pkg/controller/provisionmanager/provisionmanager_controller_test.go index 1aaaddd14..94d82b0db 100644 --- a/pkg/controller/provisionmanager/provisionmanager_controller_test.go +++ b/pkg/controller/provisionmanager/provisionmanager_controller_test.go @@ -39,7 +39,6 @@ func TestProvisionManager(t *testing.T) { testcase1(), testcase2(), testcase3(), - testcase4(), } for _, tt := range tests { @@ -163,11 +162,10 @@ func TestProvisionManagerController(t *testing.T) { require.NoError(t, err, "Failed to build scheme") require.NoError(t, core.SchemeBuilder.AddToScheme(scheme), "Failed core.SchemeBuilder.AddToScheme()") require.NoError(t, apps.SchemeBuilder.AddToScheme(scheme), "Failed apps.SchemeBuilder.AddToScheme()") - pmr := newProvisionManager() + pmrs := newProvisionManagerService() initObjs := []runtime.Object{ - newManager(pmr), + newManager(pmrs), newConfigInst(), - pmr, } cl := fake.NewFakeClientWithScheme(scheme, initObjs...) @@ -257,6 +255,49 @@ func newProvisionManager() *contrail.ProvisionManager { }, }, Spec: contrail.ProvisionManagerSpec{ + CommonConfiguration: contrail.PodConfiguration{ + HostNetwork: &trueVal, + Replicas: &replica, + NodeSelector: map[string]string{"node-role.kubernetes.io/master": ""}, + }, + ServiceConfiguration: contrail.ProvisionManagerServiceConfiguration{ + ProvisionManagerConfiguration: contrail.ProvisionManagerConfiguration{ + Containers: []*contrail.Container{ + {Name: "provisioner", Image: "provisioner"}, + {Name: "init", Image: "busybox"}, + {Name: "init2", Image: "provisionmanager"}, + }, + }, + ProvisionManagerNodesConfiguration: contrail.ProvisionManagerNodesConfiguration{ + ConfigNodesConfiguration: &contrail.ConfigClusterConfiguration{ + APIServerIPList: []string{"1.1.1.1"}, + }, + }, + }, + }, + Status: contrail.ProvisionManagerStatus{ + Active: &trueVal, + }, + } +} + +func newProvisionManagerService() *contrail.ProvisionManagerService { + trueVal := true + replica := int32(1) + return &contrail.ProvisionManagerService{ + ObjectMeta: meta1.ObjectMeta{ + Name: "provisionmanager", + Namespace: "default", + Labels: map[string]string{"contrail_cluster": "cluster1"}, + OwnerReferences: []meta1.OwnerReference{ + { + Name: "cluster1", + Kind: "Manager", + Controller: &trueVal, + }, + }, + }, + Spec: contrail.ProvisionManagerServiceSpec{ CommonConfiguration: contrail.PodConfiguration{ HostNetwork: &trueVal, Replicas: &replica, @@ -270,13 +311,10 @@ func newProvisionManager() *contrail.ProvisionManager { }, }, }, - Status: contrail.ProvisionManagerStatus{ - Active: &trueVal, - }, } } -func newManager(pmr *contrail.ProvisionManager) *contrail.Manager { +func newManager(pmr *contrail.ProvisionManagerService) *contrail.Manager { return &contrail.Manager{ ObjectMeta: meta1.ObjectMeta{ Name: "cluster1", @@ -371,11 +409,11 @@ func compareConfigStatus(t *testing.T, expectedStatus, realStatus contrail.Provi } func testcase1() *TestCase { - pmr := newProvisionManager() + pmrs := newProvisionManagerService() tc := &TestCase{ name: "create a new statefulset", initObjs: []runtime.Object{ - newManager(pmr), + newManager(pmrs), newProvisionManager(), newConfigInst(), }, @@ -386,12 +424,13 @@ func testcase1() *TestCase { func testcase2() *TestCase { pmr := newProvisionManager() + pmrs := newProvisionManagerService() dt := meta1.Now() pmr.ObjectMeta.DeletionTimestamp = &dt tc := &TestCase{ name: "ProvisionManager deletion timestamp set", initObjs: []runtime.Object{ - newManager(pmr), + newManager(pmrs), newProvisionManager(), newConfigInst(), }, @@ -402,13 +441,14 @@ func testcase2() *TestCase { func testcase3() *TestCase { pmr := newProvisionManager() + pmrs := newProvisionManagerService() instanceContainer := utils.GetContainerFromList("provisioner", pmr.Spec.ServiceConfiguration.Containers) instanceContainer.Command = []string{"bash", "/runner/run.sh"} tc := &TestCase{ name: "Preset provisionmanager command verification", initObjs: []runtime.Object{ - newManager(pmr), + newManager(pmrs), newConfigInst(), newProvisionManager(), }, @@ -416,17 +456,3 @@ func testcase3() *TestCase { } return tc } - -func testcase4() *TestCase { - trueVal := true - pmr := newProvisionManager() - tc := &TestCase{ - name: "Preset provisionmanagerPod Test", - initObjs: []runtime.Object{ - newManager(pmr), - pmr, - }, - expectedStatus: contrail.ProvisionManagerStatus{Active: &trueVal}, - } - return tc -} diff --git a/pkg/controller/provisionmanager/sts.go b/pkg/controller/provisionmanager/sts.go index e5b436238..51e534a4f 100644 --- a/pkg/controller/provisionmanager/sts.go +++ b/pkg/controller/provisionmanager/sts.go @@ -36,7 +36,7 @@ spec: - -c - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init resources: {} securityContext: @@ -59,7 +59,7 @@ spec: fieldRef: fieldPath: metadata.name image: docker.io/kaweue/contrail-provisioner:master.1175 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/lib/provisionmanager name: provisionmanager-data diff --git a/pkg/controller/rabbitmq/sts.go b/pkg/controller/rabbitmq/sts.go index d6ce279bb..dee602fda 100644 --- a/pkg/controller/rabbitmq/sts.go +++ b/pkg/controller/rabbitmq/sts.go @@ -37,7 +37,7 @@ func GetSTS() *apps.StatefulSet { MountPath: "/tmp/podinfo", }, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", TerminationMessagePath: "/dev/termination-log", TerminationMessagePolicy: core.TerminationMessageReadFile, }, diff --git a/pkg/controller/swift/swift_controller.go b/pkg/controller/swift/swift_controller.go index a7144044b..fddbd0a8c 100644 --- a/pkg/controller/swift/swift_controller.go +++ b/pkg/controller/swift/swift_controller.go @@ -312,7 +312,7 @@ func (r *ReconcileSwift) removeRingReconcilingJobs(swiftName types.NamespacedNam err := r.client.Get(context.Background(), jobName, ringJob) existingJob := err == nil if existingJob { - if job.Status(ringJob.Status).Pending() { + if job.Status(ringJob.Status).JobPending() { // Wait until job finish executing to avoid breaking the ongoing ring reconciliation return reconcile.Result{ Requeue: true, diff --git a/pkg/controller/swiftproxy/swiftproxy_controller.go b/pkg/controller/swiftproxy/swiftproxy_controller.go index 690390749..cd5582885 100644 --- a/pkg/controller/swiftproxy/swiftproxy_controller.go +++ b/pkg/controller/swiftproxy/swiftproxy_controller.go @@ -133,8 +133,9 @@ func (r *ReconcileSwiftProxy) Reconcile(request reconcile.Request) (reconcile.Re return reconcile.Result{}, err } + annotations := swiftProxy.GetServiceAnnotations() servicePortsMap := map[int32]string{int32(swiftProxy.Spec.ServiceConfiguration.ListenPort): ""} - svc := r.kubernetes.Service(request.Name+"-swiftproxy", core.ServiceTypeLoadBalancer, servicePortsMap, contrail.SwiftProxyInstanceType, swiftProxy) + svc := r.kubernetes.Service(request.Name+"-swiftproxy", swiftProxy.GetServiceType(), servicePortsMap, contrail.SwiftProxyInstanceType, swiftProxy).WithAnnotations(annotations) if err := svc.EnsureExists(); err != nil { return reconcile.Result{}, err } @@ -338,7 +339,7 @@ func updatePodTemplate( pod.InitContainers = []core.Container{ { Name: "wait-for-ready-conf", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: getImage(containers, "wait-for-ready-conf"), Command: getCommand(containers, "wait-for-ready-conf"), VolumeMounts: []core.VolumeMount{{ diff --git a/pkg/controller/swiftproxy/swiftproxy_controller_test.go b/pkg/controller/swiftproxy/swiftproxy_controller_test.go index 8ceef41f9..0b2e6eff3 100644 --- a/pkg/controller/swiftproxy/swiftproxy_controller_test.go +++ b/pkg/controller/swiftproxy/swiftproxy_controller_test.go @@ -347,7 +347,7 @@ func newExpectedDeployment(status apps.DeploymentStatus) *apps.Deployment { InitContainers: []core.Container{ { Name: "wait-for-ready-conf", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "localhost:5000/busybox", Command: []string{"sh", "-c", expectedCommandWaitForReadyContainer}, VolumeMounts: []core.VolumeMount{{ @@ -500,7 +500,7 @@ func newExpectedDeploymentWithCustomImages() *apps.Deployment { deployment.Spec.Template.Spec.InitContainers = []core.Container{ { Name: "wait-for-ready-conf", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "image3", Command: []string{"cmd"}, VolumeMounts: []core.VolumeMount{{ diff --git a/pkg/controller/swiftproxy/swiftproxy_register.go b/pkg/controller/swiftproxy/swiftproxy_register.go index fdd4bcccc..29eba4df7 100644 --- a/pkg/controller/swiftproxy/swiftproxy_register.go +++ b/pkg/controller/swiftproxy/swiftproxy_register.go @@ -161,7 +161,7 @@ func newBootstrapJob( { Name: "register", Image: getImage(containers, "init"), - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Command: getCommand(containers, "init"), VolumeMounts: []core.VolumeMount{ {Name: "register", MountPath: "/var/lib/ansible/register", ReadOnly: true}, diff --git a/pkg/controller/utils/utils_test.go b/pkg/controller/utils/utils_test.go index db9330a53..624c99a2a 100644 --- a/pkg/controller/utils/utils_test.go +++ b/pkg/controller/utils/utils_test.go @@ -550,7 +550,7 @@ const expectedCommandImage = `DB_USER=${DB_USER:-root}` var InitContainers = []core.Container{ { Name: "wait-for-ready-conf", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Image: "localhost:5000/busybox", Command: []string{"sh", "-c", expectedCommandWaitForReadyContainer}, VolumeMounts: []core.VolumeMount{{ @@ -561,7 +561,7 @@ var InitContainers = []core.Container{ { Name: "keystone-db-init", Image: "localhost:5000/postgresql-client", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Command: []string{"/bin/sh"}, Args: []string{"-c", expectedCommandImage}, Env: []core.EnvVar{ @@ -578,7 +578,7 @@ var InitContainers = []core.Container{ { Name: "keystone-init", Image: "localhost:5000/centos-binary-keystone:train", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Env: []core.EnvVar{{ Name: "KOLLA_SERVICE_NAME", Value: "keystone", diff --git a/pkg/controller/vrouter/daemonset.go b/pkg/controller/vrouter/daemonset.go index eb003948e..51501a376 100644 --- a/pkg/controller/vrouter/daemonset.go +++ b/pkg/controller/vrouter/daemonset.go @@ -52,7 +52,7 @@ func GetDaemonset() *apps.DaemonSet { MountPath: "/tmp/podinfo", }, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", }, { Name: "nodeinit", @@ -67,7 +67,7 @@ func GetDaemonset() *apps.DaemonSet { MountPath: "/host/usr/bin", }, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", SecurityContext: &core.SecurityContext{ Privileged: &trueVal, }, @@ -100,7 +100,7 @@ func GetDaemonset() *apps.DaemonSet { MountPath: "/lib/modules", }, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", SecurityContext: &core.SecurityContext{ Privileged: &trueVal, }, @@ -153,7 +153,7 @@ func GetDaemonset() *apps.DaemonSet { MountPath: "/etc/resolv.conf", }, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", SecurityContext: &core.SecurityContext{ Privileged: &trueVal, }, @@ -189,7 +189,7 @@ func GetDaemonset() *apps.DaemonSet { MountPath: "/mnt", }, }, - ImagePullPolicy: "Always", + ImagePullPolicy: "IfNotPresent", }, } diff --git a/pkg/controller/webui/sts.go b/pkg/controller/webui/sts.go index 25f9615fb..e714ecef5 100644 --- a/pkg/controller/webui/sts.go +++ b/pkg/controller/webui/sts.go @@ -29,7 +29,7 @@ spec: - sh - -c - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done - imagePullPolicy: Always + imagePullPolicy: IfNotPresent resources: {} securityContext: privileged: false @@ -55,7 +55,7 @@ spec: value: "true" - name: ANALYTICS_SNMP_ENABLE value: "true" - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: webui-logs @@ -70,7 +70,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: webui-logs @@ -81,7 +81,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /var/log/contrail name: webui-logs diff --git a/pkg/controller/zookeeper/sts.go b/pkg/controller/zookeeper/sts.go index 122fa4d34..df7bc6572 100644 --- a/pkg/controller/zookeeper/sts.go +++ b/pkg/controller/zookeeper/sts.go @@ -37,7 +37,7 @@ spec: - -c - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init resources: {} securityContext: @@ -58,7 +58,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: conf-init resources: {} securityContext: @@ -78,7 +78,7 @@ spec: fieldRef: fieldPath: metadata.name image: docker.io/michaelhenkel/contrail-external-zookeeper:5.2.0-dev1 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent resources: requests: memory: "1Gi" diff --git a/pkg/job/job.go b/pkg/job/job.go index 822967f39..a9dbb42ad 100644 --- a/pkg/job/job.go +++ b/pkg/job/job.go @@ -7,7 +7,7 @@ import ( type Status batch.JobStatus -func (s Status) Pending() bool { +func (s Status) JobPending() bool { if len(s.Conditions) == 0 { return true } @@ -19,7 +19,7 @@ func (s Status) Pending() bool { return true } -func (s Status) Completed() bool { +func (s Status) JobComplete() bool { if len(s.Conditions) == 0 { return false } @@ -30,3 +30,15 @@ func (s Status) Completed() bool { } return false } + +func (s Status) JobFailed() bool { + if len(s.Conditions) == 0 { + return false + } + for _, condition := range s.Conditions { + if condition.Status == v1.ConditionTrue && condition.Type == batch.JobFailed { + return true + } + } + return false +} diff --git a/pkg/job/job_test.go b/pkg/job/job_test.go index 70a0f9189..2cc494fdf 100644 --- a/pkg/job/job_test.go +++ b/pkg/job/job_test.go @@ -76,7 +76,7 @@ func TestStatus_Pending(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { // when - pending := job.Status(test.jobStatus).Pending() + pending := job.Status(test.jobStatus).JobPending() // then assert.Equal(t, test.expectedPending, pending) }) diff --git a/pkg/k8s/service.go b/pkg/k8s/service.go index 7a5cd0ec4..fec788c5c 100644 --- a/pkg/k8s/service.go +++ b/pkg/k8s/service.go @@ -15,15 +15,16 @@ import ( // Service is used to create and manage kubernetes services type Service struct { - name string - servType core.ServiceType - ports map[int32]string - ownerType string - labels map[string]string - owner v1.Object - scheme *runtime.Scheme - client client.Client - svc core.Service + name string + servType core.ServiceType + ports map[int32]string + ownerType string + labels map[string]string + annotations map[string]string + owner v1.Object + scheme *runtime.Scheme + client client.Client + svc core.Service } // EnsureExists is used to make sure that kubernetes service exists and is correctly configured @@ -34,9 +35,10 @@ func (s *Service) EnsureExists() error { } s.svc = core.Service{ ObjectMeta: meta.ObjectMeta{ - Name: s.name, - Namespace: s.owner.GetNamespace(), - Labels: labels, + Name: s.name, + Namespace: s.owner.GetNamespace(), + Labels: labels, + Annotations: s.annotations, }, } _, err := controllerutil.CreateOrUpdate(context.Background(), s.client, &s.svc, func() error { @@ -92,3 +94,9 @@ func (s *Service) WithLabels(labels map[string]string) *Service { s.labels = labels return s } + +// WithAnnotations is used to set annotations on Service +func (s *Service) WithAnnotations(annotations map[string]string) *Service { + s.annotations = annotations + return s +} diff --git a/pkg/swift/ring/ring.go b/pkg/swift/ring/ring.go index 2e6c4404d..6167fb992 100644 --- a/pkg/swift/ring/ring.go +++ b/pkg/swift/ring/ring.go @@ -103,7 +103,7 @@ func (r *Ring) BuildJob(name types.NamespacedName, nodeSelector map[string]strin { Name: "contrail-operator-ringcontroller", Image: "localhost:5000/contrail-operator/engprod-269421/contrail-operator-ringcontroller:master.latest", - ImagePullPolicy: core.PullAlways, + ImagePullPolicy: core.PullIfNotPresent, Args: r.args(), }, }, diff --git a/pods/cassandra.yaml b/pods/cassandra.yaml index 21ffe1e6f..c67583d71 100644 --- a/pods/cassandra.yaml +++ b/pods/cassandra.yaml @@ -9,7 +9,7 @@ spec: terminationGracePeriodSeconds: 1800 containers: - image: hub.juniper.net/contrail-nightly/contrail-external-cassandra:5.2.0-0.740 - imagePullPolicy: Always + imagePullPolicy: IfNotPresent env: - name: POD_IP valueFrom: @@ -54,7 +54,7 @@ spec: - -c - until grep ready /tmp/podinfo/pod_labels > /dev/null 2>&1; do sleep 1; done image: busybox - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: init resources: {} securityContext: diff --git a/test/e2e/aio/manager_test.go b/test/e2e/aio/manager_test.go index 791ff7b08..0151b2d45 100644 --- a/test/e2e/aio/manager_test.go +++ b/test/e2e/aio/manager_test.go @@ -311,13 +311,13 @@ func getManager(namespace string, replicas int32, hostNetwork bool, versionMap m }, }, }}, - ProvisionManager: &v1alpha1.ProvisionManager{ + ProvisionManager: &v1alpha1.ProvisionManagerService{ ObjectMeta: metav1.ObjectMeta{ Name: "provmanager1", Namespace: namespace, Labels: map[string]string{"contrail_cluster": "cluster1"}, }, - Spec: v1alpha1.ProvisionManagerSpec{ + Spec: v1alpha1.ProvisionManagerServiceSpec{ CommonConfiguration: v1alpha1.PodConfiguration{ Replicas: &replicas, }, diff --git a/test/e2e/ha/command_test.go b/test/e2e/ha/command_test.go index ea5922e9f..82af7eaad 100644 --- a/test/e2e/ha/command_test.go +++ b/test/e2e/ha/command_test.go @@ -21,6 +21,7 @@ import ( contrail "github.com/Juniper/contrail-operator/pkg/apis/contrail/v1alpha1" "github.com/Juniper/contrail-operator/pkg/client/keystone" "github.com/Juniper/contrail-operator/pkg/client/kubeproxy" + "github.com/Juniper/contrail-operator/pkg/controller/utils" "github.com/Juniper/contrail-operator/test/logger" "github.com/Juniper/contrail-operator/test/wait" ) @@ -178,6 +179,48 @@ func TestHACommand(t *testing.T) { }) + t.Run("when upgrade to invalid image is performed", func(t *testing.T) { + badImage := "registry:5000/common-docker-third-party/contrail/busybox:1.31" + require.NoError(t, f.Client.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: "command-ha"}, cluster)) + containers := cluster.Spec.Services.Command.Spec.ServiceConfiguration.Containers + utils.GetContainerFromList("api", containers).Image = badImage + require.NoError(t, f.Client.Update(context.TODO(), cluster)) + + t.Run("then command reports failed upgrade", func(t *testing.T) { + err := wait.Contrail{ + Namespace: cluster.Namespace, + Timeout: 5 * time.Minute, + RetryInterval: retryInterval, + Client: f.Client, + Logger: log, + }.ForCommandUpgradeState("command", contrail.CommandUpgradeFailed) + require.NoError(t, err) + }) + }) + + t.Run("when previous image is restored", func(t *testing.T) { + goodImage := "registry:5000/contrail-nightly/contrail-command:" + cemRelease + require.NoError(t, f.Client.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: "command-ha"}, cluster)) + containers := cluster.Spec.Services.Command.Spec.ServiceConfiguration.Containers + utils.GetContainerFromList("api", containers).Image = goodImage + require.NoError(t, f.Client.Update(context.TODO(), cluster)) + + t.Run("then command reports not upgrading state", func(t *testing.T) { + err := wait.Contrail{ + Namespace: cluster.Namespace, + Timeout: 5 * time.Minute, + RetryInterval: retryInterval, + Client: f.Client, + Logger: log, + }.ForCommandUpgradeState("command", contrail.CommandNotUpgrading) + require.NoError(t, err) + }) + + t.Run("then Command services is correctly responding", func(t *testing.T) { + assertCommandServiceIsResponding(t, proxy, f, namespace) + }) + }) + t.Run("when reference cluster is deleted", func(t *testing.T) { pp := meta.DeletePropagationForeground err = f.Client.Delete(context.TODO(), cluster, &client.DeleteOptions{ diff --git a/test/e2e/ha/core_contrail_services_test.go b/test/e2e/ha/core_contrail_services_test.go index 60d4bea63..58bf7de2f 100644 --- a/test/e2e/ha/core_contrail_services_test.go +++ b/test/e2e/ha/core_contrail_services_test.go @@ -569,13 +569,13 @@ func getHACluster(namespace, nodeLabel, storagePath string) *contrail.Manager { }, } - provisionManager := &contrail.ProvisionManager{ + provisionManager := &contrail.ProvisionManagerService{ ObjectMeta: meta.ObjectMeta{ Name: "hatest-provmanager", Namespace: namespace, Labels: map[string]string{"contrail_cluster": "cluster1"}, }, - Spec: contrail.ProvisionManagerSpec{ + Spec: contrail.ProvisionManagerServiceSpec{ CommonConfiguration: contrail.PodConfiguration{ Replicas: &oneVal, }, diff --git a/test/wait/contrail.go b/test/wait/contrail.go index 259ff0323..d5476350f 100644 --- a/test/wait/contrail.go +++ b/test/wait/contrail.go @@ -171,6 +171,26 @@ func (c Contrail) ForManagerDeletion(name string) error { return err } +// ForCommandUpgradeStateChange is used to wait until Command Upgrade State is updated +func (c Contrail) ForCommandUpgradeState(name string, newState contrail.CommandUpgradeState) error { + command := &contrail.Command{} + err := wait.Poll(c.RetryInterval, c.Timeout, func() (done bool, err error) { + err = c.Client.Get(context.Background(), types.NamespacedName{ + Namespace: c.Namespace, + Name: name, + }, command) + if apierrors.IsNotFound(err) { + return false, nil + } + if command.Status.UpgradeState == newState { + return true, nil + } + return false, err + }) + c.dumpPodsOnError(err) + return err +} + func (c Contrail) dumpPodsOnError(err error) { if err != nil { c.Logger.DumpPods()