Switch control-plane v2

openapi/config-values.yaml

@@ -66,3 +66,7 @@ properties:
         type: string
         default: "0 3 * * *"
         description: Backup schedule
+  useLinstor:
+    type: boolean
+    default: true
+    description: Use LINSTOR as a backend for the module

openapi/doc-ru-config-values.yaml

@@ -1,7 +1,18 @@
 type: object
 properties:
+  registryScheme:
+    description: Устаревший параметр. Удалите его из ModuleConfig для Deckhouse версии выше 1.57.
   logLevel:
     description: Уровень логирования для приложений модуля
+  drbdPortRange:
+    description: Настройки TCP-портов DRBD
+    properties:
+      minPort:
+        description: Начало диапазона портов DRBD
+      maxPort:
+        description: Конец диапазона портов DRBD
+  enableThinProvisioning:
+    description: Разрешить использование тонких томов LVM
   dataNodes:
     description: Настройки для узлов Linstor с данными
     properties:
@@ -20,3 +31,5 @@ properties:
         description: Количество  резервных копий для хранения
       schedule:
         description: Расписание резервного копирования
+  useLinstor:
+    description: Использовать LINSTOR как бэкенд для модуля

templates/agent/configmap.yaml → templates/agent/configmap-agent-config.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.sdsReplicatedVolume.useLinstor }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -20,3 +21,4 @@ data:
 
     render=true
     stringValues=true
+{{- end }}

templates/agent/configmap-drbd-reactor-config.yaml

@@ -0,0 +1,15 @@
+{{- if not .Values.sdsReplicatedVolume.useLinstor }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: drbd-reactor-config
+  namespace: d8-{{ .Chart.Name }}
+  {{- include "helm_lib_module_labels" (list . (dict "app" "agent")) | nindent 2 }}
+data:
+  prometheus.toml: |2
+
+    [[prometheus]]
+    address = "127.0.0.1:4215"
+    enums = true
+{{- end }}

templates/agent/daemonset.yaml

@@ -1,4 +1,4 @@
-{{- define "sds_utils_installer_resources" }}
+{{- define "drbd_wait_resources" }}
 cpu: 10m
 memory: 25Mi
 {{- end }}
@@ -8,6 +8,12 @@ cpu: 50m
 memory: 50Mi
 {{- end }}
 
+{{- define "drbd_prometheus_exporter_resources" }}
+cpu: 50m
+memory: 25Mi
+{{- end }}
+
+{{- if not .Values.sdsReplicatedVolume.useLinstor }}
 {{- if (.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }}
 ---
 apiVersion: autoscaling.k8s.io/v1
@@ -31,9 +37,14 @@ spec:
       maxAllowed:
         cpu: 200m
         memory: 100Mi
+    - containerName: drbd-prometheus-exporter
+      minAllowed:
+        {{- include "drbd_prometheus_exporter_resources" . | nindent 8 }}
+      maxAllowed:
+        cpu: 100m
+        memory: 50Mi
 {{- end }}
 
-{{- if not .Values.sdsReplicatedVolume.disableDs }}
 ---
 apiVersion: apps/v1
 kind: DaemonSet
@@ -53,6 +64,7 @@ spec:
     spec:
       {{- include "helm_lib_priority_class" (tuple . "cluster-medium") | nindent 6 }}
       {{- include "helm_lib_tolerations" (tuple . "any-node" "storage-problems") | nindent 6 }}
+      {{- include "helm_lib_module_pod_security_context_run_as_user_root" . | nindent 6 }}
       affinity: {}
       nodeSelector:
         storage.deckhouse.io/sds-replicated-volume-node: ""
@@ -61,15 +73,24 @@ spec:
         - name: {{ .Chart.Name }}-module-registry
       serviceAccountName: agent
       hostNetwork: true
-      # We need root privileges to perform drbd operations on the node.
-      securityContext:
-        runAsUser: 0
-        runAsNonRoot: false
-        runAsGroup: 0
-        readOnlyRootFilesystem: true
-        seLinuxOptions:
-          level: s0
-          type: spc_t
+      initContainers:
+      - name: wait-drbd-ver-9-2
+        command:
+          - /linstor-drbd-wait
+        env:
+          - name: FILE_CONTENT
+            value: "version: 9.2"
+          - name: WAITING_MSG
+            value: "Waiting for DRBD version 9.2.x on host"
+        image: {{ include "helm_lib_module_image" (list . "linstorDrbdWait") }}
+        imagePullPolicy: IfNotPresent
+        {{- include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all" . | nindent 8 }}
+        resources:
+          requests:
+            {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 12 }}
+            {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }}
+              {{- include "drbd_wait_resources" . | nindent 12 }}
+            {{- end }}
       containers:
       - name: agent
         image: {{ include "helm_lib_module_image" (list . "agent") }}
@@ -116,15 +137,93 @@ spec:
           name: var-lock
         resources:
           requests:
-            {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }}
-{{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }}
-            {{- include "sds_replicated_volume_agent_resources" . | nindent 14 }}
-{{- end }}
+            {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 12 }}
+            {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }}
+              {{- include "sds_replicated_volume_agent_resources" . | nindent 12 }}
+            {{- end }}
+      - name: drbd-prometheus-exporter
+        image: {{ include "helm_lib_module_image" (list . "drbdReactor") }}
+        imagePullPolicy: IfNotPresent
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        securityContext:
+          readOnlyRootFilesystem: true
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            host: 127.0.0.1
+            path: /
+            port: 4215
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        ports:
+          - containerPort: 4215
+            hostPort: 4215
+            name: prometheus
+            protocol: TCP
+        resources:
+          requests:
+            {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 12 }}
+            {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }}
+              {{- include "drbd_prometheus_exporter_resources" . | nindent 12 }}
+            {{- end }}
+        volumeMounts:
+          - mountPath: /etc/drbd-reactor.d/
+            name: drbd-reactor-config
+      - name: kube-rbac-proxy
+        image: {{ include "helm_lib_module_common_image" (list . "kubeRbacProxy") }}
+        imagePullPolicy: IfNotPresent
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        args:
+          - --secure-listen-address=$(KUBE_RBAC_PROXY_LISTEN_ADDRESS):4215
+          - --v=2
+          - --logtostderr=true
+          - --stale-cache-interval=1h30m
+          - "--livez-path=/livez"
+        env:
+          - name: KUBE_RBAC_PROXY_LISTEN_ADDRESS
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: status.podIP
+          - name: KUBE_RBAC_PROXY_CONFIG
+            value: |
+              upstreams:
+              - upstream: http://127.0.0.1:4215
+                path: /
+                authorization:
+                  resourceAttributes:
+                    namespace: d8-{{ .Chart.Name }}
+                    apiGroup: apps
+                    apiVersion: v1
+                    resource: daemonsets
+                    subresource: prometheus-metrics
+                    name: agent
+        livenessProbe:
+          httpGet:
+            path: /livez
+            port: 4215
+            scheme: HTTPS
+        readinessProbe:
+          httpGet:
+            path: /livez
+            port: 4215
+            scheme: HTTPS
+        resources:
+          requests:
+            {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 12 }}
+            {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }}
+              {{- include "helm_lib_container_kube_rbac_proxy_resources" . | nindent 12 }}
+            {{- end }}
+        {{- include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all" . | nindent 8 }}
       volumes:
-      - hostPath:
+      - name: host-device-dir
+        hostPath:
           path: /dev/
           type: ""
-        name: host-device-dir
       - name: config
         configMap:
           name: agent-config
@@ -139,4 +238,8 @@ spec:
         emptyDir: {}
       - name: var-lock
         emptyDir: {}
+      - name: drbd-reactor-config
+        configMap:
+          defaultMode: 420
+          name: drbd-reactor-config
 {{- end }}

templates/agent/podmonitor.yaml

@@ -0,0 +1,39 @@
+{{- if not .Values.sdsReplicatedVolume.useLinstor }}
+{{- if (.Values.global.enabledModules | has "operator-prometheus-crd") }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: sds-replicated-volume-agent
+  namespace: d8-monitoring
+  {{- include "helm_lib_module_labels" (list $ (dict "prometheus" "main")) | nindent 2 }}
+spec:
+  podMetricsEndpoints:
+  - port: prometheus
+    scheme: https
+    path: /metrics
+    bearerTokenSecret:
+      name: "prometheus-token"
+      key: "token"
+    tlsConfig:
+      insecureSkipVerify: true
+    relabelings:
+    - regex: endpoint|namespace|pod|container
+      action: labeldrop
+    - targetLabel: job
+      replacement: sds-replicated-volume-agent
+    - sourceLabels: [__meta_kubernetes_pod_node_name]
+      targetLabel: node
+    - targetLabel: tier
+      replacement: cluster
+    - sourceLabels: [__meta_kubernetes_pod_ready]
+      regex: "true"
+      action: keep
+  selector:
+    matchLabels:
+      app: agent
+  namespaceSelector:
+    matchNames:
+    - d8-{{ .Chart.Name }}
+{{- end }}
+{{- end }}

templates/agent/rbac-for-us.yaml

@@ -1,15 +1,17 @@
+{{- if not .Values.sdsReplicatedVolume.useLinstor }}
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: agent
   namespace: d8-{{ .Chart.Name }}
-  {{- include "helm_lib_module_labels" (list .) | nindent 2 }}
+  {{- include "helm_lib_module_labels" (list . (dict "app" "agent")) | nindent 2 }}
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: d8:{{ .Chart.Name }}:sds-replicated-volume
-  {{- include "helm_lib_module_labels" (list .) | nindent 2 }}
+  name: d8:{{ .Chart.Name }}:agent
+  {{- include "helm_lib_module_labels" (list . (dict "app" "agent")) | nindent 2 }}
 rules:
   - apiGroups: ["storage.deckhouse.io"]
     resources: ["replicatedvolumereplicas"]
@@ -21,13 +23,28 @@ rules:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: d8:{{ .Chart.Name }}:sds-replicated-volume
-  {{- include "helm_lib_module_labels" (list .) | nindent 2 }}
+  name: d8:{{ .Chart.Name }}:agent
+  {{- include "helm_lib_module_labels" (list . (dict "app" "agent")) | nindent 2 }}
 subjects:
   - kind: ServiceAccount
     name: agent
     namespace: d8-{{ .Chart.Name }}
 roleRef:
   kind: ClusterRole
-  name: d8:{{ .Chart.Name }}:sds-replicated-volume
+  name: d8:{{ .Chart.Name }}:agent
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: d8:{{ .Chart.Name }}:agent-rbac-proxy
+  {{- include "helm_lib_module_labels" (list . (dict "app" "agent")) | nindent 2 }}
+subjects:
+  - kind: ServiceAccount
+    name: agent
+    namespace: d8-{{ .Chart.Name }}
+roleRef:
   apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: d8:rbac-proxy
+{{- end }}

templates/certs.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.sdsReplicatedVolume.useLinstor }}
 ---
 apiVersion: v1
 kind: Secret
@@ -54,3 +55,4 @@ data:
   tls.key: {{ .key | b64enc }}
   ca.crt: {{ .ca | b64enc }}
   {{- end }}
+{{- end }}

templates/controller/configmap.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.sdsReplicatedVolume.useLinstor }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -22,3 +23,4 @@ data:
 
     render=true
     stringValues=true
+{{- end }}