Support enterprise encryption

CHANGELOG.md

@@ -7,6 +7,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 # [Unreleased](https://github.com/cockroachdb/cockroach-operator/compare/v2.8.0...master)
 
+## Added
+
+* Add support for enabling Enterprise Encryption at Rest
+
 # [v2.8.0](https://github.com/cockroachdb/cockroach-operator/compare/v2.7.0...v2.8.0)
 
 ## Added

DEVELOPER.md

@@ -116,7 +116,7 @@ The examples directory contains various examples, for example you can run `kubec
 First you need to create a new Kubernetes Custom Resource, in this example we will use the example/example.yaml file.
 
 ```bash
-kubectl create -f example/example.yaml
+kubectl create -f examples/example.yaml
 ```
 
 When the database is up and running run the following command to get the first pod that is creasted.

apis/v1alpha1/cluster_types.go

@@ -145,6 +145,26 @@ type CrdbClusterSpec struct {
 	// Default: false
 	// +optional
 	AutomountServiceAccountToken bool `json:"automountServiceAccountToken,omitempty"`
+	// (Optional) EncryptionEnabled determines if enterprise encryption is enabled for your CockroachDB Cluster
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Enterprise Encryption Enabled",xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
+	// +optional
+	EncryptionEnabled bool `json:"encryptionEnabled,omitempty"`
+	// (Optional) The secret with the encryption store keys
+	// The naming of files is expected as (key) for the active store key
+	// and (old-key) for the previous store key.
+	// Default: ""
+	// +optional
+	EncryptionStoreKeySecret string `json:"encryptionStoreKeySecret,omitempty"`
+	// (Optional) Whether the active encryption type is plaintext. This is only applicable
+	// if encryptionEnabled is set to true
+	// Default: ""
+	// +optional
+	EncryptionTypePlain bool `json:"encryptionTypePlain,omitempty"`
+	// (Optional) Whether the previous encryption type is plaintext. This is only applicable
+	// if encryptionEnabled is set to true
+	// Default: ""
+	// +optional
+	OldEncryptionTypePlain bool `json:"oldEncryptionTypePlain,omitempty"`
 }
 
 // +k8s:openapi-gen=true

config/crd/bases/crdb.cockroachlabs.com_crdbclusters.yaml

@@ -851,6 +851,20 @@ spec:
                       resize without restarting the entire cluster Default: false'
                     type: boolean
                 type: object
+              encryptionEnabled:
+                description: (Optional) EncryptionEnabled determines if enterprise
+                  encryption is enabled for your CockroachDB Cluster
+                type: boolean
+              encryptionStoreKeySecret:
+                description: '(Optional) The secret with the encryption store keys
+                  The naming of files is expected as (key) for the active store key
+                  and (old-key) for the previous store key. Default: ""'
+                type: string
+              encryptionTypePlain:
+                description: '(Optional) Whether the active encryption type is plaintext.
+                  This is only applicable if encryptionEnabled is set to true Default:
+                  ""'
+                type: boolean
               grpcPort:
                 description: '(Optional) The database port (`--port` CLI parameter
                   when starting the service) Default: 26258'
@@ -1024,6 +1038,11 @@ spec:
                 format: int32
                 minimum: 3
                 type: integer
+              oldEncryptionTypePlain:
+                description: '(Optional) Whether the previous encryption type is plaintext.
+                  This is only applicable if encryptionEnabled is set to true Default:
+                  ""'
+                type: boolean
               podEnvVariables:
                 description: '(Optional) PodEnvVariables is a slice of environment
                   variables that are added to the pods Default: (empty list)'

examples/example-with-encryption.yaml

@@ -0,0 +1,72 @@
+# Copyright 2022 The Cockroach Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Generated, do not edit. Please edit this file instead: config/templates/example.yaml.in
+#
+
+apiVersion: crdb.cockroachlabs.com/v1alpha1
+kind: CrdbCluster
+metadata:
+  # this translates to the name of the statefulset that is created
+  name: cockroachdb
+spec:
+  dataStore:
+    pvc:
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: "10Gi"
+        volumeMode: Filesystem
+  resources:
+    requests:
+      # This is intentionally low to make it work on local k3d clusters.
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: 2
+      memory: 8Gi
+  tlsEnabled: true
+  encryptionEnabled: true
+  encryptionStoreKeySecret: store-encryption-key
+  encryptionTypePlain: false
+  oldEncryptionTypePlain: true
+  image:
+    name: cockroachdb/cockroach:v22.1.7
+  # nodes refers to the number of crdb pods that are created
+  # via the statefulset
+  nodes: 3
+  additionalLabels:
+    crdb: is-cool
+  # affinity is a new API field that is behind a feature gate that is
+  # disabled by default.  To enable please see the operator.yaml file.
+
+  # The affinity field will accept any podSpec affinity rule.
+  # affinity:
+  #   podAntiAffinity:
+  #      preferredDuringSchedulingIgnoredDuringExecution:
+  #      - weight: 100
+  #        podAffinityTerm:
+  #          labelSelector:
+  #            matchExpressions:
+  #            - key: app.kubernetes.io/instance
+  #              operator: In
+  #              values:
+  #              - cockroachdb
+  #          topologyKey: kubernetes.io/hostname
+
+  # nodeSelectors used to match against
+  # nodeSelector:
+  #   worker-pool-name: crdb-workers

pkg/resource/cluster.go

@@ -199,7 +199,7 @@ func (cluster Cluster) LookupSupportedVersion(version string) (string, bool) {
 	return "", false
 }
 
-//GetVersionAnnotation  gets the current version of the cluster  retrieved by version checker action
+// GetVersionAnnotation  gets the current version of the cluster  retrieved by version checker action
 func (cluster Cluster) GetVersionAnnotation() string {
 	return cluster.getAnnotation(CrdbVersionAnnotation)
 }
@@ -303,6 +303,14 @@ func (cluster Cluster) ClientTLSSecretName() string {
 
 	return fmt.Sprintf("%s-root", cluster.Name())
 }
+
+func (cluster Cluster) EncryptionKeySecretName() string {
+	if cluster.Spec().EncryptionStoreKeySecret != "" {
+		return cluster.Spec().EncryptionStoreKeySecret
+	}
+	return "store-encryption-key"
+}
+
 func (cluster Cluster) CASecretName() string {
 	return fmt.Sprintf("%s-ca", cluster.Name())
 }

pkg/resource/statefulset.go

@@ -42,6 +42,9 @@ const (
 	dataDirName      = "datadir"
 	dataDirMountPath = "/cockroach/cockroach-data/"
 
+	encryptionKeyDirName      = "encryptionkeys"
+	encryptionKeyDirMountPath = "/cockroach/encryption-keys/"
+
 	certsDirName = "certs"
 	certCpCmd    = ">- cp -p /cockroach/cockroach-certs-prestage/..data/* /cockroach/cockroach-certs/ && chmod 700 /cockroach/cockroach-certs/*.key && chown 1000581000:1000581000 /cockroach/cockroach-certs/*.key"
 	emptyDirName = "emptydir"
@@ -98,6 +101,51 @@ func (b StatefulSetBuilder) Build(obj client.Object) error {
 		return err
 	}
 
+	if b.Spec().EncryptionEnabled {
+		if err := addStoreKeysVolumeMountOnInitContiners(DbContainerName, &ss.Spec.Template.Spec); err != nil {
+			return err
+		}
+		if err := addStoreKeysVolumeMount(DbContainerName, &ss.Spec.Template.Spec); err != nil {
+			return err
+		}
+		items := make([]corev1.KeyToPath, 0)
+		if b.Spec().EncryptionStoreKeySecret != "" {
+			if !b.Spec().EncryptionTypePlain {
+				items = append(items, corev1.KeyToPath{
+					Key:  "key",
+					Path: "key",
+					Mode: ptr.Int32(400),
+				})
+			}
+
+			if !b.Spec().OldEncryptionTypePlain {
+				items = append(items, corev1.KeyToPath{
+					Key:  "old-key",
+					Path: "old-key",
+					Mode: ptr.Int32(400),
+				})
+			}
+		}
+		ss.Spec.Template.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
+			Name: encryptionKeyDirName,
+			VolumeSource: corev1.VolumeSource{
+				Projected: &corev1.ProjectedVolumeSource{
+					DefaultMode: ptr.Int32(400),
+					Sources: []corev1.VolumeProjection{
+						{
+							Secret: &corev1.SecretProjection{
+								LocalObjectReference: corev1.LocalObjectReference{
+									Name: b.encryptionKeySecretName(),
+								},
+								Items: items,
+							},
+						},
+					},
+				},
+			},
+		})
+	}
+
 	if b.Spec().TLSEnabled {
 		if err := addCertsVolumeMountOnInitContiners(DbContainerName, &ss.Spec.Template.Spec); err != nil {
 			return err
@@ -350,6 +398,14 @@ func (b StatefulSetBuilder) clientTLSSecretName() string {
 	return b.Spec().ClientTLSSecret
 }
 
+func (b StatefulSetBuilder) encryptionKeySecretName() string {
+	if b.Spec().EncryptionStoreKeySecret == "" {
+		return b.Cluster.EncryptionKeySecretName()
+	}
+
+	return b.Spec().EncryptionStoreKeySecret
+}
+
 func (b StatefulSetBuilder) commandArgs() []string {
 	exec := "exec " + strings.Join(b.dbArgs(), " ")
 	return []string{"/bin/bash", "-ecx", exec}
@@ -386,6 +442,18 @@ func (b StatefulSetBuilder) dbArgs() []string {
 		aa = append(aa, "--max-sql-memory $(expr $MEMORY_LIMIT_MIB / 4)MiB")
 	}
 
+	if b.Spec().EncryptionEnabled {
+		key := "plain"
+		oldKey := "plain"
+		if !b.Spec().EncryptionTypePlain {
+			key = encryptionKeyDirMountPath + "key"
+		}
+		if !b.Spec().OldEncryptionTypePlain {
+			oldKey = encryptionKeyDirMountPath + "old-key"
+		}
+		aa = append(aa, fmt.Sprintf("--enterprise-encryption=path=%s,key=%s,old-key=%s", dataDirMountPath, key, oldKey))
+	}
+
 	aa = append(aa, b.Spec().AdditionalArgs...)
 
 	needsDefaultJoin := true
@@ -412,6 +480,53 @@ func (b StatefulSetBuilder) joinStr() string {
 
 	return strings.Join(seeds, ",")
 }
+
+func addStoreKeysVolumeMountOnInitContiners(container string, spec *corev1.PodSpec) error {
+	found := false
+	initContainer := fmt.Sprintf("%s-init", container)
+	for i := range spec.InitContainers {
+		c := &spec.InitContainers[i]
+		if c.Name == initContainer {
+			found = true
+
+			c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{
+				Name:      encryptionKeyDirName,
+				MountPath: encryptionKeyDirMountPath,
+			})
+
+			break
+		}
+	}
+
+	if !found {
+		return fmt.Errorf("failed to find container %s to attach volume", container)
+	}
+
+	return nil
+}
+
+func addStoreKeysVolumeMount(container string, spec *corev1.PodSpec) error {
+	found := false
+	for i := range spec.Containers {
+		c := &spec.Containers[i]
+		if c.Name == container {
+			found = true
+
+			c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{
+				Name:      encryptionKeyDirName,
+				MountPath: encryptionKeyDirMountPath,
+			})
+			break
+		}
+	}
+
+	if !found {
+		return fmt.Errorf("failed to find container %s to attach volume", container)
+	}
+
+	return nil
+}
+
 func addCertsVolumeMountOnInitContiners(container string, spec *corev1.PodSpec) error {
 	found := false
 	initContainer := fmt.Sprintf("%s-init", container)