PMM-13132 Encryption rotation.

Makefile

@@ -38,3 +38,6 @@ TARGET ?= _bash
 env:								## Run `make TARGET` in devcontainer (`make env TARGET=help`); TARGET defaults to bash
 	COMPOSE_PROFILES=$(PROFILES) \
 	docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-server make $(TARGET)
+
+rotate-encryption: 							## Rotate encryption key
+	go run ./encryption-rotation/main.go

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/AlekSi/pointer v1.2.0
 	github.com/ClickHouse/clickhouse-go/v2 v2.23.0
 	github.com/DATA-DOG/go-sqlmock v1.5.0
+	github.com/Percona-Lab/kingpin v2.2.6+incompatible
 	github.com/alecthomas/kong v0.9.0
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9
 	github.com/aws/aws-sdk-go v1.55.3

go.sum

@@ -34,6 +34,8 @@ github.com/Percona-Lab/go-grpc-prometheus v0.0.0-20230116133345-3487748d4592 h1:
 github.com/Percona-Lab/go-grpc-prometheus v0.0.0-20230116133345-3487748d4592/go.mod h1:xCJfGpj56ERA85Mj1VfBzoeWW4lZ00xXXkvG0LJQjZU=
 github.com/Percona-Lab/kingpin v2.2.6-percona+incompatible h1:N5oM40aAatvf8bCYjv69YsVdxJLIUhY/MerUG1jRL9Y=
 github.com/Percona-Lab/kingpin v2.2.6-percona+incompatible/go.mod h1:UC6j/e2eqpHBB/vn+5214ExsoDLiEo6BfUGBhbtf+x0=
+github.com/Percona-Lab/kingpin v2.2.6+incompatible h1:i7fo0CKR6IGSxe9ErG2DMFz/shUK6vRigVfyQqOyWvs=
+github.com/Percona-Lab/kingpin v2.2.6+incompatible/go.mod h1:UC6j/e2eqpHBB/vn+5214ExsoDLiEo6BfUGBhbtf+x0=
 github.com/Percona-Lab/spec v0.20.5-percona h1:ViCJVq52QIZxpP8/Nv4/nIed+WnqUirNjPtXvHhset4=
 github.com/Percona-Lab/spec v0.20.5-percona/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=

managed/cmd/pmm-encryption-rotation/main.go

@@ -0,0 +1,90 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+// Package main is the main package for encryption keys rotation.
+package main
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/Percona-Lab/kingpin"
+	"github.com/sirupsen/logrus"
+
+	"github.com/percona/pmm/managed/models"
+	"github.com/percona/pmm/utils/logger"
+)
+
+func main() {
+	signal.Ignore(syscall.SIGINT, syscall.SIGTERM) // to prevent any interuptions during process
+
+	logger.SetupGlobalLogger()
+
+	sqlDB, err := models.OpenDB(setupParams())
+	if err != nil {
+		logrus.Error(err)
+		os.Exit(1)
+	}
+
+	statusCode := models.RotateEncryptionKey(sqlDB, "pmm-managed")
+	sqlDB.Close() //nolint:errcheck
+
+	os.Exit(statusCode)
+}
+
+func setupParams() models.SetupDBParams {
+	postgresAddrF := kingpin.Flag("postgres-addr", "PostgreSQL address").
+		Default(models.DefaultPostgreSQLAddr).
+		Envar("PMM_POSTGRES_ADDR").
+		String()
+	postgresDBNameF := kingpin.Flag("postgres-name", "PostgreSQL database name").
+		Default("pmm-managed").
+		Envar("PMM_POSTGRES_DBNAME").
+		String()
+	postgresDBUsernameF := kingpin.Flag("postgres-username", "PostgreSQL database username").
+		Default("pmm-managed").
+		Envar("PMM_POSTGRES_USERNAME").
+		String()
+	postgresSSLModeF := kingpin.Flag("postgres-ssl-mode", "PostgreSQL SSL mode").
+		Default(models.DisableSSLMode).
+		Envar("PMM_POSTGRES_SSL_MODE").
+		Enum(models.DisableSSLMode, models.RequireSSLMode, models.VerifyCaSSLMode, models.VerifyFullSSLMode)
+	postgresSSLCAPathF := kingpin.Flag("postgres-ssl-ca-path", "PostgreSQL SSL CA root certificate path").
+		Envar("PMM_POSTGRES_SSL_CA_PATH").
+		String()
+	postgresDBPasswordF := kingpin.Flag("postgres-password", "PostgreSQL database password").
+		Default("pmm-managed").
+		Envar("PMM_POSTGRES_DBPASSWORD").
+		String()
+	postgresSSLKeyPathF := kingpin.Flag("postgres-ssl-key-path", "PostgreSQL SSL key path").
+		Envar("PMM_POSTGRES_SSL_KEY_PATH").
+		String()
+	postgresSSLCertPathF := kingpin.Flag("postgres-ssl-cert-path", "PostgreSQL SSL certificate path").
+		Envar("PMM_POSTGRES_SSL_CERT_PATH").
+		String()
+
+	kingpin.Parse()
+
+	return models.SetupDBParams{
+		Address:     *postgresAddrF,
+		Name:        *postgresDBNameF,
+		Username:    *postgresDBUsernameF,
+		Password:    *postgresDBPasswordF,
+		SSLMode:     *postgresSSLModeF,
+		SSLCAPath:   *postgresSSLCAPathF,
+		SSLKeyPath:  *postgresSSLKeyPathF,
+		SSLCertPath: *postgresSSLCertPathF,
+	}
+}

managed/models/database.go

@@ -27,7 +27,6 @@ import (
 	"net"
 	"net/url"
 	"os"
-	"slices"
 	"strconv"
 	"strings"
 
@@ -61,6 +60,25 @@ const (
 	VerifyFullSSLMode string = "verify-full"
 )
 
+// DefaultAgentEncryptionColumns contains all tables and it's columns to be encrypted in PMM Server DB.
+var DefaultAgentEncryptionColumns = []encryption.Table{
+	{
+		Name:        "agents",
+		Identifiers: []string{"agent_id"},
+		Columns: []encryption.Column{
+			{Name: "username"},
+			{Name: "password"},
+			{Name: "aws_access_key"},
+			{Name: "aws_secret_key"},
+			{Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler},
+			{Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler},
+			{Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler},
+			{Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler},
+			{Name: "agent_password"},
+		},
+	},
+}
+
 // databaseSchema maps schema version from schema_migrations table (id column) to a slice of DDL queries.
 var databaseSchema = [][]string{
 	1: {
@@ -1149,79 +1167,76 @@ func SetupDB(ctx context.Context, sqlDB *sql.DB, params SetupDBParams) (*reform.
 		return nil, errCV
 	}
 
-	agentColumnsToEncrypt := []encryption.Column{
-		{Name: "username"},
-		{Name: "password"},
-		{Name: "aws_access_key"},
-		{Name: "aws_secret_key"},
-		{Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler},
-		{Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler},
-		{Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler},
-		{Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler},
-		{Name: "agent_password"},
-	}
-
-	itemsToEncrypt := []encryption.Table{
-		{
-			Name:        "agents",
-			Identifiers: []string{"agent_id"},
-			Columns:     agentColumnsToEncrypt,
-		},
-	}
-
-	if err := migrateDB(db, params, itemsToEncrypt); err != nil {
+	if err := migrateDB(db, params, DefaultAgentEncryptionColumns); err != nil {
 		return nil, err
 	}
 
 	return db, nil
 }
 
 // EncryptDB encrypts a set of columns in a specific database and table.
-func EncryptDB(tx *reform.TX, params SetupDBParams, itemsToEncrypt []encryption.Table) error {
-	if len(itemsToEncrypt) == 0 {
+func EncryptDB(tx *reform.TX, database string, itemsToEncrypt []encryption.Table) error {
+	return dbEncryption(tx, database, itemsToEncrypt, encryption.EncryptItems, true)
+}
+
+// DecryptDB decrypts a set of columns in a specific database and table.
+func DecryptDB(tx *reform.TX, database string, itemsToEncrypt []encryption.Table) error {
+	return dbEncryption(tx, database, itemsToEncrypt, encryption.DecryptItems, false)
+}
+
+func dbEncryption(tx *reform.TX, database string, items []encryption.Table,
+	encryptionHandler func(tx *reform.TX, tables []encryption.Table) error,
+	expectedState bool,
+) error {
+	if len(items) == 0 {
 		return nil
 	}
 
 	settings, err := GetSettings(tx)
 	if err != nil {
 		return err
 	}
-	alreadyEncrypted := make(map[string]bool)
+	currentColumns := make(map[string]bool)
 	for _, v := range settings.EncryptedItems {
-		alreadyEncrypted[v] = true
+		currentColumns[v] = true
 	}
 
-	notEncrypted := []encryption.Table{}
-	newlyEncrypted := []string{}
-	for _, table := range itemsToEncrypt {
+	tables := []encryption.Table{}
+	prepared := []string{}
+	for _, table := range items {
 		columns := []encryption.Column{}
 		for _, column := range table.Columns {
-			dbTableColumn := fmt.Sprintf("%s.%s.%s", params.Name, table.Name, column.Name)
-			if alreadyEncrypted[dbTableColumn] {
+			dbTableColumn := fmt.Sprintf("%s.%s.%s", database, table.Name, column.Name)
+			if currentColumns[dbTableColumn] == expectedState {
 				continue
 			}
 
 			columns = append(columns, column)
-			newlyEncrypted = append(newlyEncrypted, dbTableColumn)
+			prepared = append(prepared, dbTableColumn)
 		}
 		if len(columns) == 0 {
 			continue
 		}
 
 		table.Columns = columns
-		notEncrypted = append(notEncrypted, table)
+		tables = append(tables, table)
 	}
-
-	if len(notEncrypted) == 0 {
+	if len(tables) == 0 {
 		return nil
 	}
 
-	err = encryption.EncryptItems(tx, notEncrypted)
+	err = encryptionHandler(tx, tables)
 	if err != nil {
 		return err
 	}
+
+	encryptedItems := []string{}
+	if expectedState {
+		encryptedItems = prepared
+	}
+
 	_, err = UpdateSettings(tx, &ChangeSettingsParams{
-		EncryptedItems: slices.Concat(settings.EncryptedItems, newlyEncrypted),
+		EncryptedItems: encryptedItems,
 	})
 	if err != nil {
 		return err
@@ -1325,7 +1340,7 @@ func migrateDB(db *reform.DB, params SetupDBParams, itemsToEncrypt []encryption.
 			}
 		}
 
-		err := EncryptDB(tx, params, itemsToEncrypt)
+		err := EncryptDB(tx, params.Name, itemsToEncrypt)
 		if err != nil {
 			return err
 		}

managed/models/encryption_rotation.go

@@ -0,0 +1,128 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package models
+
+import (
+	"database/sql"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/reform.v1"
+	"gopkg.in/reform.v1/dialects/postgresql"
+
+	"github.com/percona/pmm/managed/utils/encryption"
+)
+
+// RotateEncryptionKey will stop PMM server, decrypt data, create new encryption key and encrypt them and start PMM Server again.
+func RotateEncryptionKey(sqlDB *sql.DB, dbName string) int {
+	db := reform.NewDB(sqlDB, postgresql.Dialect, nil)
+
+	err := stopPMMServer()
+	if err != nil {
+		logrus.Errorf("Failed to stop PMM Server: %+v", err)
+		return 2
+	}
+
+	err = rotateEncryptionKey(db, dbName)
+	if err != nil {
+		logrus.Errorf("Failed to rotate encryption key: %+v", err)
+		return 3
+	}
+
+	err = startPMMServer()
+	if err != nil {
+		logrus.Errorf("Failed to start PMM Server: %+v", err)
+		return 4
+	}
+
+	return 0
+}
+
+func startPMMServer() error {
+	if isPMMServerStatus("RUNNING") {
+		return nil
+	}
+
+	cmd := exec.Command("supervisorctl", "start pmm-managed")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%w: %s", err, output)
+	}
+
+	if !isPMMServerStatus("RUNNING") {
+		return errors.New("cannot start pmm-managed")
+	}
+
+	return nil
+}
+
+func stopPMMServer() error {
+	if isPMMServerStatus("STOPPED") {
+		return nil
+	}
+
+	cmd := exec.Command("supervisorctl", "stop pmm-managed")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("%w: %s", err, output)
+	}
+
+	if !isPMMServerStatus("STOPPED") {
+		return errors.New("cannot stop pmm-managed")
+	}
+
+	return nil
+}
+
+func isPMMServerStatus(status string) bool {
+	cmd := exec.Command("supervisorctl", "status pmm-managed")
+	output, _ := cmd.CombinedOutput()
+
+	return strings.Contains(string(output), strings.ToUpper(status))
+}
+
+func rotateEncryptionKey(db *reform.DB, dbName string) error {
+	return db.InTransaction(func(tx *reform.TX) error {
+		logrus.Infof("DB %s is being decrypted", dbName)
+		err := DecryptDB(tx, dbName, DefaultAgentEncryptionColumns)
+		if err != nil {
+			return err
+		}
+		logrus.Infof("DB %s is successfully decrypted", dbName)
+
+		logrus.Infoln("Rotating encryption key")
+		err = encryption.RotateEncryptionKey()
+		if err != nil {
+			return err
+		}
+		logrus.Infof("New encryption key generated")
+
+		logrus.Infof("DB %s is being encrypted", dbName)
+		err = EncryptDB(tx, dbName, DefaultAgentEncryptionColumns)
+		if err != nil {
+			if e := encryption.RestoreOldEncryptionKey(); e != nil {
+				return errors.Wrap(err, e.Error())
+			}
+			return err
+		}
+		logrus.Infof("DB %s is successfully encrypted", dbName)
+
+		return nil
+	})
+}