diff --git a/client/pkg/transport/listener.go b/client/pkg/transport/listener.go index 5e0e13e25a73..8d5c4669437a 100644 --- a/client/pkg/transport/listener.go +++ b/client/pkg/transport/listener.go @@ -194,6 +194,9 @@ type TLSInfo struct { // EmptyCN indicates that the cert must have empty CN. // If true, ClientConfig() will return an error for a cert with non empty CN. EmptyCN bool + + // EnableRootCAReload indicates whether to reload root CA dynamically. + EnableRootCAReload bool } func (info TLSInfo) String() string { @@ -435,10 +438,21 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) { } } - // this only reloads certs when there's a client request - // TODO: support server-side refresh (e.g. inotify, SIGHUP), caching - cfg.GetCertificate = func(clientHello *tls.ClientHelloInfo) (cert *tls.Certificate, err error) { - cert, err = tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc) + if info.EnableRootCAReload { + cfg.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + cfg, err := info.ServerConfig() + if err != nil { + if info.Logger != nil { + info.Logger.Warn( + "failed to create tls config", + zap.Error(err), + ) + } + } + return cfg, err + } + + cert, err := tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc) if os.IsNotExist(err) { if info.Logger != nil { info.Logger.Warn( @@ -458,7 +472,33 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) { ) } } - return cert, err + cfg.Certificates = []tls.Certificate{*cert} + } else { + // this only reloads certs when there's a client request + // TODO: support server-side refresh (e.g. inotify, SIGHUP), caching + cfg.GetCertificate = func(clientHello *tls.ClientHelloInfo) (cert *tls.Certificate, err error) { + cert, err = tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc) + if os.IsNotExist(err) { + if info.Logger != nil { + info.Logger.Warn( + "failed to find peer cert files", + zap.String("cert-file", info.CertFile), + zap.String("key-file", info.KeyFile), + zap.Error(err), + ) + } + } else if err != nil { + if info.Logger != nil { + info.Logger.Warn( + "failed to create peer certificate", + zap.String("cert-file", info.CertFile), + zap.String("key-file", info.KeyFile), + zap.Error(err), + ) + } + } + return cert, err + } } cfg.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (cert *tls.Certificate, err error) { certfile, keyfile := info.CertFile, info.KeyFile @@ -557,6 +597,32 @@ func (info TLSInfo) ClientConfig() (*tls.Config, error) { if info.selfCert { cfg.InsecureSkipVerify = true + } else if info.EnableRootCAReload { + if len(cs) == 0 { + return nil, fmt.Errorf("cannot enable root CA reloading without a trusted CA file") + } + + // Set InsecureSkipVerify to skip the default validation we are replacing. + // This will not disable VerifyConnection. + cfg.InsecureSkipVerify = true + + cfg.VerifyConnection = func(connState tls.ConnectionState) error { + // dynamically load CA from file + rootCAs, err := tlsutil.NewCertPool(cs) + if err != nil { + return err + } + opts := x509.VerifyOptions{ + DNSName: connState.ServerName, + Intermediates: x509.NewCertPool(), + Roots: rootCAs, + } + for _, cert := range connState.PeerCertificates[1:] { + opts.Intermediates.AddCert(cert) + } + _, err = connState.PeerCertificates[0].Verify(opts) + return err + } } if info.EmptyCN { diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index fa5d6d161a12..0251eab0f098 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -205,11 +205,13 @@ func newConfig() *config { fs.StringVar(&cfg.ec.ClientTLSInfo.CRLFile, "client-crl-file", "", "Path to the client certificate revocation list file.") fs.StringVar(&cfg.ec.ClientTLSInfo.AllowedHostname, "client-cert-allowed-hostname", "", "Allowed TLS hostname for client cert authentication.") fs.StringVar(&cfg.ec.ClientTLSInfo.TrustedCAFile, "trusted-ca-file", "", "Path to the client server TLS trusted CA cert file.") + fs.BoolVar(&cfg.ec.ClientTLSInfo.EnableRootCAReload, "client-root-ca-reload", false, "Enable client server TLS root CA dynamic reload to support root CA rotation") fs.BoolVar(&cfg.ec.ClientAutoTLS, "auto-tls", false, "Client TLS using generated certificates") fs.StringVar(&cfg.ec.PeerTLSInfo.CertFile, "peer-cert-file", "", "Path to the peer server TLS cert file.") fs.StringVar(&cfg.ec.PeerTLSInfo.KeyFile, "peer-key-file", "", "Path to the peer server TLS key file.") fs.StringVar(&cfg.ec.PeerTLSInfo.ClientCertFile, "peer-client-cert-file", "", "Path to an explicit peer client TLS cert file otherwise peer cert file will be used when client auth is required.") fs.StringVar(&cfg.ec.PeerTLSInfo.ClientKeyFile, "peer-client-key-file", "", "Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required.") + fs.BoolVar(&cfg.ec.PeerTLSInfo.EnableRootCAReload, "peer-root-ca-reload", false, "Enable peer client TLS root CA dynamic reload to support root CA rotation") fs.BoolVar(&cfg.ec.PeerTLSInfo.ClientCertAuth, "peer-client-cert-auth", false, "Enable peer client cert authentication.") fs.StringVar(&cfg.ec.PeerTLSInfo.TrustedCAFile, "peer-trusted-ca-file", "", "Path to the peer server TLS trusted CA file.") fs.BoolVar(&cfg.ec.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates") diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index 1f139680d3d7..e97e870d91d1 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -187,6 +187,8 @@ Security: Allowed TLS hostname for client cert authentication. --trusted-ca-file '' Path to the client server TLS trusted CA cert file. + --client-root-ca-reload 'false' + Enable client server TLS root CA dynamic reload to support root CA rotation. --auto-tls 'false' Client TLS using generated certificates. --peer-cert-file '' @@ -201,6 +203,8 @@ Security: Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required. --peer-trusted-ca-file '' Path to the peer server TLS trusted CA file. + --peer-root-ca-reload 'false' + Enable peer client TLS root CA dynamic reload to support root CA rotation. --peer-cert-allowed-cn '' Required CN for client certs connecting to the peer endpoint. --peer-cert-allowed-hostname '' diff --git a/tests/common/e2e_test.go b/tests/common/e2e_test.go index 11c4f94a335b..6413f712f96a 100644 --- a/tests/common/e2e_test.go +++ b/tests/common/e2e_test.go @@ -18,6 +18,7 @@ package common import ( "go.etcd.io/etcd/client/pkg/v3/fileutil" + "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" @@ -81,3 +82,7 @@ func WithAuth(userName, password string) config.ClientOption { func WithEndpoints(endpoints []string) config.ClientOption { return e2e.WithEndpoints(endpoints) } + +func WithTLSInfo(tlsInfo *transport.TLSInfo) config.ClientOption { + return e2e.WithTLSInfo(tlsInfo) +} diff --git a/tests/common/integration_test.go b/tests/common/integration_test.go index c4cabeeb1f98..b17bb4062181 100644 --- a/tests/common/integration_test.go +++ b/tests/common/integration_test.go @@ -17,6 +17,7 @@ package common import ( + "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/integration" @@ -59,3 +60,7 @@ func WithAuth(userName, password string) config.ClientOption { func WithEndpoints(endpoints []string) config.ClientOption { return integration.WithEndpoints(endpoints) } + +func WithTLSInfo(tlsInfo *transport.TLSInfo) config.ClientOption { + return integration.WithTLSInfo(tlsInfo) +} diff --git a/tests/common/root_ca_rotation_test.go b/tests/common/root_ca_rotation_test.go new file mode 100644 index 000000000000..a70ae43c2e94 --- /dev/null +++ b/tests/common/root_ca_rotation_test.go @@ -0,0 +1,197 @@ +package common + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "net" + "os" + "path" + "testing" + "time" + + "go.etcd.io/etcd/client/pkg/v3/transport" + "go.etcd.io/etcd/tests/v3/framework/config" + "go.etcd.io/etcd/tests/v3/framework/testutils" +) + +func newSerialNumber(t *testing.T) *big.Int { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + t.Fail() + } + return serialNumber +} + +func createRootCertificateAuthority(rootCaPath string, oldPem []byte, t *testing.T) (*x509.Certificate, []byte, *ecdsa.PrivateKey) { + serialNumber := newSerialNumber(t) + priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader) + if err != nil { + t.Fatal(err) + } + + tmpl := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{Organization: []string{"etcd"}}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(365 * (24 * time.Hour)), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageContentCommitment, + BasicConstraintsValid: true, + IsCA: true, + } + + caBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv) + if err != nil { + t.Fatal(err) + } + + ca, err := x509.ParseCertificate(caBytes) + if err != nil { + t.Fatal(err) + } + caBlocks := [][]byte{caBytes} + if len(oldPem) > 0 { + caBlocks = append(caBlocks, oldPem) + } + marshalCerts(caBlocks, rootCaPath, t) + return ca, caBytes, priv +} + +func generateCerts(privKey *ecdsa.PrivateKey, rootCA *x509.Certificate, dir, suffix string, t *testing.T) { + priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader) + if err != nil { + t.Fatal(err) + } + serialNumber := newSerialNumber(t) + tmpl := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{Organization: []string{"etcd"}}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(365 * (24 * time.Hour)), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageContentCommitment, + BasicConstraintsValid: true, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + } + caBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, rootCA, &priv.PublicKey, privKey) + if err != nil { + t.Fatal(err) + } + marshalCerts([][]byte{caBytes}, path.Join(dir, fmt.Sprintf("cert%s.pem", suffix)), t) + marshalKeys(priv, path.Join(dir, fmt.Sprintf("key%s.pem", suffix)), t) +} + +func marshalCerts(caBytes [][]byte, certPath string, t *testing.T) { + var caPerm bytes.Buffer + for _, caBlock := range caBytes { + err := pem.Encode(&caPerm, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBlock, + }) + if err != nil { + t.Fatal(err) + } + } + ioutil.WriteFile(certPath, caPerm.Bytes(), 0644) +} + +func marshalKeys(privKey *ecdsa.PrivateKey, keyPath string, t *testing.T) { + privBytes, err := x509.MarshalECPrivateKey(privKey) + if err != nil { + t.Fatal(err) + } + + var keyPerm bytes.Buffer + err = pem.Encode(&keyPerm, &pem.Block{ + Type: "EC PRIVATE KEY", + Bytes: privBytes, + }) + if err != nil { + t.Fatal(err) + } + ioutil.WriteFile(keyPath, keyPerm.Bytes(), 0644) +} + +func TestRootCARotation(t *testing.T) { + testRunner.BeforeTest(t) + + t.Run("server CA rotation", func(t *testing.T) { + tmpdir, err := ioutil.TempDir(os.TempDir(), "tlsdir-integration-reload") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + rootCAPath := path.Join(tmpdir, "ca-cert.pem") + rootCA, caBytes, privKey := createRootCertificateAuthority(rootCAPath, []byte{}, t) + generateCerts(privKey, rootCA, tmpdir, "_itest_old", t) + + tlsInfo := &transport.TLSInfo{ + TrustedCAFile: rootCAPath, + CertFile: path.Join(tmpdir, "cert_itest_old.pem"), + KeyFile: path.Join(tmpdir, "key_itest_old.pem"), + ClientCertFile: path.Join(tmpdir, "cert_itest_old.pem"), + ClientKeyFile: path.Join(tmpdir, "key_itest_old.pem"), + EnableRootCAReload: true, + } + clusConfig := config.ClusterConfig{ClusterSize: 1, ClientTLS: config.ManualTLS, ClientTLSInfo: tlsInfo} + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(clusConfig)) + defer clus.Close() + + cc := testutils.MustClient(clus.Client(WithTLSInfo(tlsInfo))) + testutils.ExecuteUntil(ctx, t, func() { + key := "foo" + _, err := cc.Get(ctx, key, config.GetOptions{}) + if err != nil { + t.Fatalf("Unexpeted result, err: %s", err) + } + }) + + // regenerate rootCA and sign new certs + rootCA, _, privKey = createRootCertificateAuthority(rootCAPath, caBytes, t) + generateCerts(privKey, rootCA, tmpdir, "_itest_new", t) + + // old rootCA certs + cc = testutils.MustClient(clus.Client(WithTLSInfo(tlsInfo))) + testutils.ExecuteUntil(ctx, t, func() { + key := "foo" + _, err := cc.Get(ctx, key, config.GetOptions{}) + if err != nil { + t.Fatalf("Unexpeted result, err: %s", err) + } + }) + + // new rootCA certs + newClientTlsinfo := &transport.TLSInfo{ + TrustedCAFile: rootCAPath, + CertFile: path.Join(tmpdir, "cert_itest_new.pem"), + KeyFile: path.Join(tmpdir, "key_itest_new.pem"), + ClientCertFile: path.Join(tmpdir, "cert_itest_new.pem"), + ClientKeyFile: path.Join(tmpdir, "key_itest_new.pem"), + } + + cc = testutils.MustClient(clus.Client(WithTLSInfo(newClientTlsinfo))) + testutils.ExecuteUntil(ctx, t, func() { + key := "foo" + _, err := cc.Get(ctx, key, config.GetOptions{}) + if err != nil { + t.Fatalf("Unexpeted result, err: %s", err) + } + }) + }) + + // TODO(hongbin): added test for peer CA rotation +} diff --git a/tests/common/unit_test.go b/tests/common/unit_test.go index 4b172e7a3cb4..208b53361eac 100644 --- a/tests/common/unit_test.go +++ b/tests/common/unit_test.go @@ -17,6 +17,7 @@ package common import ( + "go.etcd.io/etcd/client/pkg/v3/transport" "go.etcd.io/etcd/tests/v3/framework" "go.etcd.io/etcd/tests/v3/framework/config" ) @@ -40,3 +41,7 @@ func WithAuth(userName, password string) config.ClientOption { func WithEndpoints(endpoints []string) config.ClientOption { return func(any) {} } + +func WithTLSInfo(tlsInfo *transport.TLSInfo) config.ClientOption { + return func(any) {} +} diff --git a/tests/e2e/utils.go b/tests/e2e/utils.go index b49239b73d73..ab46783c65f1 100644 --- a/tests/e2e/utils.go +++ b/tests/e2e/utils.go @@ -71,6 +71,8 @@ func tlsInfo(t testing.TB, cfg e2e.ClientConfig) (*transport.TLSInfo, error) { return nil, fmt.Errorf("failed to generate cert: %s", err) } return &tls, nil + } else if cfg.TLSInfo != nil { + return cfg.TLSInfo, nil } else { return &integration.TestTLSInfo, nil } diff --git a/tests/framework/config/cluster.go b/tests/framework/config/cluster.go index 0e6ec561afb9..89092704671e 100644 --- a/tests/framework/config/cluster.go +++ b/tests/framework/config/cluster.go @@ -16,6 +16,8 @@ package config import ( "time" + + "go.etcd.io/etcd/client/pkg/v3/transport" ) type TLSConfig string @@ -32,6 +34,8 @@ type ClusterConfig struct { ClusterSize int PeerTLS TLSConfig ClientTLS TLSConfig + PeerTLSInfo *transport.TLSInfo + ClientTLSInfo *transport.TLSInfo QuotaBackendBytes int64 StrictReconfigCheck bool AuthToken string @@ -77,6 +81,14 @@ func WithClientTLS(tls TLSConfig) ClusterOption { return func(c *ClusterConfig) { c.ClientTLS = tls } } +func WithPeerTLSInfo(tlsInfo *transport.TLSInfo) ClusterOption { + return func(c *ClusterConfig) { c.PeerTLSInfo = tlsInfo } +} + +func WithClientTLSInfo(tlsInfo *transport.TLSInfo) ClusterOption { + return func(c *ClusterConfig) { c.ClientTLSInfo = tlsInfo } +} + func WithQuotaBackendBytes(bytes int64) ClusterOption { return func(c *ClusterConfig) { c.QuotaBackendBytes = bytes } } diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index 17c3c37a0d3e..aa6720ca9dba 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -30,6 +30,7 @@ import ( "go.uber.org/zap/zaptest" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/proxy" "go.etcd.io/etcd/server/v3/etcdserver" @@ -51,6 +52,7 @@ type ClientConfig struct { CertAuthority bool AutoTLS bool RevokeCerts bool + TLSInfo *transport.TLSInfo } // allow alphanumerics, underscores and dashes @@ -159,6 +161,7 @@ type EtcdProcessClusterConfig struct { ClientHttpSeparate bool IsPeerTLS bool IsPeerAutoTLS bool + PeerTLSInfo *transport.TLSInfo CN bool CipherSuites []string @@ -684,6 +687,20 @@ func (cfg *EtcdProcessClusterConfig) TlsArgs() (args []string) { if cfg.Client.ConnectionType != ClientNonTLS { if cfg.Client.AutoTLS { args = append(args, "--auto-tls") + } else if cfg.Client.TLSInfo != nil { + tlsClientArgs := []string{ + "--cert-file", cfg.Client.TLSInfo.CertFile, + "--key-file", cfg.Client.TLSInfo.KeyFile, + "--trusted-ca-file", cfg.Client.TLSInfo.TrustedCAFile, + } + args = append(args, tlsClientArgs...) + + if cfg.Client.TLSInfo.ClientCertAuth { + args = append(args, "--client-cert-auth") + } + if cfg.Client.TLSInfo.EnableRootCAReload { + args = append(args, "--client-root-ca-reload") + } } else { tlsClientArgs := []string{ "--cert-file", CertPath, @@ -701,6 +718,17 @@ func (cfg *EtcdProcessClusterConfig) TlsArgs() (args []string) { if cfg.IsPeerTLS { if cfg.IsPeerAutoTLS { args = append(args, "--peer-auto-tls") + } else if cfg.PeerTLSInfo != nil { + tlsPeerArgs := []string{ + "--peer-cert-file", cfg.PeerTLSInfo.CertFile, + "--peer-key-file", cfg.PeerTLSInfo.KeyFile, + "--peer-trusted-ca-file", cfg.PeerTLSInfo.TrustedCAFile, + } + args = append(args, tlsPeerArgs...) + + if cfg.PeerTLSInfo.EnableRootCAReload { + args = append(args, "--peer-root-ca-reload") + } } else { tlsPeerArgs := []string{ "--peer-cert-file", CertPath, diff --git a/tests/framework/e2e/curl.go b/tests/framework/e2e/curl.go index 3639bc3a9700..dc95ac2e5e82 100644 --- a/tests/framework/e2e/curl.go +++ b/tests/framework/e2e/curl.go @@ -69,7 +69,15 @@ func CURLPrefixArgs(clientURL string, cfg ClientConfig, CN bool, method string, if cfg.ConnectionType != ClientTLSAndNonTLS { panic("should not use cURLPrefixArgsUseTLS when serving only TLS or non-TLS") } - cmdArgs = append(cmdArgs, "--cacert", CaPath, "--cert", CertPath, "--key", PrivateKeyPath) + caPath := CaPath + certPath := CertPath + keyPath := PrivateKeyPath + if cfg.TLSInfo != nil { + caPath = cfg.TLSInfo.TrustedCAFile + certPath = cfg.TLSInfo.ClientCertFile + keyPath = cfg.TLSInfo.ClientKeyFile + } + cmdArgs = append(cmdArgs, "--cacert", caPath, "--cert", certPath, "--key", keyPath) clientURL = ToTLS(clientURL) } else if cfg.ConnectionType == ClientTLS { if CN { diff --git a/tests/framework/e2e/e2e.go b/tests/framework/e2e/e2e.go index f78df57926ea..0885f0061696 100644 --- a/tests/framework/e2e/e2e.go +++ b/tests/framework/e2e/e2e.go @@ -67,6 +67,7 @@ func (e e2eRunner) NewCluster(ctx context.Context, t testing.TB, opts ...config. case config.ManualTLS: e2eConfig.Client.AutoTLS = false e2eConfig.Client.ConnectionType = ClientTLS + e2eConfig.Client.TLSInfo = cfg.ClientTLSInfo default: t.Fatalf("ClientTLS config %q not supported", cfg.ClientTLS) } @@ -80,6 +81,7 @@ func (e e2eRunner) NewCluster(ctx context.Context, t testing.TB, opts ...config. case config.ManualTLS: e2eConfig.IsPeerTLS = true e2eConfig.IsPeerAutoTLS = false + e2eConfig.PeerTLSInfo = cfg.PeerTLSInfo default: t.Fatalf("PeerTLS config %q not supported", cfg.PeerTLS) } diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index 62f8a48d9d51..40e94a14ae1b 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -27,6 +27,7 @@ import ( "go.etcd.io/etcd/api/v3/authpb" "go.etcd.io/etcd/api/v3/etcdserverpb" + "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/pkg/v3/expect" "go.etcd.io/etcd/tests/v3/framework/config" @@ -80,6 +81,13 @@ func WithEndpoints(endpoints []string) config.ClientOption { } } +func WithTLSInfo(tlsInfo *transport.TLSInfo) config.ClientOption { + return func(c any) { + ctl := c.(*EtcdctlV3) + ctl.cfg.TLSInfo = tlsInfo + } +} + func (ctl *EtcdctlV3) DowngradeEnable(ctx context.Context, version string) error { _, err := SpawnWithExpectLines(ctx, ctl.cmdArgs("downgrade", "enable", version), nil, expect.ExpectedResponse{Value: "Downgrade enable success"}) return err @@ -323,6 +331,10 @@ func (ctl *EtcdctlV3) flags() map[string]string { if ctl.cfg.AutoTLS { fmap["insecure-transport"] = "false" fmap["insecure-skip-tls-verify"] = "true" + } else if ctl.cfg.TLSInfo != nil { + fmap["cacert"] = ctl.cfg.TLSInfo.TrustedCAFile + fmap["cert"] = ctl.cfg.TLSInfo.ClientCertFile + fmap["key"] = ctl.cfg.TLSInfo.ClientKeyFile } else if ctl.cfg.RevokeCerts { fmap["cacert"] = CaPath fmap["cert"] = RevokedCertPath diff --git a/tests/framework/integration/cluster.go b/tests/framework/integration/cluster.go index 4dcfdfda47f1..c62be1001f9e 100644 --- a/tests/framework/integration/cluster.go +++ b/tests/framework/integration/cluster.go @@ -1487,6 +1487,17 @@ func WithEndpoints(endpoints []string) framecfg.ClientOption { } } +func WithTLSInfo(tlsInfo *transport.TLSInfo) framecfg.ClientOption { + return func(c any) { + cfg := c.(*clientv3.Config) + tls, err := tlsInfo.ClientConfig() + if err != nil { + panic(err) + } + cfg.TLS = tls + } +} + func (c *Cluster) newClientCfg() (*clientv3.Config, error) { cfg := &clientv3.Config{ Endpoints: c.Endpoints(), diff --git a/tests/framework/integration/integration.go b/tests/framework/integration/integration.go index 8d5f786e7177..6bd0bbfecb20 100644 --- a/tests/framework/integration/integration.go +++ b/tests/framework/integration/integration.go @@ -57,13 +57,21 @@ func (e integrationRunner) NewCluster(ctx context.Context, t testing.TB, opts .. AuthToken: cfg.AuthToken, SnapshotCount: uint64(cfg.SnapshotCount), } - integrationCfg.ClientTLS, err = tlsInfo(t, cfg.ClientTLS) - if err != nil { - t.Fatalf("ClientTLS: %s", err) + if cfg.ClientTLSInfo != nil { + integrationCfg.ClientTLS = cfg.ClientTLSInfo + } else { + integrationCfg.ClientTLS, err = tlsInfo(t, cfg.ClientTLS) + if err != nil { + t.Fatalf("ClientTLS: %s", err) + } } - integrationCfg.PeerTLS, err = tlsInfo(t, cfg.PeerTLS) - if err != nil { - t.Fatalf("PeerTLS: %s", err) + if cfg.PeerTLSInfo != nil { + integrationCfg.PeerTLS = cfg.PeerTLSInfo + } else { + integrationCfg.PeerTLS, err = tlsInfo(t, cfg.PeerTLS) + if err != nil { + t.Fatalf("PeerTLS: %s", err) + } } return &integrationCluster{ Cluster: NewCluster(t, &integrationCfg),