WIP: Enable reloading CA without a restart

* Add two options to server: "client-root-ca-reload" and "peer-root-ca-reload".
  By default, these options are set to false. Whenever the options are enabled,
  the server will dynamically load CA keys & certs.
* Provide implementation for "GetConfigForClient". This will allow server to
  load CA files on each TLS handshake.
* Provide implementation for "VerifyConnection". This will clients (for peer connection)
  to load CA files per request.

Note: this patch implements CA reloading without performance optimization.
Optimization could be done in the future. Potential optimization is
to avoid loading CA on each request. We could implement a background
routine to periodically loading CA files instead.

client/pkg/transport/listener.go

@@ -194,6 +194,9 @@ type TLSInfo struct {
 	// EmptyCN indicates that the cert must have empty CN.
 	// If true, ClientConfig() will return an error for a cert with non empty CN.
 	EmptyCN bool
+
+	// EnableRootCAReload indicates whether to reload root CA dynamically.
+	EnableRootCAReload bool
 }
 
 func (info TLSInfo) String() string {
@@ -435,10 +438,21 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) {
 		}
 	}
 
-	// this only reloads certs when there's a client request
-	// TODO: support server-side refresh (e.g. inotify, SIGHUP), caching
-	cfg.GetCertificate = func(clientHello *tls.ClientHelloInfo) (cert *tls.Certificate, err error) {
-		cert, err = tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc)
+	if info.EnableRootCAReload {
+		cfg.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) {
+			cfg, err := info.ServerConfig()
+			if err != nil {
+				if info.Logger != nil {
+					info.Logger.Warn(
+						"failed to create tls config",
+						zap.Error(err),
+					)
+				}
+			}
+			return cfg, err
+		}
+
+		cert, err := tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc)
 		if os.IsNotExist(err) {
 			if info.Logger != nil {
 				info.Logger.Warn(
@@ -458,7 +472,33 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) {
 				)
 			}
 		}
-		return cert, err
+		cfg.Certificates = []tls.Certificate{*cert}
+	} else {
+		// this only reloads certs when there's a client request
+		// TODO: support server-side refresh (e.g. inotify, SIGHUP), caching
+		cfg.GetCertificate = func(clientHello *tls.ClientHelloInfo) (cert *tls.Certificate, err error) {
+			cert, err = tlsutil.NewCert(info.CertFile, info.KeyFile, info.parseFunc)
+			if os.IsNotExist(err) {
+				if info.Logger != nil {
+					info.Logger.Warn(
+						"failed to find peer cert files",
+						zap.String("cert-file", info.CertFile),
+						zap.String("key-file", info.KeyFile),
+						zap.Error(err),
+					)
+				}
+			} else if err != nil {
+				if info.Logger != nil {
+					info.Logger.Warn(
+						"failed to create peer certificate",
+						zap.String("cert-file", info.CertFile),
+						zap.String("key-file", info.KeyFile),
+						zap.Error(err),
+					)
+				}
+			}
+			return cert, err
+		}
 	}
 	cfg.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (cert *tls.Certificate, err error) {
 		certfile, keyfile := info.CertFile, info.KeyFile
@@ -557,6 +597,32 @@ func (info TLSInfo) ClientConfig() (*tls.Config, error) {
 
 	if info.selfCert {
 		cfg.InsecureSkipVerify = true
+	} else if info.EnableRootCAReload {
+		if len(cs) == 0 {
+			return nil, fmt.Errorf("cannot enable root CA reloading without a trusted CA file")
+		}
+
+		// Set InsecureSkipVerify to skip the default validation we are replacing.
+		// This will not disable VerifyConnection.
+		cfg.InsecureSkipVerify = true
+
+		cfg.VerifyConnection = func(connState tls.ConnectionState) error {
+			// dynamically load CA from file
+			rootCAs, err := tlsutil.NewCertPool(cs)
+			if err != nil {
+				return err
+			}
+			opts := x509.VerifyOptions{
+				DNSName:       connState.ServerName,
+				Intermediates: x509.NewCertPool(),
+				Roots:         rootCAs,
+			}
+			for _, cert := range connState.PeerCertificates[1:] {
+				opts.Intermediates.AddCert(cert)
+			}
+			_, err = connState.PeerCertificates[0].Verify(opts)
+			return err
+		}
 	}
 
 	if info.EmptyCN {

server/etcdmain/config.go

@@ -205,11 +205,13 @@ func newConfig() *config {
 	fs.StringVar(&cfg.ec.ClientTLSInfo.CRLFile, "client-crl-file", "", "Path to the client certificate revocation list file.")
 	fs.StringVar(&cfg.ec.ClientTLSInfo.AllowedHostname, "client-cert-allowed-hostname", "", "Allowed TLS hostname for client cert authentication.")
 	fs.StringVar(&cfg.ec.ClientTLSInfo.TrustedCAFile, "trusted-ca-file", "", "Path to the client server TLS trusted CA cert file.")
+	fs.BoolVar(&cfg.ec.ClientTLSInfo.EnableRootCAReload, "client-root-ca-reload", false, "Enable client server TLS root CA dynamic reload to support root CA rotation")
 	fs.BoolVar(&cfg.ec.ClientAutoTLS, "auto-tls", false, "Client TLS using generated certificates")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.CertFile, "peer-cert-file", "", "Path to the peer server TLS cert file.")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.KeyFile, "peer-key-file", "", "Path to the peer server TLS key file.")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.ClientCertFile, "peer-client-cert-file", "", "Path to an explicit peer client TLS cert file otherwise peer cert file will be used when client auth is required.")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.ClientKeyFile, "peer-client-key-file", "", "Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required.")
+	fs.BoolVar(&cfg.ec.PeerTLSInfo.EnableRootCAReload, "peer-root-ca-reload", false, "Enable peer client TLS root CA dynamic reload to support root CA rotation")
 	fs.BoolVar(&cfg.ec.PeerTLSInfo.ClientCertAuth, "peer-client-cert-auth", false, "Enable peer client cert authentication.")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.TrustedCAFile, "peer-trusted-ca-file", "", "Path to the peer server TLS trusted CA file.")
 	fs.BoolVar(&cfg.ec.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates")