From 5200d62a6a72239e0749b5efba4e268616e2421b Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Sat, 15 Jul 2023 01:34:09 +0300 Subject: [PATCH 01/10] etcdctl: add etcdctl snapshot pipe command etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- client/v3/snapshot/v3_snapshot.go | 47 +++++++++++++++-------- etcdctl/README.md | 15 ++++++++ etcdctl/ctlv3/command/snapshot_command.go | 30 +++++++++++++++ 3 files changed, 75 insertions(+), 17 deletions(-) diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index b2db5f0c137..ac4aba5a738 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "os" + "strings" "time" "github.com/dustin/go-humanize" @@ -37,15 +38,14 @@ func hasChecksum(n int64) bool { return (n % 512) == sha256.Size } -// SaveWithVersion fetches snapshot from remote etcd server, saves data +// GetSnapshotWithVersion fetches snapshot from remote etcd server, saves data // to target path and returns server version. If the context "ctx" is canceled or timed out, // snapshot save stream will error out (e.g. context.Canceled, // context.DeadlineExceeded). Make sure to specify only one endpoint // in client configuration. Snapshot API must be requested to a // selected node, and saved snapshot is the point-in-time state of -// the selected node. -// Etcd SNAPSHOT SAVE writes a point-in-time snapshot of the etcd backend database to a file. diff --git a/etcdctl/ctlv3/command/snapshot_command.go b/etcdctl/ctlv3/command/snapshot_command.go index df317e23cc7..94daf2ff415 100644 --- a/etcdctl/ctlv3/command/snapshot_command.go +++ b/etcdctl/ctlv3/command/snapshot_command.go @@ -33,6 +33,7 @@ func NewSnapshotCommand() *cobra.Command { Short: "Manages etcd node snapshots", } cmd.AddCommand(NewSnapshotSaveCommand()) + cmd.AddCommand(NewSnapshotPipeCommand()) return cmd } @@ -44,6 +45,14 @@ func NewSnapshotSaveCommand() *cobra.Command { } } +func NewSnapshotPipeCommand() *cobra.Command { + return &cobra.Command{ + Use: "pipe", + Short: "Streams an etcd node backend snapshot to STDOUT", + Run: snapshotPipeCommandFunc, + } +} + func snapshotSaveCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 1 { err := fmt.Errorf("snapshot save expects one argument") @@ -73,3 +82,24 @@ func snapshotSaveCommandFunc(cmd *cobra.Command, args []string) { fmt.Printf("Server version %s\n", version) } } + +func snapshotPipeCommandFunc(cmd *cobra.Command, args []string) { + + lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) + if err != nil { + cobrautl.ExitWithError(cobrautl.ExitError, err) + } + cfg := mustClientCfgFromCmd(cmd) + + // if user does not specify "--command-timeout" flag, there will be no timeout for snapshot pipe command + ctx, cancel := context.WithCancel(context.Background()) + if isCommandTimeoutFlagSet(cmd) { + ctx, cancel = commandCtx(cmd) + } + defer cancel() + + _, err = snapshot.PipeWithVersion(ctx, lg, *cfg) + if err != nil { + cobrautl.ExitWithError(cobrautl.ExitInterrupted, err) + } +} From 9db4f02289fd0429616a355dff4a48ba601a21ec Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Tue, 19 Mar 2024 00:15:30 +0300 Subject: [PATCH 02/10] etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- client/v3/snapshot/v3_snapshot.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index ac4aba5a738..b06a7a288f1 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -38,14 +38,14 @@ func hasChecksum(n int64) bool { return (n % 512) == sha256.Size } -// GetSnapshotWithVersion fetches snapshot from remote etcd server, saves data +// WriteSnapshotWithVersion fetches snapshot from remote etcd server, saves data // to target path and returns server version. If the context "ctx" is canceled or timed out, // snapshot save stream will error out (e.g. context.Canceled, // context.DeadlineExceeded). Make sure to specify only one endpoint // in client configuration. Snapshot API must be requested to a // selected node, and saved snapshot is the point-in-time state of // the selected node. Etcd < v3.6 will return "" as version. -func GetSnapshotWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, f *os.File) (string, error) { +func WriteSnapshotWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, f *os.File) (string, error) { cfg.Logger = lg.Named("client") if len(cfg.Endpoints) != 1 { return "", fmt.Errorf("snapshot must be requested to one selected node, not multiple %v", cfg.Endpoints) @@ -107,9 +107,9 @@ func SaveWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, d defer os.RemoveAll(partPath) defer f.Close() - return GetSnapshotWithVersion(ctx, lg, cfg, f) + return WriteSnapshotWithVersion(ctx, lg, cfg, f) } func PipeWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config) (string, error) { - return GetSnapshotWithVersion(ctx, lg, cfg, os.Stdout) + return WriteSnapshotWithVersion(ctx, lg, cfg, os.Stdout) } From 3601a078790a980c705605d1be1a63c9ccb87fc2 Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Fri, 22 Mar 2024 21:52:35 +0300 Subject: [PATCH 03/10] etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- client/v3/snapshot/v3_snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index b06a7a288f1..f6cdb0b3a28 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -103,7 +103,7 @@ func SaveWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, d return "", fmt.Errorf("could not open %s (%v)", partPath, err) } lg.Info("created temporary db file", zap.String("path", partPath)) - + defer os.RemoveAll(partPath) defer f.Close() From 2ddb9f40a274f032ca87af78aca86fd9ead92fd5 Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:34:20 +0300 Subject: [PATCH 04/10] etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- CHANGELOG/CHANGELOG-3.6.md | 1 + client/v3/snapshot/v3_snapshot.go | 2 +- etcdctl/README.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md index 08924e1cd1f..defb4da8823 100644 --- a/CHANGELOG/CHANGELOG-3.6.md +++ b/CHANGELOG/CHANGELOG-3.6.md @@ -28,6 +28,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0). ### etcdctl v3 - Add command to generate [shell completion](https://github.com/etcd-io/etcd/pull/13133). +- Add command [snapshot pipe](https://github.com/etcd-io/etcd/pull/16243). - When print endpoint status, [show db size in use](https://github.com/etcd-io/etcd/pull/13639) - [Always print the raft_term in decimal](https://github.com/etcd-io/etcd/pull/13711) when displaying member list in json. - [Add one more field `storageVersion`](https://github.com/etcd-io/etcd/pull/13773) into the response of command `etcdctl endpoint status`. diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index f6cdb0b3a28..20cf01392b1 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -44,7 +44,7 @@ func hasChecksum(n int64) bool { // context.DeadlineExceeded). Make sure to specify only one endpoint // in client configuration. Snapshot API must be requested to a // selected node, and saved snapshot is the point-in-time state of -// the selected node. Etcd < v3.6 will return "" as version. +// the selected node. Nota bene: etcd < v3.6 will return "" as version. func WriteSnapshotWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, f *os.File) (string, error) { cfg.Logger = lg.Named("client") if len(cfg.Endpoints) != 1 { diff --git a/etcdctl/README.md b/etcdctl/README.md index e654eff49ee..5cc825cd85a 100644 --- a/etcdctl/README.md +++ b/etcdctl/README.md @@ -1024,7 +1024,7 @@ The backend snapshot is written to stdout. Write a snapshot to stdout: ``` -./etcdctl snapshot pipe +./etcdctl snapshot pipe > snapshot.db ``` ### SNAPSHOT SAVE \ From fb107ec8066e05d475aae3a97dfd2f74a75fdbad Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:39:35 +0300 Subject: [PATCH 05/10] etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- etcdctl/ctlv3/command/snapshot_command.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etcdctl/ctlv3/command/snapshot_command.go b/etcdctl/ctlv3/command/snapshot_command.go index 94daf2ff415..818c17f76e8 100644 --- a/etcdctl/ctlv3/command/snapshot_command.go +++ b/etcdctl/ctlv3/command/snapshot_command.go @@ -48,7 +48,7 @@ func NewSnapshotSaveCommand() *cobra.Command { func NewSnapshotPipeCommand() *cobra.Command { return &cobra.Command{ Use: "pipe", - Short: "Streams an etcd node backend snapshot to STDOUT", + Short: "Streams an etcd node backend snapshot to stdout", Run: snapshotPipeCommandFunc, } } From 447125c78433044db0cdcdcf8a3158322d9420d6 Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Fri, 29 Mar 2024 14:44:05 +0300 Subject: [PATCH 06/10] etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- client/v3/snapshot/v3_snapshot.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index 20cf01392b1..868090ec51a 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -20,7 +20,6 @@ import ( "fmt" "io" "os" - "strings" "time" "github.com/dustin/go-humanize" @@ -84,15 +83,6 @@ func WriteSnapshotWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3. zap.Duration("took", time.Since(start)), zap.String("etcd-version", resp.Version), ) - - partPath := f.Name() - dbPath := strings.TrimSuffix(partPath, ".part") - if f != os.Stdout { - if err := os.Rename(partPath, dbPath); err != nil { - return resp.Version, fmt.Errorf("could not rename %s to %s (%v)", partPath, dbPath, err) - } - } - lg.Info("finished", zap.String("path", dbPath)) return resp.Version, nil } @@ -107,7 +97,12 @@ func SaveWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, d defer os.RemoveAll(partPath) defer f.Close() - return WriteSnapshotWithVersion(ctx, lg, cfg, f) + version, err := WriteSnapshotWithVersion(ctx, lg, cfg, f) + if err := os.Rename(partPath, dbPath); err != nil { + return version, fmt.Errorf("could not rename %s to %s (%v)", partPath, dbPath, err) + } + lg.Info("saved", zap.String("path", dbPath)) + return version, err } func PipeWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config) (string, error) { From 3b0689e89137f085ba71444c778d2d7cbce86ea6 Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Fri, 29 Mar 2024 14:55:41 +0300 Subject: [PATCH 07/10] etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- client/v3/snapshot/v3_snapshot.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index 868090ec51a..10561ea9548 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -97,7 +97,8 @@ func SaveWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, d defer os.RemoveAll(partPath) defer f.Close() - version, err := WriteSnapshotWithVersion(ctx, lg, cfg, f) + var version string + version, err = WriteSnapshotWithVersion(ctx, lg, cfg, f) if err := os.Rename(partPath, dbPath); err != nil { return version, fmt.Errorf("could not rename %s to %s (%v)", partPath, dbPath, err) } From a5389fc8443bd7f0837f27727eb9dd9be97e2f95 Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Fri, 29 Mar 2024 15:05:03 +0300 Subject: [PATCH 08/10] etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- client/v3/snapshot/v3_snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index 10561ea9548..e239b1b62b0 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -99,7 +99,7 @@ func SaveWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, d var version string version, err = WriteSnapshotWithVersion(ctx, lg, cfg, f) - if err := os.Rename(partPath, dbPath); err != nil { + if err = os.Rename(partPath, dbPath); err != nil { return version, fmt.Errorf("could not rename %s to %s (%v)", partPath, dbPath, err) } lg.Info("saved", zap.String("path", dbPath)) From 23f758fb59160ca0c96b65fe42960051c2cdbab6 Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Fri, 29 Mar 2024 15:19:44 +0300 Subject: [PATCH 09/10] etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- client/v3/snapshot/v3_snapshot.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index e239b1b62b0..11b14a3c186 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -97,13 +97,12 @@ func SaveWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, d defer os.RemoveAll(partPath) defer f.Close() - var version string - version, err = WriteSnapshotWithVersion(ctx, lg, cfg, f) + version, _ := WriteSnapshotWithVersion(ctx, lg, cfg, f) if err = os.Rename(partPath, dbPath); err != nil { return version, fmt.Errorf("could not rename %s to %s (%v)", partPath, dbPath, err) } lg.Info("saved", zap.String("path", dbPath)) - return version, err + return version, nil } func PipeWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config) (string, error) { From d2ffdeda9c9e9bf917259c6da292d1c34ca70e4e Mon Sep 17 00:00:00 2001 From: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> Date: Sat, 30 Mar 2024 12:23:06 +0300 Subject: [PATCH 10/10] etcdctl: add etcdctl snapshot pipe command To improve the security of etcdctl. Added the ability to write snapshots to stdout without writing data to disk. Signed-off-by: Ais8Ooz8 <47941654+Ais8Ooz8@users.noreply.github.com> --- client/v3/snapshot/v3_snapshot.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/v3/snapshot/v3_snapshot.go b/client/v3/snapshot/v3_snapshot.go index 11b14a3c186..ac1d1046630 100644 --- a/client/v3/snapshot/v3_snapshot.go +++ b/client/v3/snapshot/v3_snapshot.go @@ -97,7 +97,11 @@ func SaveWithVersion(ctx context.Context, lg *zap.Logger, cfg clientv3.Config, d defer os.RemoveAll(partPath) defer f.Close() - version, _ := WriteSnapshotWithVersion(ctx, lg, cfg, f) + var version string + version, err = WriteSnapshotWithVersion(ctx, lg, cfg, f) + if err != nil { + return version, err + } if err = os.Rename(partPath, dbPath); err != nil { return version, fmt.Errorf("could not rename %s to %s (%v)", partPath, dbPath, err) }