From c228cf8bd85292f2ce8dfd2ac8fc44473e82fae8 Mon Sep 17 00:00:00 2001 From: Konrad Wojas Date: Wed, 1 Mar 2023 12:33:13 +0800 Subject: [PATCH 1/5] Add subcommands for managing snapshots Added the following subcommands: - snapshots list: list remote snapshots - snapshots remove: remove a snapshot - snapshots dump: dump a snapshot for debugging - snapshots get: download a snapshot - snapshots put: upload a snapshot --- cmd/lightningstream/commands/snapshots.go | 277 ++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 cmd/lightningstream/commands/snapshots.go diff --git a/cmd/lightningstream/commands/snapshots.go b/cmd/lightningstream/commands/snapshots.go new file mode 100644 index 0000000..eb55b75 --- /dev/null +++ b/cmd/lightningstream/commands/snapshots.go @@ -0,0 +1,277 @@ +package commands + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/PowerDNS/simpleblob" + "github.com/gogo/protobuf/proto" + "github.com/samber/lo" + "github.com/spf13/cobra" + "golang.org/x/exp/slices" + "powerdns.com/platform/lightningstream/lmdbenv/dbiflags" + "powerdns.com/platform/lightningstream/lmdbenv/header" + "powerdns.com/platform/lightningstream/snapshot" + "powerdns.com/platform/lightningstream/utils" +) + +func init() { + rootCmd.AddCommand(snapshotsCmd) + + snapshotsCmd.AddCommand(snapshotsListCmd) + snapshotsListCmd.Flags().StringP("prefix", "p", "", "Prefix filter") + snapshotsListCmd.Flags().BoolP("long", "l", false, "Add extra information, like size") + snapshotsListCmd.Flags().BoolP("time", "t", false, "Sort by snapshot time") + + snapshotsCmd.AddCommand(snapshotsRemoveCmd) + + snapshotsCmd.AddCommand(snapshotsDumpCmd) + snapshotsDumpCmd.Flags().StringP("format", "f", "debug", + "Output format, one of: 'debug' (default), 'text'") + snapshotsDumpCmd.Flags().StringP("dbi", "d", "", "Only output DBI with this exact name") + + snapshotsCmd.AddCommand(snapshotsGetCmd) + snapshotsGetCmd.Flags().StringP("output", "o", "", + "Output filename, if not the same as the remote name") + + snapshotsCmd.AddCommand(snapshotsPutCmd) + snapshotsPutCmd.Flags().StringP("name", "n", "", + "Name to store the snapshot as, if different from the local name") +} + +var snapshotsCmd = &cobra.Command{ + Use: "snapshots", + Short: "Remote snapshot operations (list, dump, remove, etc)", + Run: func(cmd *cobra.Command, args []string) { + _ = cmd.Help() + }, +} + +var snapshotsListCmd = &cobra.Command{ + Use: "list", + Short: "List snapshots", + Args: cobra.NoArgs, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithTimeout(rootCtx, time.Minute) + defer cancel() + + st, err := simpleblob.GetBackend(ctx, conf.Storage.Type, conf.Storage.Options) + if err != nil { + return err + } + + prefix, err := cmd.Flags().GetString("prefix") + if err != nil { + return err + } + long, err := cmd.Flags().GetBool("long") + if err != nil { + return err + } + byTime, err := cmd.Flags().GetBool("time") + if err != nil { + return err + } + + list, err := st.List(ctx, prefix) + if err != nil { + return err + } + if byTime { + sortByTime(list) + } + + for _, blob := range list { + if long { + fmt.Printf("%12d\t%s\n", blob.Size, blob.Name) + } else { + fmt.Printf("%s\n", blob.Name) + } + } + return nil + }, +} + +var snapshotsRemoveCmd = &cobra.Command{ + Use: "remove", + Short: "Remove snapshot", + Args: cobra.ExactArgs(1), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithTimeout(rootCtx, time.Minute) + defer cancel() + + st, err := simpleblob.GetBackend(ctx, conf.Storage.Type, conf.Storage.Options) + if err != nil { + return err + } + + return st.Delete(ctx, args[0]) + }, +} + +var snapshotsDumpCmd = &cobra.Command{ + Use: "dump", + Short: "Dump snapshot contents for debugging", + Args: cobra.ExactArgs(1), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithTimeout(rootCtx, time.Minute) + defer cancel() + + format, err := cmd.Flags().GetString("format") + if err != nil { + return err + } + if format != "debug" && format != "text" { + return fmt.Errorf("output format not supported: %s", format) + } + dbiName, err := cmd.Flags().GetString("dbi") + if err != nil { + return err + } + + // Load snapshot + st, err := simpleblob.GetBackend(ctx, conf.Storage.Type, conf.Storage.Options) + if err != nil { + return err + } + data, err := st.Load(ctx, args[0]) + if err != nil { + return err + } + snap, err := snapshot.LoadData(data) + if err != nil { + return err + } + + // Filter DBIs if needed + if dbiName != "" { + snap.Databases = lo.Filter(snap.Databases, func(item *snapshot.DBI, index int) bool { + return item.Name == dbiName + }) + } + + // Buffered output speeds things up + out := bufio.NewWriter(os.Stdout) + defer out.Flush() + outf := func(sfmt string, args ...any) { + _, _ = fmt.Fprintf(out, sfmt, args...) + } + + switch format { + case "debug": + // Print top level fields using prototext marshaler, so that we + // do not forget any new attributes. + databases := snap.Databases + snap.Databases = nil + tm := proto.TextMarshaler{} + _ = tm.Marshal(out, snap) + + // Print DBI contents + now := time.Now() + for _, dbi := range databases { + outf("\n### %s (transform=%q, flags=%q)\n\n", + dbi.Name, dbi.Transform, dbiflags.Flags(dbi.Flags)) + for _, e := range dbi.Entries { + t := header.Timestamp(e.TimestampNano).Time() + outf("%s = %s (%s, %s ago; flags=%02x)\n", + utils.DisplayASCII(e.Key), + utils.DisplayASCII(e.Value), + t, + now.Sub(t).Round(time.Second), + e.Flags, + ) + } + } + return nil + case "text": + tm := proto.TextMarshaler{} + return tm.Marshal(out, snap) + default: + panic("unhandled output format: " + format) + } + }, +} + +var snapshotsGetCmd = &cobra.Command{ + Use: "get", + Short: "Download a snapshot", + Args: cobra.ExactArgs(1), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithTimeout(rootCtx, time.Minute) + defer cancel() + + outName, err := cmd.Flags().GetString("output") + if err != nil { + return err + } + if outName == "" { + outName = args[0] + } + + st, err := simpleblob.GetBackend(ctx, conf.Storage.Type, conf.Storage.Options) + if err != nil { + return err + } + data, err := st.Load(ctx, args[0]) + if err != nil { + return err + } + + return os.WriteFile(outName, data, 0666) + }, +} + +var snapshotsPutCmd = &cobra.Command{ + Use: "put", + Short: "Upload a snapshot", + Args: cobra.ExactArgs(1), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithTimeout(rootCtx, time.Minute) + defer cancel() + + name, err := cmd.Flags().GetString("name") + if err != nil { + return err + } + if name == "" { + name = filepath.Base(args[0]) + } + if _, err = snapshot.ParseName(name); err != nil { + return fmt.Errorf("invalid snapshot name (use -n to specify a different one): %v", err) + } + + st, err := simpleblob.GetBackend(ctx, conf.Storage.Type, conf.Storage.Options) + if err != nil { + return err + } + + data, err := os.ReadFile(args[0]) + if err != nil { + return err + } + return st.Store(ctx, name, data) + }, +} + +func sortByTime(list simpleblob.BlobList) { + slices.SortFunc(list, func(a, b simpleblob.Blob) bool { + na, err := snapshot.ParseName(a.Name) + if err != nil { + return true + } + nb, err := snapshot.ParseName(b.Name) + if err != nil { + return false + } + return na.Timestamp.Before(nb.Timestamp) + }) +} From 75e1fec8efce8c3f0d573d9ab325a45d48cabe24 Mon Sep 17 00:00:00 2001 From: Konrad Wojas Date: Wed, 1 Mar 2023 12:34:44 +0800 Subject: [PATCH 2/5] Add docs subcommand to generate markdown docs --- cmd/lightningstream/commands/docs.go | 50 ++++++++++++++++++++++++++++ go.mod | 2 ++ go.sum | 2 ++ 3 files changed, 54 insertions(+) create mode 100644 cmd/lightningstream/commands/docs.go diff --git a/cmd/lightningstream/commands/docs.go b/cmd/lightningstream/commands/docs.go new file mode 100644 index 0000000..1095007 --- /dev/null +++ b/cmd/lightningstream/commands/docs.go @@ -0,0 +1,50 @@ +package commands + +import ( + "bytes" + "io" + "os" + "regexp" + + "github.com/spf13/cobra" + "github.com/spf13/cobra/doc" +) + +func init() { + rootCmd.AddCommand(docsCmd) +} + +var docsCmd = &cobra.Command{ + Use: "docs", + Short: "Generate markdown documentation for all commands to stdout", + Args: cobra.NoArgs, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + return genDocs(rootCmd, os.Stdout) + }, +} + +func genDocs(cmd *cobra.Command, w io.Writer) error { + if cmd.Name() == "completion" { + return nil + } + b := bytes.NewBuffer(nil) + if err := doc.GenMarkdown(cmd, b); err != nil { + return err + } + re := regexp.MustCompile(`(?s)### (SEE ALSO|Options inherited from parent commands).*`) + t := re.ReplaceAll(b.Bytes(), nil) + if _, err := w.Write(t); err != nil { + return err + } + + for _, c := range cmd.Commands() { + //if _, err := fmt.Fprintf(w, "\n\n---\n\n"); err != nil { + // return err + //} + if err := genDocs(c, w); err != nil { + return err + } + } + return nil +} diff --git a/go.mod b/go.mod index e9421d4..a00bbf8 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -52,6 +53,7 @@ require ( github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rs/xid v1.4.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/twitchtv/twirp v8.1.0+incompatible // indirect go.opencensus.io v0.23.0 // indirect diff --git a/go.sum b/go.sum index e9437bf..fb103de 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,7 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -403,6 +404,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= From 68ca5ef40ca2bfa9b2ffbf6ec55ab3e728f08faf Mon Sep 17 00:00:00 2001 From: Konrad Wojas Date: Wed, 1 Mar 2023 12:35:02 +0800 Subject: [PATCH 3/5] dbiflags: extra tests --- lmdbenv/dbiflags/flags_test.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lmdbenv/dbiflags/flags_test.go b/lmdbenv/dbiflags/flags_test.go index a506a5e..1701a0e 100644 --- a/lmdbenv/dbiflags/flags_test.go +++ b/lmdbenv/dbiflags/flags_test.go @@ -27,9 +27,16 @@ func TestFlags_String(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := tt.f.String(); got != tt.want { + got := tt.f.String() + if got != tt.want { t.Errorf("String() = %v, want %v", got, tt.want) } + // Check that we do not modify the underlying flag value, in + // case we accidentally change it to a pointer receiver. + got2 := tt.f.String() + if got != got2 { + t.Errorf("String() output changed, first %v, then %v", got, got2) + } }) } } @@ -55,9 +62,16 @@ func TestFlags_FriendlyString(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := tt.f.FriendlyString(); got != tt.want { + got := tt.f.FriendlyString() + if got != tt.want { t.Errorf("FriendlyString() = %v, want %v", got, tt.want) } + // Check that we do not modify the underlying flag value, in + // case we accidentally change it to a pointer receiver. + got2 := tt.f.FriendlyString() + if got != got2 { + t.Errorf("FriendlyString() output changed, first %v, then %v", got, got2) + } }) } } From 47345d6132da05f12171dafd40ed787eadf3ee20 Mon Sep 17 00:00:00 2001 From: Konrad Wojas Date: Wed, 1 Mar 2023 12:49:59 +0800 Subject: [PATCH 4/5] snapshot command: stable sort, add put --force - Stable sort when sorting invalid snapshot names by time - Add put --force to force upload of invalid snapshot names (e.g. marker files) --- cmd/lightningstream/commands/snapshots.go | 28 +++++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/cmd/lightningstream/commands/snapshots.go b/cmd/lightningstream/commands/snapshots.go index eb55b75..4276b36 100644 --- a/cmd/lightningstream/commands/snapshots.go +++ b/cmd/lightningstream/commands/snapshots.go @@ -11,6 +11,7 @@ import ( "github.com/PowerDNS/simpleblob" "github.com/gogo/protobuf/proto" "github.com/samber/lo" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/exp/slices" "powerdns.com/platform/lightningstream/lmdbenv/dbiflags" @@ -41,6 +42,7 @@ func init() { snapshotsCmd.AddCommand(snapshotsPutCmd) snapshotsPutCmd.Flags().StringP("name", "n", "", "Name to store the snapshot as, if different from the local name") + snapshotsPutCmd.Flags().Bool("force", false, "Force the use of an invalid snapshot name") } var snapshotsCmd = &cobra.Command{ @@ -245,8 +247,18 @@ var snapshotsPutCmd = &cobra.Command{ if name == "" { name = filepath.Base(args[0]) } + force, err := cmd.Flags().GetBool("force") + if err != nil { + return err + } + if _, err = snapshot.ParseName(name); err != nil { - return fmt.Errorf("invalid snapshot name (use -n to specify a different one): %v", err) + if !force { + return fmt.Errorf( + "invalid snapshot name (use -n to specify a different one, or "+ + "--force to skip this check): %v", err) + } + logrus.WithError(err).Warn("Invalid snapshot name forced") } st, err := simpleblob.GetBackend(ctx, conf.Storage.Type, conf.Storage.Options) @@ -264,14 +276,20 @@ var snapshotsPutCmd = &cobra.Command{ func sortByTime(list simpleblob.BlobList) { slices.SortFunc(list, func(a, b simpleblob.Blob) bool { - na, err := snapshot.ParseName(a.Name) - if err != nil { + na, errA := snapshot.ParseName(a.Name) + nb, errB := snapshot.ParseName(b.Name) + // Invalid names are sorted by name + if errA != nil && errB != nil { + return a.Name < b.Name + } + // Invalid names come before valid names + if errA != nil { return true } - nb, err := snapshot.ParseName(b.Name) - if err != nil { + if errB != nil { return false } + // Valid names are sorted by timestamp return na.Timestamp.Before(nb.Timestamp) }) } From 78c3b27bb02dc06b550f91074b75f56b2fd5a03a Mon Sep 17 00:00:00 2001 From: Konrad Wojas Date: Wed, 1 Mar 2023 13:52:25 +0800 Subject: [PATCH 5/5] Add snapshots dump --local flag to inspect local file --- cmd/lightningstream/commands/snapshots.go | 26 +++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/cmd/lightningstream/commands/snapshots.go b/cmd/lightningstream/commands/snapshots.go index 4276b36..b0ce454 100644 --- a/cmd/lightningstream/commands/snapshots.go +++ b/cmd/lightningstream/commands/snapshots.go @@ -34,6 +34,8 @@ func init() { snapshotsDumpCmd.Flags().StringP("format", "f", "debug", "Output format, one of: 'debug' (default), 'text'") snapshotsDumpCmd.Flags().StringP("dbi", "d", "", "Only output DBI with this exact name") + snapshotsDumpCmd.Flags().BoolP("local", "l", false, + "Dump a local file instead of a remote snapshot") snapshotsCmd.AddCommand(snapshotsGetCmd) snapshotsGetCmd.Flags().StringP("output", "o", "", @@ -137,15 +139,27 @@ var snapshotsDumpCmd = &cobra.Command{ if err != nil { return err } - - // Load snapshot - st, err := simpleblob.GetBackend(ctx, conf.Storage.Type, conf.Storage.Options) + local, err := cmd.Flags().GetBool("local") if err != nil { return err } - data, err := st.Load(ctx, args[0]) - if err != nil { - return err + + // Load snapshot + var data []byte + if local { + data, err = os.ReadFile(args[0]) + if err != nil { + return err + } + } else { + st, err := simpleblob.GetBackend(ctx, conf.Storage.Type, conf.Storage.Options) + if err != nil { + return err + } + data, err = st.Load(ctx, args[0]) + if err != nil { + return err + } } snap, err := snapshot.LoadData(data) if err != nil {